@sous-io/sous 0.2.14 → 0.2.16

package/src/lib/repos/catalog-inputs.ts

@@ -8,15 +8,18 @@
  * is pinned, the links map and the store for a recipe's own files, and
  * `recipeOutputs` for where a content kind lands.
  *
- * Nothing here downloads anything. A repository whose index has never been
- * fetched is left out of the catalog and named separately, so a browsing command
- * is safe offline.
+ * By default nothing here downloads anything: a repository whose index has
+ * never been fetched is left out of the catalog and named separately, so a
+ * browsing command is safe offline. Asked for the latest, it reads each index
+ * from upstream instead and writes none of it to the cache; a repository that
+ * cannot be reached is read from the cache and named as not checked.
  */
 
 import type { Settings, VarScope } from "../settings.js";
 import type { SubscriptionService } from "./subscription-service.js";
 import type { CatalogInputs, CatalogRepo } from "./catalog.js";
-import { linkedPathFor } from "./links.js";
+import type { IndexFile } from "./formats/index-file.js";
+import { linkedPathFor, readEffectiveLinks } from "./links.js";
 import { mapLinkedRecipes, readRecipeManifestIn } from "./locked-recipes.js";
 import { WRITABLE_CONTENT_KINDS, destinationsFor } from "./recipe-targets.js";
 import type { WritableContentKind } from "./recipe-targets.js";
@@ -37,8 +40,94 @@ export type CatalogInputsOptions = {
   scope?: VarScope;
   /** The environment to read; decides where the store and the links map are. */
   env?: NodeJS.ProcessEnv;
+  /**
+   * The indexes to read, when the caller has already gathered them (with
+   * `readTrustedIndexes`, say, to read upstream). The cached indexes otherwise.
+   */
+  indexes?: TrustedIndexes;
+};
+
+/** Where to read the trusted repositories' indexes from. */
+export type TrustedIndexesOptions = {
+  /** Read each index from upstream rather than the cache, writing none of it. */
+  latest?: boolean;
 };
 
+/** One trusted repository's index, and where it was read from. */
+export type TrustedIndex = {
+  /** The repository's short name. */
+  name: string;
+  /** Where it lives, as the project's config records it. */
+  url?: string;
+  /** The index that was read. */
+  index: IndexFile;
+  /** Whether it came from upstream just now or from the cache. */
+  source: "upstream" | "cache";
+};
+
+/** Every trusted repository's index sous could read, and what it could not. */
+export type TrustedIndexes = {
+  /** The indexes, by repository short name, sorted. */
+  repos: TrustedIndex[];
+  /**
+   * Trusted repositories with no index at all: never fetched, and (when the
+   * latest was asked for) not reachable either. Sorted.
+   */
+  notFetched: string[];
+  /**
+   * Repositories the latest was asked for that could not be reached, and were
+   * read from the cache instead. Sorted; always empty when reading the cache.
+   */
+  notChecked: string[];
+};
+
+/**
+ * Reads the index of every repository this project trusts. From the cache by
+ * default, which downloads nothing. With `latest`, from upstream, all at once,
+ * and nothing fetched is written to the cache: only a command that resolves
+ * versions changes what the cache holds. A repository upstream cannot answer
+ * for is read from the cache and named in `notChecked`.
+ *
+ * @param service - The subscription service for this project.
+ * @param options - Whether to read upstream.
+ */
+export async function readTrustedIndexes(
+  service: SubscriptionService,
+  options: TrustedIndexesOptions = {}
+): Promise<TrustedIndexes> {
+  if (options.latest !== true) return cachedTrustedIndexes(service);
+
+  const trusted = service.currentRepos();
+  const names = Object.keys(trusted).sort();
+  const answers = await Promise.allSettled(names.map((name) => service.upstreamIndex(name)));
+
+  const result: TrustedIndexes = { repos: [], notFetched: [], notChecked: [] };
+  names.forEach((name, position) => {
+    const url = trusted[name]?.url;
+    const answer = answers[position]!;
+    if (answer.status === "fulfilled") {
+      result.repos.push({
+        name,
+        ...(url === undefined ? {} : { url }),
+        index: answer.value,
+        source: "upstream",
+      });
+      return;
+    }
+
+    // Upstream could not answer, so the cached copy stands in and says so. A
+    // repository with no cached copy either is named once, as never fetched.
+    const cached = service.cachedIndex(name);
+    if (cached === undefined) {
+      result.notFetched.push(name);
+      return;
+    }
+    result.notChecked.push(name);
+    result.repos.push({ name, ...(url === undefined ? {} : { url }), index: cached, source: "cache" });
+  });
+  return result;
+}
+
 /** The catalog's inputs, plus what could not be read. */
 export type CatalogContext = {
   /** What the catalog functions read. */
@@ -48,6 +137,11 @@ export type CatalogContext = {
    * could be listed. Sorted.
    */
   notFetched: string[];
+  /**
+   * Repositories the latest was asked for that could not be reached, listed
+   * from the cache instead. Sorted.
+   */
+  notChecked: string[];
 };
 
 /**
@@ -60,25 +154,23 @@ export type CatalogContext = {
 export function catalogContextFor(options: CatalogInputsOptions): CatalogContext {
   const { service } = options;
   const env = options.env ?? process.env;
+  const indexes = options.indexes ?? cachedTrustedIndexes(service);
 
-  const repos: CatalogRepo[] = [];
-  const notFetched: string[] = [];
+  const repos: CatalogRepo[] = indexes.repos.map((entry) => ({
+    name: entry.name,
+    ...(entry.url === undefined ? {} : { url: entry.url }),
+    index: entry.index,
+  }));
 
-  const trusted = service.currentRepos();
-  for (const name of Object.keys(trusted).sort()) {
-    const index = service.cachedIndex(name);
-    if (index === undefined) {
-      notFetched.push(name);
-      continue;
-    }
-    const url = trusted[name]?.url;
-    repos.push({ name, ...(url === undefined ? {} : { url }), index });
-  }
+  const links = readEffectiveLinks(options.sousDir, env);
+  const linked: Record<string, string> = {};
+  for (const [name, link] of Object.entries(links)) linked[name] = link.path;
 
   const inputs: CatalogInputs = {
     repos,
     lock: service.lockService.read(),
     subscriptions: Object.keys(service.allSubscriptions()).sort(),
+    linked,
     readManifest: (recipe) => {
       const directory = recipeFilesDirectory({
         service,
@@ -105,7 +197,43 @@ export function catalogContextFor(options: CatalogInputsOptions): CatalogContext
     },
   };
 
-  return { inputs, notFetched };
+  return { inputs, notFetched: indexes.notFetched, notChecked: indexes.notChecked };
+}
+
+/**
+ * The catalog's inputs, reading the indexes the way the options say: from the
+ * cache, or with `latest` from upstream without writing to the cache.
+ *
+ * @param options - The subscription service, the project's directory and config,
+ *   and whether to read upstream.
+ */
+export async function loadCatalogContext(
+  options: Omit<CatalogInputsOptions, "indexes"> & TrustedIndexesOptions
+): Promise<CatalogContext> {
+  const indexes = await readTrustedIndexes(options.service, {
+    ...(options.latest === undefined ? {} : { latest: options.latest }),
+  });
+  return catalogContextFor({ ...options, indexes });
+}
+
+/**
+ * Every trusted repository's cached index, read synchronously.
+ *
+ * @param service - The subscription service for this project.
+ */
+function cachedTrustedIndexes(service: SubscriptionService): TrustedIndexes {
+  const trusted = service.currentRepos();
+  const result: TrustedIndexes = { repos: [], notFetched: [], notChecked: [] };
+  for (const name of Object.keys(trusted).sort()) {
+    const index = service.cachedIndex(name);
+    if (index === undefined) {
+      result.notFetched.push(name);
+      continue;
+    }
+    const url = trusted[name]?.url;
+    result.repos.push({ name, ...(url === undefined ? {} : { url }), index, source: "cache" });
+  }
+  return result;
 }
 
 /** Which recipe, at which version, in which of this project's repositories. */

package/src/lib/repos/catalog.ts

@@ -78,6 +78,11 @@ export type CatalogInputs = {
   lock: Lockfile;
   /** The ref keys the project subscribes to: namespaces, and `namespace/recipe`. */
   subscriptions: string[];
+  /**
+   * The repositories read from a linked working copy instead of the store, by
+   * short name, each with the checkout's path.
+   */
+  linked?: Record<string, string>;
   /**
    * Reads one published recipe's manifest, when its files are on this machine.
    * Returning undefined means "not available", and the recipe is described from
@@ -119,6 +124,11 @@ export type RecipeListing = {
   latest?: string;
   /** The version this project's lockfile pins, when it pins one. */
   pinned?: string;
+  /**
+   * The linked checkout builds read this recipe from instead of the pinned
+   * version, when the project pins it and its repository is linked.
+   */
+  linkedPath?: string;
   /** True when the project subscribes to this recipe, or to the whole namespace holding it. */
   subscribed: boolean;
   /** The recipe's one-paragraph summary, when its index carries one. */
@@ -212,6 +222,11 @@ export type RecipeDetail = {
   latest?: string;
   /** The version this project's lockfile pins, when it pins one. */
   pinned?: string;
+  /**
+   * The linked checkout builds read this recipe from instead of the pinned
+   * version, when the project pins it and its repository is linked.
+   */
+  linkedPath?: string;
   /** True when the project subscribes to this recipe, or to the whole namespace holding it. */
   subscribed: boolean;
   /**
@@ -275,7 +290,7 @@ export function listRecipes(inputs: CatalogInputs): RecipeListing[] {
 
   for (const repo of inputs.repos) {
     for (const key of Object.keys(repo.index.recipes)) {
-      listings.push(recipeListing(key, repo, inputs.lock, subscriptions));
+      listings.push(recipeListing(key, repo, inputs, subscriptions));
     }
   }
 
@@ -305,7 +320,7 @@ export function describeNamespace(inputs: CatalogInputs, ref: string): Namespace
     ...(declared.description === undefined ? {} : { description: declared.description }),
     subscribed: coverageOf(found.namespace, found.repo.index, subscriptions),
     recipes: recipeKeysIn(found.repo.index, found.namespace).map((key) =>
-      recipeListing(key, found.repo, inputs.lock, subscriptions)
+      recipeListing(key, found.repo, inputs, subscriptions)
     ),
   };
 }
@@ -320,7 +335,7 @@ export function describeNamespace(inputs: CatalogInputs, ref: string): Namespace
 export function describeRecipe(inputs: CatalogInputs, ref: string): RecipeDetail {
   const found = resolveRecipeRef(inputs, ref);
   const subscriptions = new Set(inputs.subscriptions);
-  const listing = recipeListing(found.key, found.repo, inputs.lock, subscriptions);
+  const listing = recipeListing(found.key, found.repo, inputs, subscriptions);
   const entry = found.repo.index.recipes[found.key]!;
 
   const describing = listing.pinned ?? listing.latest;
@@ -355,6 +370,75 @@ export function describeRecipe(inputs: CatalogInputs, ref: string): RecipeDetail
   };
 }
 
+/**
+ * The same inputs, narrowed to what this project has installed: each
+ * repository's index keeps only the recipes the lockfile pins from that
+ * repository, and only the namespaces holding one of them. Every listing and
+ * every ref lookup over the result therefore sees only installed recipes. A
+ * recipe's published version history is kept whole, so the installed version
+ * still sits among the others.
+ *
+ * narrowToInstalled(inputs)
+ * // -> the index of "r" holds "a/x" only, when the lockfile pins "a/x" from "r"
+ *
+ * @param inputs - The catalog's inputs.
+ */
+export function narrowToInstalled(inputs: CatalogInputs): CatalogInputs {
+  const repos = inputs.repos.map((repo) => {
+    const recipes = Object.fromEntries(
+      Object.entries(repo.index.recipes).filter(
+        ([key]) => inputs.lock.recipes[key]?.repo === repo.name
+      )
+    );
+    const held = new Set(Object.keys(recipes).map((key) => key.slice(0, key.indexOf("/"))));
+    const namespaces = Object.fromEntries(
+      Object.entries(repo.index.namespaces).filter(([namespace]) => held.has(namespace))
+    );
+    return { ...repo, index: { ...repo.index, recipes, namespaces } };
+  });
+  return { ...inputs, repos };
+}
+
+/**
+ * Looks a ref up among the installed recipes only, so an ambiguity between an
+ * installed recipe and one the project does not use settles itself. A ref that
+ * names something published but not installed is an error saying exactly that,
+ * rather than the "no repository publishes" error the narrowed lookup alone
+ * would raise.
+ *
+ * describeInstalled(inputs, "workflow/task-files", describeRecipe, "recipe")
+ * // -> the recipe, when the lockfile pins it; otherwise an error saying the
+ * //    project has not installed it
+ *
+ * @param inputs - The catalog's inputs, not yet narrowed.
+ * @param ref - The ref being looked up.
+ * @param describe - The lookup to run: `describeNamespace` or `describeRecipe`.
+ * @param what - What the ref names, for the error.
+ */
+export function describeInstalled<T>(
+  inputs: CatalogInputs,
+  ref: string,
+  describe: (inputs: CatalogInputs, ref: string) => T,
+  what: "namespace" | "recipe"
+): T {
+  try {
+    return describe(narrowToInstalled(inputs), ref);
+  } catch {
+    // The full lookup either explains the ref better (ambiguous, unknown) or
+    // proves it is published and simply not installed.
+    describe(inputs, ref);
+    throw what === "namespace"
+      ? new ConfigError(
+          `This project has installed no recipe from the namespace '${ref}', so there is ` +
+            `nothing to show with '--installed'.`
+        )
+      : new ConfigError(
+          `This project has not installed the recipe '${ref}', so there is nothing to ` +
+            `show with '--installed'.`
+        );
+  }
+}
+
 // --- Resolving a ref ----------------------------------------------------------------------------
 
 /** One namespace a ref resolved to. */
@@ -488,15 +572,16 @@ function coverageOf(
  *
  * @param key - The recipe key.
  * @param repo - The repository publishing it.
- * @param lock - The project's lockfile.
+ * @param inputs - The lockfile and the linked repositories.
  * @param subscriptions - The ref keys the project subscribes to.
  */
 function recipeListing(
   key: string,
   repo: CatalogRepo,
-  lock: Lockfile,
+  inputs: Pick<CatalogInputs, "lock" | "linked">,
   subscriptions: Set<string>
 ): RecipeListing {
+  const lock = inputs.lock;
   const entry = repo.index.recipes[key]!;
   const namespace = key.slice(0, key.indexOf("/"));
   const name = key.slice(namespace.length + 1);
@@ -507,6 +592,7 @@ function recipeListing(
   // the recipe came from this repository.
   const locked = lock.recipes[key];
   const pinned = locked !== undefined && locked.repo === repo.name ? locked.version : undefined;
+  const linkedPath = pinned === undefined ? undefined : inputs.linked?.[repo.name];
 
   return {
     key,
@@ -515,6 +601,7 @@ function recipeListing(
     repo: repo.name,
     ...(latest === undefined ? {} : { latest }),
     ...(pinned === undefined ? {} : { pinned }),
+    ...(linkedPath === undefined ? {} : { linkedPath }),
     subscribed: subscriptions.has(key) || subscriptions.has(namespace),
     ...(entry.description === undefined ? {} : { description: entry.description }),
   };

package/src/lib/repos/freshness.ts

@@ -206,3 +206,59 @@ export function effectiveRangeForHolders(
   const combined = constraints.join(" ");
   return semver.validRange(combined) === null ? undefined : combined;
 }
+
+/**
+ * How long a build waits for a repository that it is only checking so it can
+ * say whether a newer version exists: three seconds. Past that the check is
+ * abandoned, quietly, and the build goes on with what the cache already knew.
+ */
+export const NEWER_VERSION_CHECK_TIMEOUT_MS = 3000;
+
+/**
+ * Runs a piece of upstream work with a deadline. The work is handed an abort
+ * signal that fires at the deadline, so a request that honors it is cancelled
+ * rather than left to keep the process alive; either way the returned promise
+ * rejects at the deadline, and the timer never holds the process open itself.
+ *
+ * await withDeadline((signal) => fetchSomething(signal), 3000);
+ * // -> the result, or an error saying the repository did not answer in time
+ *
+ * @param work - The work to run, given the signal that cancels it.
+ * @param milliseconds - How long to wait.
+ */
+export async function withDeadline<T>(
+  work: (signal: AbortSignal) => Promise<T>,
+  milliseconds: number
+): Promise<T> {
+  const controller = new AbortController();
+  let timer: ReturnType<typeof setTimeout> | undefined;
+
+  const deadline = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => {
+      controller.abort();
+      reject(
+        new Error(
+          `The repository did not answer within ${formatSeconds(milliseconds)}, so sous ` +
+            `stopped waiting for it.`
+        )
+      );
+    }, milliseconds);
+    timer.unref?.();
+  });
+
+  try {
+    return await Promise.race([work(controller.signal), deadline]);
+  } finally {
+    if (timer !== undefined) clearTimeout(timer);
+  }
+}
+
+/**
+ * A duration in plain words, such as "3 seconds" or "1 second".
+ *
+ * @param milliseconds - The duration.
+ */
+function formatSeconds(milliseconds: number): string {
+  const seconds = Math.round((milliseconds / 1000) * 10) / 10;
+  return seconds === 1 ? "1 second" : `${seconds} seconds`;
+}