@sous-io/sous 0.1.0 → 0.2.0

package/src/lib/repos/providers/index-cache.ts

@@ -0,0 +1,382 @@
+/**
+ * The index cache.
+ *
+ * Adding a repository fetches exactly one file, its `sous.index.json`, and that
+ * file is all sous needs to resolve a ref, enumerate versions and decide what to
+ * download. The cache keeps one copy per repository under
+ * `<storeRoot>/_indexes/`, beside a small sidecar recording when it was fetched
+ * and what the host called that version of it.
+ *
+ * The rule the whole Repositories system follows applies here too: a failed
+ * upstream check never breaks a build. When a refetch fails and a cached copy
+ * exists, the cached copy is used and the failure is reported as a warning.
+ *
+ * One thing the cache can do beyond storing and serving: an OVERLAY. Sous ships
+ * a copy of the core recipe inside its own package, and that copy has to be
+ * resolvable even when the repository publishing it has not published that
+ * version yet. An overlay folds such a fact into every copy of an index the
+ * cache hands out, and never into the file on disk, so the cached copy stays an
+ * honest record of what the repository actually served.
+ *
+ * The cache takes the store's root directory as a plain string, so it does not
+ * depend on the store implementation at all.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import { parseIndexFile, type IndexFile } from "../formats/index-file.js";
+import { INDEX_CACHE_DIRNAME, stableJsonStringify } from "../formats/common.js";
+import { identitySegments } from "../identity.js";
+import { ConfigError, isConfigError } from "../../errors.js";
+import { warning } from "../../../utils/formatting.js";
+// The freshness window has one definition, and it lives with the rest of the
+// freshness rules; that module only borrows a type from here, so nothing loads
+// in a circle at run time.
+import { DEFAULT_FRESHNESS_SECONDS } from "../store/settings.js";
+import type { ProviderOptions, RepoProvider } from "./provider.js";
+import { ensureIndexCacheDirectory } from "../../../utils/sous-directory.js";
+
+export { INDEX_CACHE_DIRNAME };
+
+/** The suffix of the sidecar written beside each cached index. */
+export const INDEX_SIDECAR_SUFFIX = ".meta.json";
+
+/**
+ * What sous remembers about a cached index. Written as JSON beside the index
+ * itself; a missing or unreadable sidecar simply means "no idea", never an
+ * error.
+ */
+export type IndexMeta = {
+  /** When the cached copy was fetched. */
+  fetchedAt: string;
+  /** The entity tag the host sent for that copy, when it sent one. */
+  etag?: string;
+  /** The git ref it was read at. */
+  ref?: string;
+  /** When sous last asked upstream whether there was anything newer. */
+  lastCheckedAt?: string;
+};
+
+/** Where a returned index came from. */
+export type IndexSource = "cache" | "network" | "stale";
+
+/** What a cache lookup returns. */
+export type IndexLookup = {
+  /** The validated index. */
+  index: IndexFile;
+  /** Whether it came from the cache, from the network, or from a stale fallback. */
+  source: IndexSource;
+  /** What sous remembers about the copy that was returned. */
+  meta: IndexMeta;
+};
+
+/**
+ * A fact sous knows about a repository that its published index does not carry,
+ * folded into every copy of that index the cache returns.
+ *
+ * An overlay must be pure: it is handed an index and returns one, and it never
+ * touches the cache's files. It is called for every repository, so it decides
+ * for itself which identity it applies to and returns the index unchanged for
+ * all the others.
+ */
+export type IndexOverlay = (identity: string, index: IndexFile) => IndexFile;
+
+/** Which repository to fetch, and how fresh the answer has to be. */
+export type GetIndexOptions = {
+  /** The repository URL, used when a fetch is needed. */
+  url: string;
+  /** The provider the repository entry names, when it names one. */
+  provider?: string;
+  /** How old a cached copy may be, in seconds. Defaults to five minutes. */
+  maxAgeSeconds?: number;
+  /** When true, refetch regardless of how fresh the cached copy is. */
+  force?: boolean;
+  /**
+   * What to call the repository in a message. A project's short name for it is
+   * what the person reading wrote in their own config, so that is what they are
+   * shown; the identity is used when nothing supplies one.
+   */
+  label?: string;
+};
+
+/** How the cache is built. */
+export type IndexCacheOptions = {
+  /** The store's root directory; the cache lives in a subdirectory of it. */
+  storeRoot: string;
+  /**
+   * How a provider is chosen for a repository URL. Defaults to the built-in
+   * providers; injected in tests, and by anyone with a provider list of their
+   * own.
+   */
+  resolveProvider: (url: string, providerId?: string) => RepoProvider;
+  /** Options handed to every provider call (environment, fetch, subprocess runner). */
+  providerOptions?: ProviderOptions;
+  /** The clock, so tests can decide what "now" means. */
+  now?: () => Date;
+  /** Where warnings go. Defaults to the console warning banner. */
+  warn?: (message: string) => void;
+  /**
+   * A fact to fold into every index this cache returns, without ever writing it
+   * to disk. See `IndexOverlay`; `setOverlay` installs one later.
+   */
+  overlay?: IndexOverlay;
+};
+
+/** One cached repository index, plus the machinery to keep it current. */
+export class IndexCache {
+  private readonly storeRoot: string;
+
+  private readonly resolveProvider: (url: string, providerId?: string) => RepoProvider;
+
+  private readonly providerOptions: ProviderOptions;
+
+  private readonly now: () => Date;
+
+  private readonly warn: (message: string) => void;
+
+  /**
+   * What sous itself knows about a repository, over and above what the
+   * repository published. Not readonly: the packaged core recipe's hash is only
+   * known once the store has been seeded, which happens after the cache exists.
+   */
+  private overlay: IndexOverlay | undefined;
+
+  constructor(options: IndexCacheOptions) {
+    this.storeRoot = options.storeRoot;
+    this.resolveProvider = options.resolveProvider;
+    this.providerOptions = options.providerOptions ?? {};
+    this.now = options.now ?? (() => new Date());
+    this.warn = options.warn ?? warning;
+    this.overlay = options.overlay;
+  }
+
+  /**
+   * Installs the overlay folded into every index this cache returns from now
+   * on. Nothing already written to disk changes, and nothing written later
+   * carries it.
+   *
+   * @param overlay - The overlay to apply, or undefined to apply none.
+   */
+  setOverlay(overlay: IndexOverlay | undefined): void {
+    this.overlay = overlay;
+  }
+
+  /**
+   * Folds the installed overlay, if there is one, into an index on its way out.
+   * An overlay that throws is ignored and warned about: a fact sous was trying
+   * to add is never worth losing the index the repository really published.
+   *
+   * @param identity - The repository's canonical identity.
+   * @param index - The index as it was read or fetched.
+   */
+  private applyOverlay(identity: string, index: IndexFile): IndexFile {
+    if (this.overlay === undefined) return index;
+    try {
+      return this.overlay(identity, index);
+    } catch (error) {
+      this.warn(
+        `Sous could not add what it knows about the repository '${identity}' to that ` +
+          `repository's index, so only what the repository published is available.\n` +
+          `${error instanceof Error ? error.message : String(error)}`
+      );
+      return index;
+    }
+  }
+
+  /** The directory cached indexes live in. */
+  get directory(): string {
+    return path.join(this.storeRoot, INDEX_CACHE_DIRNAME);
+  }
+
+  /**
+   * Where a repository's cached index is written. The identity is several
+   * segments, so it becomes several directories, exactly as it does in the
+   * store: `_indexes/github.com/sous-io/sous-recipes.json`.
+   *
+   * @param identity - The repository's canonical identity.
+   */
+  indexPath(identity: string): string {
+    const segments = identitySegments(identity);
+    const last = segments.pop() ?? identity;
+    return path.join(this.directory, ...segments, `${last}.json`);
+  }
+
+  /**
+   * Where a repository's index sidecar is written, beside its index.
+   *
+   * @param identity - The repository's canonical identity.
+   */
+  sidecarPath(identity: string): string {
+    const segments = identitySegments(identity);
+    const last = segments.pop() ?? identity;
+    return path.join(this.directory, ...segments, `${last}${INDEX_SIDECAR_SUFFIX}`);
+  }
+
+  /**
+   * Reads what sous remembers about a cached index. Returns undefined when
+   * there is no sidecar or it cannot be read; the sidecar is a convenience, and
+   * losing it only costs a refetch.
+   *
+   * @param identity - The repository's canonical identity.
+   */
+  readMeta(identity: string): IndexMeta | undefined {
+    try {
+      const raw = JSON.parse(fs.readFileSync(this.sidecarPath(identity), "utf8")) as IndexMeta;
+      if (typeof raw?.fetchedAt !== "string") return undefined;
+      return raw;
+    } catch {
+      return undefined;
+    }
+  }
+
+  /**
+   * Writes what sous remembers about a cached index.
+   *
+   * @param identity - The repository's canonical identity.
+   * @param meta - What to record.
+   */
+  writeMeta(identity: string, meta: IndexMeta): void {
+    const file = this.sidecarPath(identity);
+    this.prepareDirectoryFor(file);
+    fs.writeFileSync(file, stableJsonStringify(meta), "utf8");
+  }
+
+  /**
+   * Makes sure the cache directory exists and explains itself, and that the
+   * subdirectories an identity turns into are there too.
+   *
+   * @param filePath - The file about to be written.
+   */
+  private prepareDirectoryFor(filePath: string): void {
+    ensureIndexCacheDirectory(this.directory);
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  }
+
+  /**
+   * Reads and validates the cached index, or undefined when there is none. A
+   * cached file that no longer parses is treated as absent, since it is only a
+   * copy of something upstream still has. Any installed overlay is folded into
+   * the answer; the file itself is left exactly as it was cached.
+   *
+   * @param identity - The repository's canonical identity.
+   */
+  readCached(identity: string): IndexFile | undefined {
+    const file = this.indexPath(identity);
+    let text: string;
+    try {
+      text = fs.readFileSync(file, "utf8");
+    } catch {
+      return undefined;
+    }
+    try {
+      return this.applyOverlay(identity, parseIndexFile(JSON.parse(text), file));
+    } catch {
+      return undefined;
+    }
+  }
+
+  /**
+   * True when the cached copy is younger than the given window.
+   *
+   * @param meta - What sous remembers about the cached copy.
+   * @param maxAgeSeconds - How old the copy may be.
+   */
+  isFresh(meta: IndexMeta | undefined, maxAgeSeconds: number): boolean {
+    if (meta === undefined) return false;
+    const fetched = Date.parse(meta.fetchedAt);
+    if (Number.isNaN(fetched)) return false;
+    const ageSeconds = (this.now().getTime() - fetched) / 1000;
+    return ageSeconds >= 0 && ageSeconds < maxAgeSeconds;
+  }
+
+  /**
+   * Returns a repository's index: the cached copy while it is fresh, otherwise a
+   * fresh fetch through the provider. When the fetch fails and a cached copy
+   * exists, the cached copy is returned and the failure is warned about; when
+   * there is no cached copy, the failure is raised.
+   *
+   * @param identity - The repository's canonical identity.
+   * @param options - The repository URL, its provider, and the freshness window.
+   */
+  async getIndex(identity: string, options: GetIndexOptions): Promise<IndexLookup> {
+    const maxAgeSeconds = options.maxAgeSeconds ?? DEFAULT_FRESHNESS_SECONDS;
+    const meta = this.readMeta(identity);
+
+    if (options.force !== true && this.isFresh(meta, maxAgeSeconds)) {
+      const cached = this.readCached(identity);
+      if (cached !== undefined) return { index: cached, source: "cache", meta: meta! };
+    }
+
+    try {
+      return await this.refresh(identity, options);
+    } catch (error) {
+      const cached = this.readCached(identity);
+      if (cached === undefined) throw error;
+
+      const reason = isConfigError(error)
+        ? (error as ConfigError).message
+        : (error as Error).message;
+      this.warn(
+        `Sous could not check the repository '${options.label ?? identity}' for updates, ` +
+          `so it is using the copy of its index that it already had.\n${reason}`
+      );
+      const stale: IndexMeta = meta ?? { fetchedAt: new Date(0).toISOString() };
+      return { index: cached, source: "stale", meta: stale };
+    }
+  }
+
+  /**
+   * Fetches a repository's index through its provider, validates it, and writes
+   * both the index and its sidecar. Raises rather than falling back; `getIndex`
+   * is where the last-good behavior lives.
+   *
+   * @param identity - The repository's canonical identity.
+   * @param options - The repository URL and its provider.
+   */
+  async refresh(identity: string, options: GetIndexOptions): Promise<IndexLookup> {
+    const provider = this.resolveProvider(options.url, options.provider);
+    const repo = provider.canonicalize(options.url);
+    const fetched = await provider.fetchIndex(repo, this.providerOptions);
+
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(fetched.text);
+    } catch (error) {
+      throw new ConfigError(
+        `The index that ${options.url} published is not valid JSON.\n` +
+          `  ${(error as Error).message}\n` +
+          `  A repository's index is written by 'sous repo release'; this one may be ` +
+          `damaged or may not be a sous repository at all.`
+      );
+    }
+
+    const index = parseIndexFile(parsed, `${options.url} (${fetched.ref})`);
+    const timestamp = this.now().toISOString();
+    const meta: IndexMeta = {
+      fetchedAt: timestamp,
+      lastCheckedAt: timestamp,
+      ref: fetched.ref,
+      ...(fetched.etag === undefined ? {} : { etag: fetched.etag }),
+    };
+
+    const file = this.indexPath(identity);
+    this.prepareDirectoryFor(file);
+    fs.writeFileSync(file, stableJsonStringify(index), "utf8");
+    this.writeMeta(identity, meta);
+
+    // Written first, overlaid second: the file is what the repository served,
+    // and the overlay exists only in the copy handed back to the caller.
+    return { index: this.applyOverlay(identity, index), source: "network", meta };
+  }
+
+  /**
+   * Forgets a repository's cached index and sidecar, which is what removing a
+   * repository from a project does.
+   *
+   * @param identity - The repository's canonical identity.
+   */
+  forget(identity: string): void {
+    fs.rmSync(this.indexPath(identity), { force: true });
+    fs.rmSync(this.sidecarPath(identity), { force: true });
+  }
+}

package/src/lib/repos/providers/index.ts

@@ -0,0 +1,106 @@
+/**
+ * The provider registry.
+ *
+ * Import providers from here. This module is the one place that knows which
+ * concrete providers sous ships, so `provider.ts` stays free of them and the
+ * built-in list has exactly one definition.
+ *
+ * A new provider is one file: a class extending `ProviderBase` that implements
+ * the read path, declares the features it really answers, and overrides the
+ * write-path calls it supports. Adding it to `builtInProviders()` below is the
+ * only other change; nothing above the provider layer learns its name.
+ */
+
+import { LocalProvider } from "./local.js";
+import { GithubProvider } from "./github.js";
+import { GitlabProvider } from "./gitlab.js";
+import { IndexCache, type IndexCacheOptions } from "./index-cache.js";
+import {
+  detectProviderIn,
+  providerByIdIn,
+  requireProviderIn,
+  type RepoProvider,
+} from "./provider.js";
+
+// `runGit` is deliberately not re-exported here: `git-clone.ts` exports a `runGit` of its
+// own and the repos barrel cannot carry two. Import this one from `providers/git.js` directly.
+export { spawnCommand, tryCommand, fetchSubtree } from "./git.js";
+export type { CommandResult, CommandRunner, RunOptions } from "./git.js";
+export * from "./http.js";
+export * from "./provider.js";
+export * from "./base.js";
+export * from "./github.js";
+export * from "./gitlab.js";
+export * from "./local.js";
+export * from "./index-cache.js";
+
+/**
+ * The providers sous ships, in the order they are tried. A fresh array every
+ * call, so a caller may add to it without affecting anyone else.
+ */
+export function builtInProviders(): RepoProvider[] {
+  // The local provider is last, and matches only a local absolute path or a
+  // `file://` URL, so it can never intercept a hosted repository's URL.
+  return [new GithubProvider(), new GitlabProvider(), new LocalProvider()];
+}
+
+/**
+ * Finds the provider that handles a repository URL, or undefined when none
+ * does. A repository entry may also name its provider outright, which is what a
+ * self-hosted instance behind an unfamiliar host name needs; see
+ * `requireProvider`.
+ *
+ * @param url - The repository URL.
+ * @param providers - The providers to consider. Defaults to the built-ins.
+ */
+export function detectProvider(
+  url: string,
+  providers: RepoProvider[] = builtInProviders()
+): RepoProvider | undefined {
+  return detectProviderIn(url, providers);
+}
+
+/**
+ * Looks a provider up by its identifier, or undefined when there is none.
+ *
+ * @param id - The provider identifier from a repository entry.
+ * @param providers - The providers to consider. Defaults to the built-ins.
+ */
+export function providerById(
+  id: string,
+  providers: RepoProvider[] = builtInProviders()
+): RepoProvider | undefined {
+  return providerByIdIn(id, providers);
+}
+
+/**
+ * Finds the provider for a repository entry: the one it names, otherwise the
+ * one that recognizes its URL. Raises a ConfigError when neither works.
+ *
+ * @param url - The repository URL.
+ * @param providerId - The provider named by the entry, when it named one.
+ * @param providers - The providers to consider. Defaults to the built-ins.
+ */
+export function requireProvider(
+  url: string,
+  providerId?: string,
+  providers: RepoProvider[] = builtInProviders()
+): RepoProvider {
+  return requireProviderIn(url, providerId, providers);
+}
+
+/**
+ * Builds an index cache that resolves providers through the built-in list. Pass
+ * `resolveProvider` to override that, which is what tests do.
+ *
+ * @param options - The store root, and anything to override.
+ */
+export function createIndexCache(
+  options: Omit<IndexCacheOptions, "resolveProvider"> &
+    Partial<Pick<IndexCacheOptions, "resolveProvider">>
+): IndexCache {
+  return new IndexCache({
+    resolveProvider: (url, providerId) => requireProvider(url, providerId),
+    ...options,
+  });
+}