@sous-io/sous 0.2.36 → 1.0.1

package/src/lib/repos/providers/index-cache.ts

@@ -1,421 +0,0 @@
-/**
- * The index cache.
- *
- * Adding a repository fetches exactly one file, its `sous.index.json`, and that
- * file is all sous needs to resolve a ref, enumerate versions and decide what to
- * download. The cache keeps one copy per repository under
- * `<storeRoot>/_indexes/`, beside a small sidecar recording when it was fetched
- * and what the host called that version of it.
- *
- * The rule the whole Repositories system follows applies here too: a failed
- * upstream check never breaks a build. When a refetch fails and a cached copy
- * exists, the cached copy is used and the failure is reported as a warning.
- *
- * One thing the cache can do beyond storing and serving: an OVERLAY. Sous ships
- * a copy of the core recipe inside its own package, and that copy has to be
- * resolvable even when the repository publishing it has not published that
- * version yet. An overlay folds such a fact into every copy of an index the
- * cache hands out, and never into the file on disk, so the cached copy stays an
- * honest record of what the repository actually served.
- *
- * The cache takes the store's root directory as a plain string, so it does not
- * depend on the store implementation at all.
- */
-
-import fs from "node:fs";
-import path from "node:path";
-import { parseIndexFile, type IndexFile } from "../formats/index-file.js";
-import { INDEX_CACHE_DIRNAME, stableJsonStringify } from "../formats/common.js";
-import { identitySegments } from "../identity.js";
-import { ConfigError, isConfigError } from "../../errors.js";
-import { warning } from "../../../utils/formatting.js";
-// The freshness window has one definition, and it lives with the rest of the
-// freshness rules; that module only borrows a type from here, so nothing loads
-// in a circle at run time.
-import { DEFAULT_FRESHNESS_SECONDS } from "../store/settings.js";
-import type { ProviderOptions, RepoProvider } from "./provider.js";
-import { ensureIndexCacheDirectory } from "../../../utils/sous-directory.js";
-
-export { INDEX_CACHE_DIRNAME };
-
-/** The suffix of the sidecar written beside each cached index. */
-export const INDEX_SIDECAR_SUFFIX = ".meta.json";
-
-/**
- * What sous remembers about a cached index. Written as JSON beside the index
- * itself; a missing or unreadable sidecar simply means "no idea", never an
- * error.
- */
-export type IndexMeta = {
-  /** When the cached copy was fetched. */
-  fetchedAt: string;
-  /** The entity tag the host sent for that copy, when it sent one. */
-  etag?: string;
-  /** The git ref it was read at. */
-  ref?: string;
-  /** When sous last asked upstream whether there was anything newer. */
-  lastCheckedAt?: string;
-};
-
-/** Where a returned index came from. */
-export type IndexSource = "cache" | "network" | "stale";
-
-/** What a cache lookup returns. */
-export type IndexLookup = {
-  /** The validated index. */
-  index: IndexFile;
-  /** Whether it came from the cache, from the network, or from a stale fallback. */
-  source: IndexSource;
-  /** What sous remembers about the copy that was returned. */
-  meta: IndexMeta;
-};
-
-/**
- * A fact sous knows about a repository that its published index does not carry,
- * folded into every copy of that index the cache returns.
- *
- * An overlay must be pure: it is handed an index and returns one, and it never
- * touches the cache's files. It is called for every repository, so it decides
- * for itself which identity it applies to and returns the index unchanged for
- * all the others.
- */
-export type IndexOverlay = (identity: string, index: IndexFile) => IndexFile;
-
-/** Which repository to fetch, and how fresh the answer has to be. */
-export type GetIndexOptions = {
-  /** The repository URL, used when a fetch is needed. */
-  url: string;
-  /** The provider the repository entry names, when it names one. */
-  provider?: string;
-  /** How old a cached copy may be, in seconds. Defaults to five minutes. */
-  maxAgeSeconds?: number;
-  /** When true, refetch regardless of how fresh the cached copy is. */
-  force?: boolean;
-  /**
-   * What to call the repository in a message. A project's short name for it is
-   * what the person reading wrote in their own config, so that is what they are
-   * shown; the identity is used when nothing supplies one.
-   */
-  label?: string;
-  /** Cancels the request, for a caller that will not wait past a deadline. */
-  signal?: AbortSignal;
-};
-
-/** Which repository to read from upstream, without touching the cache. */
-export type FetchUpstreamOptions = Pick<GetIndexOptions, "url" | "provider" | "signal">;
-
-/** How the cache is built. */
-export type IndexCacheOptions = {
-  /** The store's root directory; the cache lives in a subdirectory of it. */
-  storeRoot: string;
-  /**
-   * How a provider is chosen for a repository URL. Defaults to the built-in
-   * providers; injected in tests, and by anyone with a provider list of their
-   * own.
-   */
-  resolveProvider: (url: string, providerId?: string) => RepoProvider;
-  /** Options handed to every provider call (environment, fetch, subprocess runner). */
-  providerOptions?: ProviderOptions;
-  /** The clock, so tests can decide what "now" means. */
-  now?: () => Date;
-  /** Where warnings go. Defaults to the console warning banner. */
-  warn?: (message: string) => void;
-  /**
-   * A fact to fold into every index this cache returns, without ever writing it
-   * to disk. See `IndexOverlay`; `setOverlay` installs one later.
-   */
-  overlay?: IndexOverlay;
-};
-
-/** One cached repository index, plus the machinery to keep it current. */
-export class IndexCache {
-  private readonly storeRoot: string;
-
-  private readonly resolveProvider: (url: string, providerId?: string) => RepoProvider;
-
-  private readonly providerOptions: ProviderOptions;
-
-  private readonly now: () => Date;
-
-  private readonly warn: (message: string) => void;
-
-  /**
-   * What sous itself knows about a repository, over and above what the
-   * repository published. Not readonly: the packaged core recipe's hash is only
-   * known once the store has been seeded, which happens after the cache exists.
-   */
-  private overlay: IndexOverlay | undefined;
-
-  constructor(options: IndexCacheOptions) {
-    this.storeRoot = options.storeRoot;
-    this.resolveProvider = options.resolveProvider;
-    this.providerOptions = options.providerOptions ?? {};
-    this.now = options.now ?? (() => new Date());
-    this.warn = options.warn ?? warning;
-    this.overlay = options.overlay;
-  }
-
-  /**
-   * Installs the overlay folded into every index this cache returns from now
-   * on. Nothing already written to disk changes, and nothing written later
-   * carries it.
-   *
-   * @param overlay - The overlay to apply, or undefined to apply none.
-   */
-  setOverlay(overlay: IndexOverlay | undefined): void {
-    this.overlay = overlay;
-  }
-
-  /**
-   * Folds the installed overlay, if there is one, into an index on its way out.
-   * An overlay that throws is ignored and warned about: a fact sous was trying
-   * to add is never worth losing the index the repository really published.
-   *
-   * @param identity - The repository's canonical identity.
-   * @param index - The index as it was read or fetched.
-   */
-  private applyOverlay(identity: string, index: IndexFile): IndexFile {
-    if (this.overlay === undefined) return index;
-    try {
-      return this.overlay(identity, index);
-    } catch (error) {
-      this.warn(
-        `Sous could not add what it knows about the repository '${identity}' to that ` +
-          `repository's index, so only what the repository published is available.\n` +
-          `${error instanceof Error ? error.message : String(error)}`
-      );
-      return index;
-    }
-  }
-
-  /** The directory cached indexes live in. */
-  get directory(): string {
-    return path.join(this.storeRoot, INDEX_CACHE_DIRNAME);
-  }
-
-  /**
-   * Where a repository's cached index is written. The identity is several
-   * segments, so it becomes several directories, exactly as it does in the
-   * store: `_indexes/github.com/sous-io/sous-recipes.json`.
-   *
-   * @param identity - The repository's canonical identity.
-   */
-  indexPath(identity: string): string {
-    const segments = identitySegments(identity);
-    const last = segments.pop() ?? identity;
-    return path.join(this.directory, ...segments, `${last}.json`);
-  }
-
-  /**
-   * Where a repository's index sidecar is written, beside its index.
-   *
-   * @param identity - The repository's canonical identity.
-   */
-  sidecarPath(identity: string): string {
-    const segments = identitySegments(identity);
-    const last = segments.pop() ?? identity;
-    return path.join(this.directory, ...segments, `${last}${INDEX_SIDECAR_SUFFIX}`);
-  }
-
-  /**
-   * Reads what sous remembers about a cached index. Returns undefined when
-   * there is no sidecar or it cannot be read; the sidecar is a convenience, and
-   * losing it only costs a refetch.
-   *
-   * @param identity - The repository's canonical identity.
-   */
-  readMeta(identity: string): IndexMeta | undefined {
-    try {
-      const raw = JSON.parse(fs.readFileSync(this.sidecarPath(identity), "utf8")) as IndexMeta;
-      if (typeof raw?.fetchedAt !== "string") return undefined;
-      return raw;
-    } catch {
-      return undefined;
-    }
-  }
-
-  /**
-   * Writes what sous remembers about a cached index.
-   *
-   * @param identity - The repository's canonical identity.
-   * @param meta - What to record.
-   */
-  writeMeta(identity: string, meta: IndexMeta): void {
-    const file = this.sidecarPath(identity);
-    this.prepareDirectoryFor(file);
-    fs.writeFileSync(file, stableJsonStringify(meta), "utf8");
-  }
-
-  /**
-   * Makes sure the cache directory exists and explains itself, and that the
-   * subdirectories an identity turns into are there too.
-   *
-   * @param filePath - The file about to be written.
-   */
-  private prepareDirectoryFor(filePath: string): void {
-    ensureIndexCacheDirectory(this.directory);
-    fs.mkdirSync(path.dirname(filePath), { recursive: true });
-  }
-
-  /**
-   * Reads and validates the cached index, or undefined when there is none. A
-   * cached file that no longer parses is treated as absent, since it is only a
-   * copy of something upstream still has. Any installed overlay is folded into
-   * the answer; the file itself is left exactly as it was cached.
-   *
-   * @param identity - The repository's canonical identity.
-   */
-  readCached(identity: string): IndexFile | undefined {
-    const file = this.indexPath(identity);
-    let text: string;
-    try {
-      text = fs.readFileSync(file, "utf8");
-    } catch {
-      return undefined;
-    }
-    try {
-      return this.applyOverlay(identity, parseIndexFile(JSON.parse(text), file));
-    } catch {
-      return undefined;
-    }
-  }
-
-  /**
-   * True when the cached copy is younger than the given window.
-   *
-   * @param meta - What sous remembers about the cached copy.
-   * @param maxAgeSeconds - How old the copy may be.
-   */
-  isFresh(meta: IndexMeta | undefined, maxAgeSeconds: number): boolean {
-    if (meta === undefined) return false;
-    const fetched = Date.parse(meta.fetchedAt);
-    if (Number.isNaN(fetched)) return false;
-    const ageSeconds = (this.now().getTime() - fetched) / 1000;
-    return ageSeconds >= 0 && ageSeconds < maxAgeSeconds;
-  }
-
-  /**
-   * Returns a repository's index: the cached copy while it is fresh, otherwise a
-   * fresh fetch through the provider. When the fetch fails and a cached copy
-   * exists, the cached copy is returned and the failure is warned about; when
-   * there is no cached copy, the failure is raised.
-   *
-   * @param identity - The repository's canonical identity.
-   * @param options - The repository URL, its provider, and the freshness window.
-   */
-  async getIndex(identity: string, options: GetIndexOptions): Promise<IndexLookup> {
-    const maxAgeSeconds = options.maxAgeSeconds ?? DEFAULT_FRESHNESS_SECONDS;
-    const meta = this.readMeta(identity);
-
-    if (options.force !== true && this.isFresh(meta, maxAgeSeconds)) {
-      const cached = this.readCached(identity);
-      if (cached !== undefined) return { index: cached, source: "cache", meta: meta! };
-    }
-
-    try {
-      return await this.refresh(identity, options);
-    } catch (error) {
-      const cached = this.readCached(identity);
-      if (cached === undefined) throw error;
-
-      const reason = isConfigError(error)
-        ? (error as ConfigError).message
-        : (error as Error).message;
-      this.warn(
-        `Sous could not check the repository '${options.label ?? identity}' for updates, ` +
-          `so it is using the copy of its index that it already had.\n${reason}`
-      );
-      const stale: IndexMeta = meta ?? { fetchedAt: new Date(0).toISOString() };
-      return { index: cached, source: "stale", meta: stale };
-    }
-  }
-
-  /**
-   * Fetches a repository's index through its provider, validates it, and writes
-   * both the index and its sidecar. Raises rather than falling back; `getIndex`
-   * is where the last-good behavior lives.
-   *
-   * @param identity - The repository's canonical identity.
-   * @param options - The repository URL and its provider.
-   */
-  async refresh(identity: string, options: GetIndexOptions): Promise<IndexLookup> {
-    const { index, fetched } = await this.fetchFromProvider(options);
-    const timestamp = this.now().toISOString();
-    const meta: IndexMeta = {
-      fetchedAt: timestamp,
-      lastCheckedAt: timestamp,
-      ref: fetched.ref,
-      ...(fetched.etag === undefined ? {} : { etag: fetched.etag }),
-    };
-
-    const file = this.indexPath(identity);
-    this.prepareDirectoryFor(file);
-    fs.writeFileSync(file, stableJsonStringify(index), "utf8");
-    this.writeMeta(identity, meta);
-
-    // Written first, overlaid second: the file is what the repository served,
-    // and the overlay exists only in the copy handed back to the caller.
-    return { index: this.applyOverlay(identity, index), source: "network", meta };
-  }
-
-  /**
-   * Reads a repository's index from upstream and returns it, WITHOUT writing
-   * anything: neither the cached copy nor its sidecar changes. This is what a
-   * browsing command reading upstream uses, because only a command that
-   * resolves versions should change what the cache holds. Any installed overlay
-   * is folded into the answer, as it is for a cached copy. A failure is raised;
-   * the caller decides whether a cached copy stands in for it.
-   *
-   * @param identity - The repository's canonical identity, for the overlay.
-   * @param options - The repository URL, its provider, and an optional cancel signal.
-   */
-  async fetchUpstream(identity: string, options: FetchUpstreamOptions): Promise<IndexFile> {
-    const { index } = await this.fetchFromProvider(options);
-    return this.applyOverlay(identity, index);
-  }
-
-  /**
-   * Fetches and validates a repository's index through its provider. Writes
-   * nothing; `refresh` and `fetchUpstream` decide what happens next.
-   *
-   * @param options - The repository URL, its provider, and an optional cancel signal.
-   */
-  private async fetchFromProvider(
-    options: FetchUpstreamOptions
-  ): Promise<{ index: IndexFile; fetched: { ref: string; etag?: string } }> {
-    const provider = this.resolveProvider(options.url, options.provider);
-    const repo = provider.canonicalize(options.url);
-    const fetched = await provider.fetchIndex(
-      repo,
-      options.signal === undefined
-        ? this.providerOptions
-        : { ...this.providerOptions, signal: options.signal }
-    );
-
-    let parsed: unknown;
-    try {
-      parsed = JSON.parse(fetched.text);
-    } catch (error) {
-      throw new ConfigError(
-        `The index that ${options.url} published is not valid JSON.\n` +
-          `  ${(error as Error).message}\n` +
-          `  A repository's index is written by 'sous repo release'; this one may be ` +
-          `damaged or may not be a sous repository at all.`
-      );
-    }
-
-    const index = parseIndexFile(parsed, `${options.url} (${fetched.ref})`);
-    return { index, fetched };
-  }
-
-  /**
-   * Forgets a repository's cached index and sidecar, which is what removing a
-   * repository from a project does.
-   *
-   * @param identity - The repository's canonical identity.
-   */
-  forget(identity: string): void {
-    fs.rmSync(this.indexPath(identity), { force: true });
-    fs.rmSync(this.sidecarPath(identity), { force: true });
-  }
-}

package/src/lib/repos/providers/index.ts

@@ -1,106 +0,0 @@
-/**
- * The provider registry.
- *
- * Import providers from here. This module is the one place that knows which
- * concrete providers sous ships, so `provider.ts` stays free of them and the
- * built-in list has exactly one definition.
- *
- * A new provider is one file: a class extending `ProviderBase` that implements
- * the read path, declares the features it really answers, and overrides the
- * write-path calls it supports. Adding it to `builtInProviders()` below is the
- * only other change; nothing above the provider layer learns its name.
- */
-
-import { LocalProvider } from "./local.js";
-import { GithubProvider } from "./github.js";
-import { GitlabProvider } from "./gitlab.js";
-import { IndexCache, type IndexCacheOptions } from "./index-cache.js";
-import {
-  detectProviderIn,
-  providerByIdIn,
-  requireProviderIn,
-  type RepoProvider,
-} from "./provider.js";
-
-// `runGit` is deliberately not re-exported here: `git-clone.ts` exports a `runGit` of its
-// own and the repos barrel cannot carry two. Import this one from `providers/git.js` directly.
-export { spawnCommand, tryCommand, fetchSubtree } from "./git.js";
-export type { CommandResult, CommandRunner, RunOptions } from "./git.js";
-export * from "./http.js";
-export * from "./provider.js";
-export * from "./base.js";
-export * from "./github.js";
-export * from "./gitlab.js";
-export * from "./local.js";
-export * from "./index-cache.js";
-
-/**
- * The providers sous ships, in the order they are tried. A fresh array every
- * call, so a caller may add to it without affecting anyone else.
- */
-export function builtInProviders(): RepoProvider[] {
-  // The local provider is last, and matches only a local absolute path or a
-  // `file://` URL, so it can never intercept a hosted repository's URL.
-  return [new GithubProvider(), new GitlabProvider(), new LocalProvider()];
-}
-
-/**
- * Finds the provider that handles a repository URL, or undefined when none
- * does. A repository entry may also name its provider outright, which is what a
- * self-hosted instance behind an unfamiliar host name needs; see
- * `requireProvider`.
- *
- * @param url - The repository URL.
- * @param providers - The providers to consider. Defaults to the built-ins.
- */
-export function detectProvider(
-  url: string,
-  providers: RepoProvider[] = builtInProviders()
-): RepoProvider | undefined {
-  return detectProviderIn(url, providers);
-}
-
-/**
- * Looks a provider up by its identifier, or undefined when there is none.
- *
- * @param id - The provider identifier from a repository entry.
- * @param providers - The providers to consider. Defaults to the built-ins.
- */
-export function providerById(
-  id: string,
-  providers: RepoProvider[] = builtInProviders()
-): RepoProvider | undefined {
-  return providerByIdIn(id, providers);
-}
-
-/**
- * Finds the provider for a repository entry: the one it names, otherwise the
- * one that recognizes its URL. Raises a ConfigError when neither works.
- *
- * @param url - The repository URL.
- * @param providerId - The provider named by the entry, when it named one.
- * @param providers - The providers to consider. Defaults to the built-ins.
- */
-export function requireProvider(
-  url: string,
-  providerId?: string,
-  providers: RepoProvider[] = builtInProviders()
-): RepoProvider {
-  return requireProviderIn(url, providerId, providers);
-}
-
-/**
- * Builds an index cache that resolves providers through the built-in list. Pass
- * `resolveProvider` to override that, which is what tests do.
- *
- * @param options - The store root, and anything to override.
- */
-export function createIndexCache(
-  options: Omit<IndexCacheOptions, "resolveProvider"> &
-    Partial<Pick<IndexCacheOptions, "resolveProvider">>
-): IndexCache {
-  return new IndexCache({
-    resolveProvider: (url, providerId) => requireProvider(url, providerId),
-    ...options,
-  });
-}