docs-cache 0.4.3 → 0.5.2

package/dist/esm/config/schema.mjs

@@ -0,0 +1,69 @@
+import * as z from "zod";
+import { assertSafeSourceId } from "#core/source-id";
+export const TargetModeSchema = z.enum(["symlink", "copy"]);
+export const CacheModeSchema = z.enum(["materialize"]);
+export const TocFormatSchema = z.enum(["tree", "compressed"]);
+export const IntegritySchema = z.object({
+  type: z.enum(["commit", "manifest"]),
+  value: z.string().nullable()
+}).strict();
+const CommonOptionsSchema = z.object({
+  ref: z.string().min(1),
+  mode: CacheModeSchema,
+  include: z.array(z.string().min(1)).min(1),
+  exclude: z.array(z.string().min(1)).optional(),
+  targetMode: TargetModeSchema.optional(),
+  required: z.boolean(),
+  maxBytes: z.number().min(1),
+  maxFiles: z.number().min(1).optional(),
+  ignoreHidden: z.boolean(),
+  toc: z.union([z.boolean(), TocFormatSchema]).optional(),
+  unwrapSingleRootDir: z.boolean().optional()
+});
+export const DefaultsSchema = CommonOptionsSchema.extend({
+  allowHosts: z.array(z.string().min(1)).min(1)
+}).strict();
+export const SourceSchema = CommonOptionsSchema.partial().extend({
+  id: z.string().min(1).superRefine((value, ctx) => {
+    try {
+      assertSafeSourceId(value, "id");
+    } catch (error) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: error instanceof Error ? error.message : "Invalid source id."
+      });
+    }
+  }),
+  repo: z.string().min(1),
+  targetDir: z.string().min(1).optional(),
+  integrity: IntegritySchema.optional()
+}).extend({
+  include: z.array(z.string().min(1)).min(1, { message: "include must be a non-empty array" }).optional()
+}).strict();
+export const ResolvedSourceSchema = SourceSchema.extend(
+  CommonOptionsSchema.shape
+).strict();
+export const ConfigSchema = z.object({
+  $schema: z.string().min(1).optional(),
+  cacheDir: z.string().min(1).optional(),
+  targetMode: TargetModeSchema.optional(),
+  defaults: DefaultsSchema.partial().optional(),
+  sources: z.array(SourceSchema)
+}).strict().superRefine((value, ctx) => {
+  const seen = /* @__PURE__ */ new Set();
+  const duplicates = /* @__PURE__ */ new Set();
+  value.sources.forEach((source) => {
+    if (seen.has(source.id)) {
+      duplicates.add(source.id);
+    } else {
+      seen.add(source.id);
+    }
+  });
+  if (duplicates.size > 0) {
+    ctx.addIssue({
+      code: z.ZodIssueCode.custom,
+      path: ["sources"],
+      message: `Duplicate source IDs found: ${Array.from(duplicates).join(", ")}.`
+    });
+  }
+});

package/dist/esm/errors.d.ts

@@ -0,0 +1,3 @@
+export type ErrnoException = NodeJS.ErrnoException;
+export declare const isErrnoException: (error: unknown) => error is ErrnoException;
+export declare const getErrnoCode: (error: unknown) => string | undefined;

package/dist/esm/errors.mjs

@@ -0,0 +1,2 @@
+export const isErrnoException = (error) => typeof error === "object" && error !== null && "code" in error && (typeof error.code === "string" || typeof error.code === "number" || error.code === void 0);
+export const getErrnoCode = (error) => isErrnoException(error) && typeof error.code === "string" ? error.code : void 0;

package/dist/esm/git/cache-dir.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Get platform-specific cache directory
+ * - macOS: ~/Library/Caches
+ * - Windows: %LOCALAPPDATA% or ~/AppData/Local
+ * - Linux: $XDG_CACHE_HOME or ~/.cache
+ */
+export declare const getCacheBaseDir: () => string;
+/**
+ * Resolve the git cache directory
+ * Can be overridden via DOCS_CACHE_GIT_DIR environment variable
+ */
+export declare const resolveGitCacheDir: () => string;
+/**
+ * Check if a file or directory exists
+ */
+export declare const exists: (filePath: string) => Promise<boolean>;

package/dist/esm/git/cache-dir.mjs

@@ -0,0 +1,23 @@
+import { access } from "node:fs/promises";
+import { homedir } from "node:os";
+import path from "node:path";
+export const getCacheBaseDir = () => {
+  const home = homedir();
+  switch (process.platform) {
+    case "darwin":
+      return path.join(home, "Library", "Caches");
+    case "win32":
+      return process.env.LOCALAPPDATA || path.join(home, "AppData", "Local");
+    default:
+      return process.env.XDG_CACHE_HOME || path.join(home, ".cache");
+  }
+};
+export const resolveGitCacheDir = () => process.env.DOCS_CACHE_GIT_DIR || path.join(getCacheBaseDir(), "docs-cache-git");
+export const exists = async (filePath) => {
+  try {
+    await access(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+};

package/dist/esm/git/fetch-source.d.ts

@@ -0,0 +1,19 @@
+type FetchParams = {
+    sourceId: string;
+    repo: string;
+    ref: string;
+    resolvedCommit: string;
+    cacheDir: string;
+    include?: string[];
+    timeoutMs?: number;
+    logger?: (message: string) => void;
+    progressLogger?: (message: string) => void;
+    offline?: boolean;
+};
+type FetchResult = {
+    repoDir: string;
+    cleanup: () => Promise<void>;
+    fromCache: boolean;
+};
+export declare const fetchSource: (params: FetchParams) => Promise<FetchResult>;
+export {};

package/dist/esm/git/fetch-source.mjs

@@ -0,0 +1,477 @@
+import { createHash } from "node:crypto";
+import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { execa } from "execa";
+import { getErrnoCode } from "#core/errors";
+import { assertSafeSourceId } from "#core/source-id";
+import { exists, resolveGitCacheDir } from "#git/cache-dir";
+const DEFAULT_TIMEOUT_MS = 12e4;
+const DEFAULT_GIT_DEPTH = 1;
+const DEFAULT_RM_RETRIES = 3;
+const DEFAULT_RM_BACKOFF_MS = 100;
+const buildGitEnv = () => {
+  const pathValue = process.env.PATH ?? process.env.Path;
+  const pathExtValue = process.env.PATHEXT ?? (process.platform === "win32" ? ".COM;.EXE;.BAT;.CMD" : void 0);
+  return {
+    ...process.env,
+    ...pathValue ? { PATH: pathValue, Path: pathValue } : {},
+    ...pathExtValue ? { PATHEXT: pathExtValue } : {},
+    HOME: process.env.HOME,
+    USER: process.env.USER,
+    USERPROFILE: process.env.USERPROFILE,
+    TMPDIR: process.env.TMPDIR,
+    TMP: process.env.TMP,
+    TEMP: process.env.TEMP,
+    SYSTEMROOT: process.env.SYSTEMROOT,
+    WINDIR: process.env.WINDIR,
+    SSH_AUTH_SOCK: process.env.SSH_AUTH_SOCK,
+    SSH_AGENT_PID: process.env.SSH_AGENT_PID,
+    HTTP_PROXY: process.env.HTTP_PROXY,
+    HTTPS_PROXY: process.env.HTTPS_PROXY,
+    NO_PROXY: process.env.NO_PROXY,
+    GIT_TERMINAL_PROMPT: "0",
+    GIT_CONFIG_NOSYSTEM: "1",
+    GIT_CONFIG_NOGLOBAL: "1",
+    ...process.platform === "win32" ? {} : { GIT_ASKPASS: "/bin/false" }
+  };
+};
+const buildGitConfigs = (allowFileProtocol) => [
+  "-c",
+  "core.hooksPath=/dev/null",
+  "-c",
+  "submodule.recurse=false",
+  "-c",
+  "protocol.ext.allow=never",
+  "-c",
+  `protocol.file.allow=${allowFileProtocol ? "always" : "never"}`
+];
+const buildCommandArgs = (args, allowFileProtocol, forceProgress) => {
+  const configs = buildGitConfigs(allowFileProtocol);
+  const commandArgs = [...configs, ...args];
+  if (forceProgress) {
+    commandArgs.push("--progress");
+  }
+  return commandArgs;
+};
+const isProgressLine = (line) => line.includes("Receiving objects") || line.includes("Resolving deltas") || line.includes("Compressing objects") || line.includes("Updating files") || line.includes("Counting objects");
+const shouldEmitProgress = (line, now, lastProgressAt, throttleMs) => now - lastProgressAt >= throttleMs || line.includes("100%") || line.includes("done");
+const attachLoggers = (subprocess, commandLabel, options) => {
+  if (!options?.logger && !options?.progressLogger) {
+    return;
+  }
+  let lastProgressAt = 0;
+  const forward = (stream) => {
+    if (!stream) return;
+    stream.on("data", (chunk) => {
+      const text = chunk instanceof Buffer ? chunk.toString("utf8") : String(chunk);
+      for (const line of text.split(/\r?\n/)) {
+        if (!line) continue;
+        options.logger?.(`${commandLabel} | ${line}`);
+        if (!options?.progressLogger) continue;
+        if (!isProgressLine(line)) continue;
+        const now = Date.now();
+        const throttleMs = options.progressThrottleMs ?? 120;
+        if (shouldEmitProgress(line, now, lastProgressAt, throttleMs)) {
+          lastProgressAt = now;
+          options.progressLogger(line);
+        }
+      }
+    });
+  };
+  forward(subprocess.stdout);
+  forward(subprocess.stderr);
+};
+const git = async (args, options) => {
+  const commandArgs = buildCommandArgs(
+    args,
+    options?.allowFileProtocol,
+    options?.forceProgress
+  );
+  const commandLabel = `git ${commandArgs.join(" ")}`;
+  options?.logger?.(commandLabel);
+  const subprocess = execa("git", commandArgs, {
+    cwd: options?.cwd,
+    timeout: options?.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+    maxBuffer: 10 * 1024 * 1024,
+    stdout: "pipe",
+    stderr: "pipe",
+    env: buildGitEnv()
+  });
+  attachLoggers(subprocess, commandLabel, options);
+  await subprocess;
+};
+const removeDir = async (dirPath, retries = DEFAULT_RM_RETRIES) => {
+  for (let attempt = 0; attempt <= retries; attempt += 1) {
+    try {
+      await rm(dirPath, { recursive: true, force: true });
+      return;
+    } catch (error) {
+      const code = getErrnoCode(error);
+      if (code !== "ENOTEMPTY" && code !== "EBUSY" && code !== "EPERM") {
+        throw error;
+      }
+      if (attempt === retries) {
+        throw error;
+      }
+      await new Promise(
+        (resolve) => setTimeout(resolve, DEFAULT_RM_BACKOFF_MS * (attempt + 1))
+      );
+    }
+  }
+};
+const hashRepoUrl = (repo) => {
+  return createHash("sha256").update(repo).digest("hex").substring(0, 16);
+};
+const getPersistentCachePath = (repo) => {
+  const repoHash = hashRepoUrl(repo);
+  return path.join(resolveGitCacheDir(), repoHash);
+};
+const isValidGitRepo = async (repoPath) => {
+  try {
+    await git(["rev-parse", "--git-dir"], { cwd: repoPath });
+    return true;
+  } catch {
+    return false;
+  }
+};
+const isPartialClone = async (repoPath) => {
+  try {
+    const configPath = path.join(repoPath, ".git", "config");
+    const raw = await readFile(configPath, "utf8");
+    const lower = raw.toLowerCase();
+    return lower.includes("partialclone") || lower.includes("promisor") || lower.includes("partialclonefilter");
+  } catch {
+    return false;
+  }
+};
+const hasCommitInRepo = async (repoPath, commit, options) => {
+  try {
+    await git(["-C", repoPath, "cat-file", "-e", `${commit}^{commit}`], {
+      timeoutMs: options?.timeoutMs,
+      allowFileProtocol: options?.allowFileProtocol,
+      logger: options?.logger
+    });
+    return true;
+  } catch {
+    return false;
+  }
+};
+const ensureCommitAvailable = async (repoPath, commit, options) => {
+  try {
+    await git(["-C", repoPath, "cat-file", "-e", `${commit}^{commit}`], {
+      timeoutMs: options?.timeoutMs,
+      allowFileProtocol: options?.allowFileProtocol,
+      logger: options?.logger
+    });
+    return;
+  } catch {
+  }
+  if (options?.offline && !options?.allowFileProtocol) {
+    throw new Error(`Commit ${commit} not found in cache (offline).`);
+  }
+  await git(["-C", repoPath, "fetch", "origin", commit], {
+    timeoutMs: options?.timeoutMs,
+    allowFileProtocol: options?.allowFileProtocol,
+    logger: options?.logger
+  });
+};
+const isSparseEligible = (include) => {
+  if (!include || include.length === 0) {
+    return false;
+  }
+  for (const pattern of include) {
+    if (!pattern || pattern.includes("**")) {
+      return false;
+    }
+  }
+  return true;
+};
+const extractSparsePaths = (include) => {
+  if (!include) {
+    return [];
+  }
+  const paths = include.map((pattern) => {
+    const normalized = pattern.replace(/\\/g, "/");
+    const starIndex = normalized.indexOf("*");
+    const base = starIndex === -1 ? normalized : normalized.slice(0, starIndex);
+    return base.replace(/\/+$|\/$/, "");
+  });
+  return Array.from(new Set(paths.filter((value) => value.length > 0)));
+};
+const cloneRepo = async (params, outDir) => {
+  if (params.offline) {
+    throw new Error(`Cannot clone ${params.repo} while offline.`);
+  }
+  const isCommitRef = /^[0-9a-f]{7,40}$/i.test(params.ref);
+  const useSparse = isSparseEligible(params.include);
+  const buildCloneArgs = () => {
+    const cloneArgs2 = [
+      "clone",
+      "--no-checkout",
+      "--depth",
+      String(DEFAULT_GIT_DEPTH),
+      "--recurse-submodules=no",
+      "--no-tags"
+    ];
+    return cloneArgs2;
+  };
+  const cloneArgs = buildCloneArgs();
+  if (useSparse) {
+    cloneArgs.push("--sparse");
+  }
+  if (!isCommitRef) {
+    cloneArgs.push("--single-branch");
+    if (params.ref !== "HEAD") {
+      cloneArgs.push("--branch", params.ref);
+    }
+  }
+  cloneArgs.push(params.repo, outDir);
+  await git(cloneArgs, {
+    timeoutMs: params.timeoutMs,
+    logger: params.logger,
+    progressLogger: params.progressLogger,
+    forceProgress: Boolean(params.progressLogger)
+  });
+  await ensureCommitAvailable(outDir, params.resolvedCommit, {
+    timeoutMs: params.timeoutMs,
+    logger: params.logger,
+    offline: params.offline
+  });
+  if (useSparse) {
+    const sparsePaths = extractSparsePaths(params.include);
+    if (sparsePaths.length > 0) {
+      await git(["-C", outDir, "sparse-checkout", "set", ...sparsePaths], {
+        timeoutMs: params.timeoutMs,
+        logger: params.logger
+      });
+    }
+  }
+  await git(
+    ["-C", outDir, "checkout", "--quiet", "--detach", params.resolvedCommit],
+    {
+      timeoutMs: params.timeoutMs,
+      logger: params.logger
+    }
+  );
+};
+const addWorktreeFromCache = async (params, cachePath, outDir) => {
+  await git(
+    [
+      "-C",
+      cachePath,
+      "worktree",
+      "add",
+      "--detach",
+      outDir,
+      params.resolvedCommit
+    ],
+    {
+      timeoutMs: params.timeoutMs,
+      logger: params.logger,
+      allowFileProtocol: true
+    }
+  );
+  await git(
+    ["-C", outDir, "checkout", "--quiet", "--detach", params.resolvedCommit],
+    {
+      timeoutMs: params.timeoutMs,
+      logger: params.logger,
+      allowFileProtocol: true
+    }
+  );
+  const sparsePaths = isSparseEligible(params.include) ? extractSparsePaths(params.include) : [];
+  if (sparsePaths.length > 0) {
+    await git(["-C", outDir, "sparse-checkout", "set", ...sparsePaths], {
+      timeoutMs: params.timeoutMs,
+      logger: params.logger,
+      allowFileProtocol: true
+    });
+  }
+  return {
+    usedCache: true,
+    cleanup: async () => {
+      try {
+        await git(["-C", cachePath, "worktree", "remove", "--force", outDir], {
+          timeoutMs: params.timeoutMs,
+          logger: params.logger,
+          allowFileProtocol: true
+        });
+      } catch {
+      }
+    }
+  };
+};
+const buildFetchArgs = (ref, isCommitRef) => {
+  const fetchArgs = ["fetch", "origin"];
+  if (!isCommitRef) {
+    const refSpec = ref === "HEAD" ? "HEAD" : `${ref}:refs/remotes/origin/${ref}`;
+    fetchArgs.push(refSpec, "--depth", String(DEFAULT_GIT_DEPTH));
+    return fetchArgs;
+  }
+  fetchArgs.push("--depth", String(DEFAULT_GIT_DEPTH));
+  return fetchArgs;
+};
+const fetchCommitFromOrigin = async (params, cachePath, isCommitRef) => {
+  const fetchArgs = buildFetchArgs(params.ref, isCommitRef);
+  await git(["-C", cachePath, ...fetchArgs], {
+    timeoutMs: params.timeoutMs,
+    logger: params.logger,
+    progressLogger: params.progressLogger,
+    forceProgress: Boolean(params.progressLogger),
+    allowFileProtocol: true
+  });
+  await ensureCommitAvailable(cachePath, params.resolvedCommit, {
+    timeoutMs: params.timeoutMs,
+    logger: params.logger,
+    offline: params.offline
+  });
+};
+const handleValidCache = async (params, cachePath, isCommitRef) => {
+  if (await isPartialClone(cachePath)) {
+    if (params.offline) {
+      throw new Error(`Cache for ${params.repo} is partial (offline).`);
+    }
+    await removeDir(cachePath);
+    await cloneRepo(params, cachePath);
+    return { usedCache: false, worktreeUsed: false };
+  }
+  try {
+    const commitExists = await hasCommitInRepo(
+      cachePath,
+      params.resolvedCommit,
+      {
+        timeoutMs: params.timeoutMs,
+        logger: params.logger
+      }
+    );
+    if (commitExists) {
+      return { usedCache: true, worktreeUsed: true };
+    }
+    if (params.offline) {
+      throw new Error(
+        `Commit ${params.resolvedCommit} not found in cache (offline).`
+      );
+    }
+    await fetchCommitFromOrigin(params, cachePath, isCommitRef);
+    return { usedCache: true, worktreeUsed: false };
+  } catch (_error) {
+    if (params.offline) {
+      throw new Error(`Cache for ${params.repo} is unavailable (offline).`);
+    }
+    await removeDir(cachePath);
+    await cloneRepo(params, cachePath);
+    return { usedCache: false, worktreeUsed: false };
+  }
+};
+const handleMissingCache = async (params, cachePath, cacheExists) => {
+  if (cacheExists) {
+    await removeDir(cachePath);
+  }
+  if (params.offline) {
+    throw new Error(`Cache for ${params.repo} is missing (offline).`);
+  }
+  await cloneRepo(params, cachePath);
+  return { usedCache: false, worktreeUsed: false };
+};
+const cloneOrUpdateRepo = async (params, outDir) => {
+  const cachePath = getPersistentCachePath(params.repo);
+  const cacheExists = await exists(cachePath);
+  const cacheValid = cacheExists && await isValidGitRepo(cachePath);
+  const isCommitRef = /^[0-9a-f]{7,40}$/i.test(params.ref);
+  const useSparse = isSparseEligible(params.include);
+  let usedCache = cacheValid;
+  let worktreeUsed = false;
+  const cacheRoot = resolveGitCacheDir();
+  await mkdir(cacheRoot, { recursive: true });
+  if (cacheValid) {
+    const result = await handleValidCache(params, cachePath, isCommitRef);
+    usedCache = result.usedCache;
+    worktreeUsed = result.worktreeUsed;
+  }
+  if (!cacheValid) {
+    const result = await handleMissingCache(params, cachePath, cacheExists);
+    usedCache = result.usedCache;
+    worktreeUsed = result.worktreeUsed;
+  }
+  if (worktreeUsed && cacheValid) {
+    return addWorktreeFromCache(params, cachePath, outDir);
+  }
+  await mkdir(outDir, { recursive: true });
+  const localCloneArgs = [
+    "clone",
+    "--no-checkout",
+    "--depth",
+    String(DEFAULT_GIT_DEPTH),
+    "--recurse-submodules=no",
+    "--no-tags"
+  ];
+  if (await isPartialClone(cachePath)) {
+    localCloneArgs.splice(2, 0, "--filter=blob:none");
+  }
+  if (useSparse) {
+    localCloneArgs.push("--sparse");
+  }
+  if (!isCommitRef) {
+    localCloneArgs.push("--single-branch");
+    if (params.ref !== "HEAD") {
+      localCloneArgs.push("--branch", params.ref);
+    }
+  }
+  const cacheUrl = pathToFileURL(cachePath).href;
+  localCloneArgs.push(cacheUrl, outDir);
+  await git(localCloneArgs, {
+    timeoutMs: params.timeoutMs,
+    allowFileProtocol: true,
+    logger: params.logger,
+    progressLogger: params.progressLogger,
+    forceProgress: Boolean(params.progressLogger)
+  });
+  if (useSparse) {
+    const sparsePaths = extractSparsePaths(params.include);
+    if (sparsePaths.length > 0) {
+      await git(["-C", outDir, "sparse-checkout", "set", ...sparsePaths], {
+        timeoutMs: params.timeoutMs,
+        allowFileProtocol: true,
+        logger: params.logger
+      });
+    }
+  }
+  await ensureCommitAvailable(outDir, params.resolvedCommit, {
+    timeoutMs: params.timeoutMs,
+    allowFileProtocol: true,
+    logger: params.logger,
+    offline: params.offline
+  });
+  await git(
+    ["-C", outDir, "checkout", "--quiet", "--detach", params.resolvedCommit],
+    {
+      timeoutMs: params.timeoutMs,
+      allowFileProtocol: true,
+      logger: params.logger
+    }
+  );
+  return { usedCache, cleanup: async () => void 0 };
+};
+export const fetchSource = async (params) => {
+  assertSafeSourceId(params.sourceId, "sourceId");
+  const tempRoot = await mkdtemp(
+    path.join(tmpdir(), `docs-cache-${params.sourceId}-`)
+  );
+  const tempDir = path.join(tempRoot, "repo");
+  try {
+    const { usedCache, cleanup } = await cloneOrUpdateRepo(params, tempDir);
+    return {
+      repoDir: tempDir,
+      cleanup: async () => {
+        await cleanup();
+        await removeDir(tempRoot);
+      },
+      fromCache: usedCache
+    };
+  } catch (error) {
+    await removeDir(tempRoot);
+    throw error;
+  }
+};

package/dist/esm/git/redact.d.ts

@@ -0,0 +1 @@
+export declare const redactRepoUrl: (repo: string) => string;

package/dist/esm/git/redact.mjs

@@ -0,0 +1,4 @@
+const CREDENTIAL_RE = /^(https?:\/\/)([^@]+)@/i;
+export const redactRepoUrl = (repo) => {
+  return repo.replace(CREDENTIAL_RE, "$1***@");
+};

package/dist/esm/git/resolve-remote.d.ts

@@ -0,0 +1,15 @@
+type ResolveRemoteParams = {
+    repo: string;
+    ref: string;
+    allowHosts: string[];
+    timeoutMs?: number;
+    logger?: (message: string) => void;
+};
+export declare const enforceHostAllowlist: (repo: string, allowHosts: string[]) => void;
+export declare const parseLsRemote: (stdout: string) => string | null;
+export declare const resolveRemoteCommit: (params: ResolveRemoteParams) => Promise<{
+    repo: string;
+    ref: string;
+    resolvedCommit: string;
+}>;
+export {};