docs-cache 0.4.3 → 0.5.1

package/dist/esm/cache/materialize.mjs

@@ -0,0 +1,442 @@
+import { createHash, randomBytes } from "node:crypto";
+import { constants, createReadStream, createWriteStream } from "node:fs";
+import {
+  access,
+  lstat,
+  mkdir,
+  mkdtemp,
+  open,
+  rename,
+  rm,
+  writeFile
+} from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { pipeline } from "node:stream/promises";
+import fg from "fast-glob";
+import { MANIFEST_FILENAME } from "#cache/manifest";
+import { symbols, ui } from "#cli/ui";
+import { getErrnoCode } from "#core/errors";
+import { getCacheLayout, toPosixPath } from "#core/paths";
+import { assertSafeSourceId } from "#core/source-id";
+const normalizePath = (value) => toPosixPath(value);
+const escapeParens = (value) => {
+  let output = "";
+  let escaped = false;
+  for (const char of value) {
+    if (escaped) {
+      output += char;
+      escaped = false;
+      continue;
+    }
+    if (char === "\\") {
+      output += char;
+      escaped = true;
+      continue;
+    }
+    if (char === "(" || char === ")") {
+      output += `\\${char}`;
+      continue;
+    }
+    output += char;
+  }
+  return output;
+};
+const normalizeIncludePatterns = (patterns) => {
+  return patterns.map((pattern) => {
+    let decoded = pattern;
+    if (pattern.includes("%")) {
+      try {
+        decoded = decodeURIComponent(pattern);
+      } catch {
+        decoded = pattern;
+      }
+    }
+    const hasExtGlob = /(^|[^\\])[@!+?*]\(/.test(decoded);
+    return hasExtGlob ? decoded : escapeParens(decoded);
+  });
+};
+const STREAM_COPY_THRESHOLD_MB = Number(
+  process.env.DOCS_CACHE_STREAM_THRESHOLD_MB ?? "2"
+);
+const STREAM_COPY_THRESHOLD_BYTES = Number.isFinite(STREAM_COPY_THRESHOLD_MB) && STREAM_COPY_THRESHOLD_MB > 0 ? Math.floor(STREAM_COPY_THRESHOLD_MB * 1024 * 1024) : 1024 * 1024;
+const ensureSafePath = (root, target) => {
+  const resolvedRoot = path.resolve(root);
+  const resolvedTarget = path.resolve(target);
+  if (!resolvedTarget.startsWith(resolvedRoot + path.sep)) {
+    throw new Error(`Path traversal detected: ${target}`);
+  }
+};
+const openFileNoFollow = async (filePath) => {
+  try {
+    return await open(filePath, constants.O_RDONLY | constants.O_NOFOLLOW);
+  } catch (error) {
+    const code = getErrnoCode(error);
+    if (code === "ELOOP") {
+      return null;
+    }
+    if (code === "EINVAL" || code === "ENOSYS" || code === "ENOTSUP") {
+      const stats = await lstat(filePath);
+      if (stats.isSymbolicLink()) {
+        return null;
+      }
+      return await open(filePath, "r");
+    }
+    throw error;
+  }
+};
+const resolveUnwrapPrefix = (entries, unwrapSingleRootDir) => {
+  if (!unwrapSingleRootDir || entries.length === 0) {
+    return null;
+  }
+  let prefix = "";
+  while (true) {
+    let rootDir = null;
+    for (const entry of entries) {
+      const remaining = prefix ? entry.normalized.slice(prefix.length) : entry.normalized;
+      const parts = remaining.split("/");
+      if (parts.length < 2) {
+        return prefix || null;
+      }
+      const nextRoot = parts[0];
+      if (!rootDir) {
+        rootDir = nextRoot;
+        continue;
+      }
+      if (rootDir !== nextRoot) {
+        return prefix || null;
+      }
+    }
+    if (!rootDir) {
+      return prefix || null;
+    }
+    const nextPrefix = `${prefix}${rootDir}/`;
+    if (nextPrefix === prefix) {
+      return prefix || null;
+    }
+    prefix = nextPrefix;
+  }
+};
+const resolveMaterializeParams = (params) => ({
+  ...params,
+  exclude: params.exclude ?? [],
+  ignoreHidden: params.ignoreHidden ?? false,
+  unwrapSingleRootDir: params.unwrapSingleRootDir ?? false,
+  json: params.json ?? false,
+  progressThrottleMs: params.progressThrottleMs ?? 120
+});
+const acquireLock = async (lockPath, timeoutMs = 5e3) => {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    try {
+      const fd = await open(lockPath, "wx");
+      return {
+        release: async () => {
+          await fd.close();
+          await rm(lockPath, { force: true });
+        }
+      };
+    } catch (error) {
+      const code = getErrnoCode(error);
+      if (code !== "EEXIST") {
+        throw error;
+      }
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+  }
+  throw new Error(`Failed to acquire lock for ${lockPath}.`);
+};
+export const materializeSource = async (params) => {
+  const resolved = resolveMaterializeParams(params);
+  assertSafeSourceId(resolved.sourceId, "sourceId");
+  const layout = getCacheLayout(resolved.cacheDir, resolved.sourceId);
+  await mkdir(resolved.cacheDir, { recursive: true });
+  const tempDir = await mkdtemp(
+    path.join(resolved.cacheDir, `.tmp-${resolved.sourceId}-`)
+  );
+  let manifestStreamRef = null;
+  const closeManifestStream = async () => {
+    const stream = manifestStreamRef;
+    if (!stream || stream.closed || stream.destroyed) {
+      return;
+    }
+    await new Promise((resolve) => {
+      const cleanup = () => {
+        stream.off("close", onClose);
+        stream.off("error", onError);
+        resolve();
+      };
+      const onClose = () => cleanup();
+      const onError = () => cleanup();
+      stream.once("close", onClose);
+      stream.once("error", onError);
+      try {
+        stream.end();
+      } catch {
+        cleanup();
+      }
+    });
+  };
+  try {
+    const ignorePatterns = [
+      ".git/**",
+      ...resolved.ignoreHidden ? [".*", "**/.*", "**/.*/**"] : [],
+      ...resolved.exclude
+    ];
+    const includePatterns = normalizeIncludePatterns(resolved.include);
+    const files = await fg(includePatterns, {
+      cwd: resolved.repoDir,
+      ignore: ignorePatterns,
+      dot: true,
+      onlyFiles: true,
+      followSymbolicLinks: false
+    });
+    if (!resolved.json && includePatterns.length > 0 && files.length === 0) {
+      ui.line(
+        `${symbols.warn} No files matched include patterns for ${resolved.sourceId}: ${includePatterns.join(", ")}`
+      );
+    }
+    const entries = files.map((relativePath) => ({
+      relativePath,
+      normalized: normalizePath(relativePath)
+    })).sort((left, right) => left.normalized.localeCompare(right.normalized));
+    const totalEntries = entries.length;
+    const unwrapPrefix = resolveUnwrapPrefix(
+      entries,
+      resolved.unwrapSingleRootDir
+    );
+    const targetDirs = /* @__PURE__ */ new Set();
+    for (const { normalized } of entries) {
+      const rootPath = unwrapPrefix ? normalized.slice(unwrapPrefix.length) : normalized;
+      targetDirs.add(path.posix.dirname(rootPath));
+    }
+    await Promise.all(
+      Array.from(
+        targetDirs,
+        (dir) => mkdir(path.join(tempDir, dir), { recursive: true })
+      )
+    );
+    let bytes = 0;
+    let fileCount = 0;
+    let lastProgressAt = 0;
+    const concurrency = Math.max(
+      1,
+      Math.min(
+        entries.length,
+        Math.max(8, Math.min(128, os.cpus().length * 8))
+      )
+    );
+    const manifestPath = path.join(tempDir, MANIFEST_FILENAME);
+    const manifestStream = createWriteStream(manifestPath, {
+      encoding: "utf8"
+    });
+    manifestStreamRef = manifestStream;
+    const manifestHash = createHash("sha256");
+    const writeManifestLine = async (line) => {
+      return new Promise((resolve, reject) => {
+        const onError = (error) => {
+          manifestStream.off("drain", onDrain);
+          reject(error);
+        };
+        const onDrain = () => {
+          manifestStream.off("error", onError);
+          resolve();
+        };
+        manifestStream.once("error", onError);
+        if (!manifestStream.write(line)) {
+          manifestStream.once("drain", onDrain);
+        } else {
+          manifestStream.off("error", onError);
+          resolve();
+        }
+      });
+    };
+    for (let i = 0; i < entries.length; i += concurrency) {
+      const batch = entries.slice(i, i + concurrency);
+      const results = await Promise.all(
+        batch.map(async (entry) => {
+          const filePath = path.join(resolved.repoDir, entry.relativePath);
+          const fileHandle = await openFileNoFollow(filePath);
+          if (!fileHandle) {
+            return null;
+          }
+          try {
+            const stats = await fileHandle.stat();
+            if (!stats.isFile()) {
+              return null;
+            }
+            const normalizedPath = unwrapPrefix ? entry.normalized.slice(unwrapPrefix.length) : entry.normalized;
+            const targetPath = path.join(tempDir, normalizedPath);
+            ensureSafePath(tempDir, targetPath);
+            if (stats.size >= STREAM_COPY_THRESHOLD_BYTES) {
+              const reader = createReadStream(filePath, {
+                fd: fileHandle.fd,
+                autoClose: false
+              });
+              const writer = createWriteStream(targetPath);
+              await pipeline(reader, writer);
+            } else {
+              const data = await fileHandle.readFile();
+              await writeFile(targetPath, data);
+            }
+            return {
+              path: unwrapPrefix ? entry.normalized.slice(unwrapPrefix.length) : entry.normalized,
+              size: stats.size
+            };
+          } finally {
+            await fileHandle.close();
+          }
+        })
+      );
+      for (const entry of results) {
+        if (!entry) {
+          continue;
+        }
+        if (resolved.maxFiles !== void 0 && fileCount + 1 > resolved.maxFiles) {
+          throw new Error(
+            `Materialized content exceeds maxFiles (${resolved.maxFiles}).`
+          );
+        }
+        bytes += entry.size;
+        if (bytes > resolved.maxBytes) {
+          throw new Error(
+            `Materialized content exceeds maxBytes (${resolved.maxBytes}).`
+          );
+        }
+        const line = `${JSON.stringify(entry)}
+`;
+        manifestHash.update(line);
+        await writeManifestLine(line);
+        fileCount += 1;
+      }
+      if (resolved.progressLogger && totalEntries > 0) {
+        const now = Date.now();
+        const shouldEmit = now - lastProgressAt >= resolved.progressThrottleMs || fileCount === totalEntries;
+        if (shouldEmit) {
+          lastProgressAt = now;
+          const percent = Math.min(
+            100,
+            Math.round(fileCount / totalEntries * 100)
+          );
+          resolved.progressLogger(
+            `materializing ${fileCount}/${totalEntries} (${percent}%)`
+          );
+        }
+      }
+    }
+    await new Promise((resolve, reject) => {
+      manifestStream.end(() => resolve());
+      manifestStream.once("error", reject);
+    });
+    const manifestSha256 = manifestHash.digest("hex");
+    const exists = async (target) => {
+      try {
+        await access(target);
+        return true;
+      } catch {
+        return false;
+      }
+    };
+    const replaceDirectory = async (source, target) => {
+      const lock = await acquireLock(`${target}.lock`);
+      try {
+        const hasTarget = await exists(target);
+        const backupPath = `${target}.bak-${randomBytes(8).toString("hex")}`;
+        if (hasTarget) {
+          await rename(target, backupPath);
+        }
+        try {
+          await rename(source, target);
+        } catch (error) {
+          if (hasTarget) {
+            try {
+              await rename(backupPath, target);
+            } catch (restoreError) {
+              const restoreMsg = restoreError instanceof Error ? restoreError.message : String(restoreError);
+              process.stderr.write(
+                `Warning: Failed to restore backup: ${restoreMsg}
+`
+              );
+            }
+          }
+          throw error;
+        }
+        if (hasTarget) {
+          await rm(backupPath, { recursive: true, force: true });
+        }
+      } finally {
+        await lock.release();
+      }
+    };
+    await replaceDirectory(tempDir, layout.sourceDir);
+    return {
+      bytes,
+      fileCount,
+      manifestSha256
+    };
+  } catch (error) {
+    try {
+      await closeManifestStream();
+    } catch {
+    }
+    await rm(tempDir, { recursive: true, force: true });
+    throw error;
+  }
+};
+export const computeManifestHash = async (params) => {
+  assertSafeSourceId(params.sourceId, "sourceId");
+  const includePatterns = normalizeIncludePatterns(params.include);
+  const files = await fg(includePatterns, {
+    cwd: params.repoDir,
+    ignore: [
+      ".git/**",
+      ...params.ignoreHidden ? [".*", "**/.*", "**/.*/**"] : [],
+      ...params.exclude ?? []
+    ],
+    dot: true,
+    onlyFiles: true,
+    followSymbolicLinks: false
+  });
+  files.sort(
+    (left, right) => normalizePath(left).localeCompare(normalizePath(right))
+  );
+  let bytes = 0;
+  let fileCount = 0;
+  const manifestHash = createHash("sha256");
+  for (const relativePath of files) {
+    const relNormalized = normalizePath(relativePath);
+    const filePath = path.join(params.repoDir, relativePath);
+    const fileHandle = await openFileNoFollow(filePath);
+    if (!fileHandle) {
+      continue;
+    }
+    try {
+      const stats = await fileHandle.stat();
+      if (!stats.isFile()) {
+        continue;
+      }
+      if (params.maxFiles !== void 0 && fileCount + 1 > params.maxFiles) {
+        throw new Error(
+          `Materialized content exceeds maxFiles (${params.maxFiles}).`
+        );
+      }
+      bytes += stats.size;
+      if (bytes > params.maxBytes) {
+        throw new Error(
+          `Materialized content exceeds maxBytes (${params.maxBytes}).`
+        );
+      }
+      const line = `${JSON.stringify({ path: relNormalized, size: stats.size })}
+`;
+      manifestHash.update(line);
+      fileCount += 1;
+    } finally {
+      await fileHandle.close();
+    }
+  }
+  return {
+    bytes,
+    fileCount,
+    manifestSha256: manifestHash.digest("hex")
+  };
+};

package/dist/esm/cache/targets.d.ts

@@ -0,0 +1,19 @@
+import { cp, mkdir, readdir, rm, symlink } from "node:fs/promises";
+type TargetDeps = {
+    cp: typeof cp;
+    mkdir: typeof mkdir;
+    readdir: typeof readdir;
+    rm: typeof rm;
+    symlink: typeof symlink;
+    stderr: NodeJS.WritableStream;
+};
+type TargetParams = {
+    sourceDir: string;
+    targetDir: string;
+    mode?: "symlink" | "copy";
+    explicitTargetMode?: boolean;
+    unwrapSingleRootDir?: boolean;
+    deps?: TargetDeps;
+};
+export declare const applyTargetDir: (params: TargetParams) => Promise<void>;
+export {};

package/dist/esm/cache/targets.mjs

@@ -0,0 +1,66 @@
+import { cp, mkdir, readdir, rm, symlink } from "node:fs/promises";
+import path from "node:path";
+import { MANIFEST_FILENAME } from "#cache/manifest";
+import { getErrnoCode } from "#core/errors";
+import { DEFAULT_TOC_FILENAME } from "#core/paths";
+const removeTarget = async (targetDir, deps) => {
+  await deps.rm(targetDir, { recursive: true, force: true });
+};
+const resolveSourceDir = async (params, deps) => {
+  if (!params.unwrapSingleRootDir) {
+    return params.sourceDir;
+  }
+  const entries = await deps.readdir(params.sourceDir, { withFileTypes: true });
+  const metaFiles = /* @__PURE__ */ new Set([MANIFEST_FILENAME, DEFAULT_TOC_FILENAME]);
+  const nonMeta = entries.filter((entry) => {
+    if (entry.isFile() && metaFiles.has(entry.name)) {
+      return false;
+    }
+    return true;
+  });
+  const directories = nonMeta.filter((entry) => entry.isDirectory());
+  const nonMetaFiles = nonMeta.filter((entry) => entry.isFile());
+  if (directories.length !== 1 || nonMetaFiles.length > 0) {
+    return params.sourceDir;
+  }
+  return path.join(params.sourceDir, directories[0].name);
+};
+export const applyTargetDir = async (params) => {
+  const deps = params.deps ?? {
+    cp,
+    mkdir,
+    readdir,
+    rm,
+    symlink,
+    stderr: process.stderr
+  };
+  const sourceDir = await resolveSourceDir(params, deps);
+  const parentDir = path.dirname(params.targetDir);
+  await deps.mkdir(parentDir, { recursive: true });
+  await removeTarget(params.targetDir, deps);
+  const defaultMode = process.platform === "win32" ? "copy" : "symlink";
+  const mode = params.mode ?? defaultMode;
+  if (mode === "copy") {
+    await deps.cp(sourceDir, params.targetDir, { recursive: true });
+    return;
+  }
+  const type = process.platform === "win32" ? "junction" : "dir";
+  try {
+    await deps.symlink(sourceDir, params.targetDir, type);
+  } catch (error) {
+    const code = getErrnoCode(error);
+    const fallbackCodes = /* @__PURE__ */ new Set(["EPERM", "EACCES", "ENOTSUP", "EINVAL"]);
+    if (code && fallbackCodes.has(code)) {
+      if (params.explicitTargetMode) {
+        const message = error instanceof Error ? error.message : String(error);
+        deps.stderr.write(
+          `Warning: Failed to create symlink at ${params.targetDir}. Falling back to copy. ${message}
+`
+        );
+      }
+      await deps.cp(sourceDir, params.targetDir, { recursive: true });
+      return;
+    }
+    throw error;
+  }
+};

package/dist/esm/cache/toc.d.ts

@@ -0,0 +1,12 @@
+import type { DocsCacheLock } from "#cache/lock";
+import type { DocsCacheResolvedSource } from "#config";
+export declare const writeToc: (params: {
+    cacheDir: string;
+    configPath: string;
+    lock: DocsCacheLock;
+    sources: DocsCacheResolvedSource[];
+    results?: Array<{
+        id: string;
+        status: "up-to-date" | "changed" | "missing";
+    }>;
+}) => Promise<void>;

package/dist/esm/cache/toc.mjs

@@ -0,0 +1,167 @@
+import { access, readFile, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { symbols, ui } from "#cli/ui";
+import {
+  DEFAULT_TOC_FILENAME,
+  resolveTargetDir,
+  toPosixPath
+} from "#core/paths";
+const createTocTree = (files) => {
+  const root = { dirs: /* @__PURE__ */ new Map(), files: [] };
+  for (const file of files) {
+    const parts = file.split("/").filter(Boolean);
+    if (parts.length === 0) {
+      continue;
+    }
+    let node = root;
+    for (const part of parts.slice(0, -1)) {
+      let child = node.dirs.get(part);
+      if (!child) {
+        child = { dirs: /* @__PURE__ */ new Map(), files: [] };
+        node.dirs.set(part, child);
+      }
+      node = child;
+    }
+    const name = parts[parts.length - 1];
+    node.files.push({ name, path: file });
+  }
+  return root;
+};
+const renderTocTree = (tree, depth, lines) => {
+  const indent = "  ".repeat(depth);
+  const dirNames = Array.from(tree.dirs.keys()).sort();
+  const files = [...tree.files].sort((a, b) => a.name.localeCompare(b.name));
+  for (const dirName of dirNames) {
+    lines.push(`${indent}- ${dirName}/`);
+    const child = tree.dirs.get(dirName);
+    if (child) {
+      renderTocTree(child, depth + 1, lines);
+    }
+  }
+  for (const file of files) {
+    lines.push(`${indent}- [${file.name}](./${file.path})`);
+  }
+};
+const renderCompressedToc = (files, lines, label) => {
+  const sortedFiles = [...files].sort((a, b) => a.localeCompare(b));
+  const dirGroups = /* @__PURE__ */ new Map();
+  for (const file of sortedFiles) {
+    const lastSlash = file.lastIndexOf("/");
+    const dir = lastSlash === -1 ? "" : file.substring(0, lastSlash);
+    const filename = lastSlash === -1 ? file : file.substring(lastSlash + 1);
+    const existing = dirGroups.get(dir);
+    if (existing) {
+      existing.push(filename);
+    } else {
+      dirGroups.set(dir, [filename]);
+    }
+  }
+  const sortedDirs = Array.from(dirGroups.keys()).sort();
+  const segments = [];
+  segments.push(`[${label}]`);
+  for (const dir of sortedDirs) {
+    const filesInDir = dirGroups.get(dir);
+    if (!filesInDir) continue;
+    const fileList = filesInDir.join(",");
+    if (dir === "") {
+      segments.push(`root:{${fileList}}`);
+    } else {
+      segments.push(`${dir}:{${fileList}}`);
+    }
+  }
+  lines.push(segments.join("|"));
+};
+const generateSourceToc = (entry, format = "compressed") => {
+  const lines = [];
+  if (format === "tree") {
+    lines.push(`# ${entry.id} - Documentation`);
+    lines.push("");
+    lines.push("## Files");
+    lines.push("");
+    const tree = createTocTree(entry.files);
+    renderTocTree(tree, 0, lines);
+  } else {
+    const label = `${entry.id} Docs Index`;
+    renderCompressedToc(entry.files, lines, label);
+  }
+  lines.push("");
+  return lines.join("\n");
+};
+const readManifest = async (sourceDir) => {
+  const manifestPath = path.join(sourceDir, ".manifest.jsonl");
+  try {
+    const raw = await readFile(manifestPath, "utf8");
+    const files = [];
+    for (const line of raw.split("\n")) {
+      if (line.trim()) {
+        const entry = JSON.parse(line);
+        if (entry.path) {
+          files.push(entry.path);
+        }
+      }
+    }
+    return files;
+  } catch {
+    return [];
+  }
+};
+export const writeToc = async (params) => {
+  const sourcesById = new Map(
+    params.sources.map((source) => [source.id, source])
+  );
+  const resultsById = new Map(
+    (params.results ?? []).map((result) => [result.id, result])
+  );
+  for (const [id, lockEntry] of Object.entries(params.lock.sources)) {
+    const source = sourcesById.get(id);
+    const targetDir = source?.targetDir ? toPosixPath(resolveTargetDir(params.configPath, source.targetDir)) : void 0;
+    const sourceDir = path.join(params.cacheDir, id);
+    try {
+      await access(sourceDir);
+    } catch {
+      continue;
+    }
+    const files = await readManifest(sourceDir);
+    const entry = {
+      id,
+      repo: lockEntry.repo,
+      ref: lockEntry.ref,
+      resolvedCommit: lockEntry.resolvedCommit,
+      fileCount: lockEntry.fileCount,
+      cachePath: toPosixPath(path.join(params.cacheDir, id)),
+      targetDir,
+      files
+    };
+    const sourceTocConfig = source?.toc;
+    const tocEnabled = sourceTocConfig !== false;
+    let tocFormat = "compressed";
+    if (typeof sourceTocConfig === "string") {
+      tocFormat = sourceTocConfig;
+    }
+    const sourceTocPath = path.join(sourceDir, DEFAULT_TOC_FILENAME);
+    if (tocEnabled) {
+      const result = resultsById.get(id);
+      let existingContent = null;
+      try {
+        existingContent = await readFile(sourceTocPath, "utf8");
+      } catch {
+        existingContent = null;
+      }
+      const sourceTocContent = generateSourceToc(entry, tocFormat);
+      if (result?.status === "up-to-date" && existingContent === sourceTocContent) {
+        continue;
+      }
+      if (existingContent !== null && existingContent !== sourceTocContent) {
+        ui.line(
+          `${symbols.warn} Overwriting existing ${DEFAULT_TOC_FILENAME} for ${id}`
+        );
+      }
+      await writeFile(sourceTocPath, sourceTocContent, "utf8");
+    } else {
+      try {
+        await rm(sourceTocPath, { force: true });
+      } catch {
+      }
+    }
+  }
+};