akm-cli 0.5.0 → 0.6.0-rc1

package/dist/stash-providers/npm.js

@@ -0,0 +1,159 @@
+/**
+ * Npm-source stash provider.
+ *
+ * `sync()` resolves the npm package tarball, downloads it, verifies its
+ * integrity, extracts it securely (via `extractTarGzSecure`), detects the
+ * stash root inside the package, and applies any nested `.akm-include`
+ * configuration. Cache hits short-circuit the fetch.
+ *
+ * Audit is intentionally NOT performed here — `akmAdd` calls
+ * `auditInstallCandidate` after `sync()` so the policy decision lives at
+ * the orchestrator layer where the `--trust` flag is known.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { ConfigError, UsageError } from "../errors";
+import { getRegistryCacheDir } from "../paths";
+import { parseRegistryRef, resolveRegistryArtifact } from "../registry-resolve";
+import { registerStashProvider } from "../stash-provider-factory";
+import { applyAkmIncludeConfig, buildInstallCacheDir, computeFileHash, detectStashRoot, downloadArchive, isDirectory, } from "./provider-utils";
+import { extractTarGzSecure, verifyArchiveIntegrity } from "./tar-utils";
+class NpmStashProvider {
+    type = "npm";
+    kind = "syncable";
+    name;
+    constructor(config) {
+        this.name = config.name ?? config.url ?? "npm";
+    }
+    /** Content is indexed through the standard FTS5 pipeline. */
+    async search(_options) {
+        return { hits: [] };
+    }
+    /** Content is local files, shown via showLocal. */
+    async show(_ref, _view) {
+        throw new Error("NPM provider content is shown via local index");
+    }
+    canShow(_ref) {
+        return false;
+    }
+    async sync(config, options) {
+        const ref = npmRefFromConfig(config);
+        return syncNpmRef(ref, options);
+    }
+    getContentDir(config) {
+        if (config.path)
+            return config.path;
+        throw new ConfigError("npm stash entry missing resolved content path");
+    }
+    async remove(config) {
+        if (config.path && isDirectory(config.path)) {
+            // Remove the whole versioned cache dir if we know the parent layout.
+            const parent = path.dirname(config.path);
+            try {
+                fs.rmSync(parent, { recursive: true, force: true });
+            }
+            catch {
+                /* best-effort */
+            }
+        }
+    }
+}
+registerStashProvider("npm", (config) => new NpmStashProvider(config));
+function npmRefFromConfig(config) {
+    // Prefer an explicit ref-bearing field (set by akmAdd when persisting), else fall back
+    // to options or url so the provider stays usable from a hand-rolled config.
+    const candidate = config.options?.ref ?? config.url ?? config.options?.package ?? config.name;
+    if (typeof candidate !== "string" || !candidate) {
+        throw new UsageError('npm stash entry must include an `options.ref` (e.g. "npm:my-pkg@1.2.3")');
+    }
+    return candidate.startsWith("npm:") ? candidate : `npm:${candidate}`;
+}
+/**
+ * Fetch and extract an npm tarball, returning a populated `StashLockData`.
+ *
+ * Mirrors the historical `installRegistryRef()` path for npm sources:
+ *   - resolve artifact URL + integrity from the npm registry
+ *   - reuse cached extraction when present
+ *   - download, verify, extract securely, then detect the stash root
+ *   - honour `.akm-include` filters
+ */
+export async function syncNpmRef(ref, options) {
+    const parsed = parseRegistryRef(ref);
+    if (parsed.source !== "npm") {
+        throw new UsageError(`syncNpmRef requires an npm: ref, got "${ref}"`);
+    }
+    return doSyncNpm(parsed, options);
+}
+async function doSyncNpm(parsed, options) {
+    const resolved = await resolveRegistryArtifact(parsed);
+    const syncedAt = (options?.now ?? new Date()).toISOString();
+    const cacheRootDir = options?.cacheRootDir ?? getRegistryCacheDir();
+    const cacheDir = buildInstallCacheDir(cacheRootDir, resolved.source, resolved.id, resolved.resolvedVersion ?? resolved.resolvedRevision);
+    const archivePath = path.join(cacheDir, "artifact.tar.gz");
+    const extractedDir = path.join(cacheDir, "extracted");
+    // Cache hit: extracted dir already valid → reuse it
+    if (!options?.force && isDirectory(extractedDir)) {
+        try {
+            const cachedStashRoot = detectStashRoot(extractedDir);
+            if (cachedStashRoot) {
+                const integrity = fs.existsSync(archivePath) ? await computeFileHash(archivePath) : undefined;
+                return {
+                    id: resolved.id,
+                    source: resolved.source,
+                    ref: resolved.ref,
+                    artifactUrl: resolved.artifactUrl,
+                    resolvedVersion: resolved.resolvedVersion,
+                    resolvedRevision: resolved.resolvedRevision,
+                    contentDir: cachedStashRoot,
+                    cacheDir,
+                    extractedDir,
+                    integrity,
+                    writable: options?.writable,
+                    syncedAt,
+                };
+            }
+        }
+        catch {
+            // Cache invalid, re-download
+        }
+    }
+    fs.mkdirSync(cacheDir, { recursive: true });
+    let integrity;
+    let provisionalKitRoot;
+    let installRoot;
+    let stashRoot;
+    try {
+        await downloadArchive(resolved.artifactUrl, archivePath);
+        verifyArchiveIntegrity(archivePath, resolved.resolvedRevision, resolved.source);
+        integrity = await computeFileHash(archivePath);
+        extractTarGzSecure(archivePath, extractedDir);
+        provisionalKitRoot = detectStashRoot(extractedDir);
+        installRoot = applyAkmIncludeConfig(provisionalKitRoot, cacheDir, extractedDir) ?? provisionalKitRoot;
+        stashRoot = detectStashRoot(installRoot);
+    }
+    catch (err) {
+        // Clean up so stale or partial extractions don't cause false cache hits.
+        try {
+            fs.rmSync(cacheDir, { recursive: true, force: true });
+        }
+        catch {
+            /* best-effort */
+        }
+        throw err;
+    }
+    return {
+        id: resolved.id,
+        source: resolved.source,
+        ref: resolved.ref,
+        artifactUrl: resolved.artifactUrl,
+        resolvedVersion: resolved.resolvedVersion,
+        resolvedRevision: resolved.resolvedRevision,
+        contentDir: stashRoot,
+        cacheDir,
+        extractedDir,
+        integrity,
+        writable: options?.writable,
+        syncedAt,
+    };
+}
+export { NpmStashProvider };

package/dist/stash-providers/provider-utils.js

@@ -1,3 +1,10 @@
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { TYPE_DIRS } from "../asset-spec";
+import { fetchWithRetry } from "../common";
+import { copyIncludedPaths, findNearestIncludeConfig } from "../stash-include";
+const REGISTRY_STASH_DIR_NAMES = new Set(Object.values(TYPE_DIRS));
 /** Strip terminal control characters from untrusted strings. */
 export function sanitizeString(value, maxLength = 255) {
     if (typeof value !== "string")
@@ -9,3 +16,158 @@ export function sanitizeString(value, maxLength = 255) {
 export function isExpired(mtimeMs, ttlMs) {
     return Date.now() - mtimeMs > ttlMs;
 }
+/**
+ * Find the directory inside `extractedDir` that should be treated as the
+ * stash root. Looks for a `.stash` marker, then well-known type dirs, then
+ * BFS for the shallowest such candidate.
+ */
+export function detectStashRoot(extractedDir) {
+    const root = path.resolve(extractedDir);
+    const rootDotStash = path.join(root, ".stash");
+    if (isDirectory(rootDotStash)) {
+        return root;
+    }
+    if (hasStashDirs(root)) {
+        return root;
+    }
+    const shallowest = findShallowestStashRoot(root);
+    if (shallowest)
+        return shallowest;
+    return root;
+}
+/**
+ * Build a per-source cache directory under `cacheRootDir`.
+ *
+ * Versioned sources get `${source}-${id}/${version}` for cache reuse;
+ * `local` sources get a unique timestamped slug so each install is isolated.
+ */
+export function buildInstallCacheDir(cacheRootDir, source, id, version) {
+    const slug = `${source}-${id.replace(/[^a-zA-Z0-9_.-]+/g, "-").replace(/^-+|-+$/g, "")}`;
+    const versionSlug = source === "local"
+        ? `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`
+        : (version?.replace(/[^a-zA-Z0-9_.-]+/g, "-") ?? `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`);
+    return path.join(cacheRootDir, slug || source, versionSlug);
+}
+/**
+ * Apply an `.akm-include` config (if any) by copying the selected paths
+ * into a sibling `selected/` directory and returning that path. Returns
+ * undefined when no include config is found.
+ */
+export function applyAkmIncludeConfig(sourceRoot, cacheDir, searchRoot = sourceRoot) {
+    const includeConfig = findNearestIncludeConfig(sourceRoot, searchRoot);
+    if (!includeConfig)
+        return undefined;
+    const selectedDir = path.join(cacheDir, "selected");
+    fs.rmSync(selectedDir, { recursive: true, force: true });
+    fs.mkdirSync(selectedDir, { recursive: true });
+    copyIncludedPaths(includeConfig.include, includeConfig.baseDir, selectedDir);
+    return selectedDir;
+}
+/** Stream a remote archive to disk using Bun.write when available. */
+export async function downloadArchive(url, destination) {
+    const response = await fetchWithRetry(url, undefined, { timeout: 120_000 });
+    if (!response.ok) {
+        throw new Error(`Failed to download archive (${response.status}) from ${url}`);
+    }
+    // Stream response to disk instead of buffering the entire archive in memory.
+    // Uses Bun.write which handles Response streaming natively.
+    const BunRuntime = globalThis
+        .Bun;
+    if (BunRuntime?.write) {
+        await BunRuntime.write(destination, response);
+    }
+    else {
+        // Fallback for non-Bun environments (e.g., tests)
+        const arrayBuffer = await response.arrayBuffer();
+        fs.writeFileSync(destination, Buffer.from(arrayBuffer));
+    }
+}
+/** SHA-256 of a file, returned as `sha256:<hex>`. */
+export async function computeFileHash(filePath) {
+    const data = fs.readFileSync(filePath);
+    const hash = createHash("sha256").update(data).digest("hex");
+    return `sha256:${hash}`;
+}
+/** Recursively copy directory contents, excluding `.git`. */
+export function copyDirectoryContents(sourceDir, destinationDir) {
+    for (const entry of fs.readdirSync(sourceDir, { withFileTypes: true })) {
+        if (entry.name === ".git")
+            continue;
+        const src = path.join(sourceDir, entry.name);
+        const dest = path.join(destinationDir, entry.name);
+        fs.mkdirSync(path.dirname(dest), { recursive: true });
+        if (entry.isDirectory()) {
+            fs.cpSync(src, dest, { recursive: true, force: true });
+        }
+        else {
+            fs.copyFileSync(src, dest);
+        }
+    }
+}
+export function isDirectory(target) {
+    try {
+        return fs.statSync(target).isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
+function hasStashDirs(dirPath) {
+    if (!isDirectory(dirPath))
+        return false;
+    const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+    return entries.some((entry) => entry.isDirectory() && REGISTRY_STASH_DIR_NAMES.has(entry.name));
+}
+function countStashDirs(dirPath) {
+    if (!isDirectory(dirPath))
+        return 0;
+    const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+    return entries.filter((entry) => entry.isDirectory() && REGISTRY_STASH_DIR_NAMES.has(entry.name)).length;
+}
+/**
+ * BFS to find the shallowest directory that looks like a stash root.
+ * Checks for both `.stash` directories and well-known type directories
+ * (scripts/, skills/, etc.), so nested layouts like `project/my-stash/scripts/`
+ * are discovered even without a `.stash` marker.
+ *
+ * Skips `root` itself since the caller already checked it via `hasStashDirs`.
+ */
+const BFS_MAX_DEPTH = 5;
+function findShallowestStashRoot(root) {
+    const queue = [{ dir: root, depth: 0 }];
+    while (queue.length > 0) {
+        const item = queue.shift();
+        if (!item) {
+            continue;
+        }
+        const { dir: current, depth } = item;
+        if (current !== root) {
+            // .stash directory is a strong stash marker
+            if (isDirectory(path.join(current, ".stash"))) {
+                return current;
+            }
+            // Require 2+ type dirs for BFS candidates to avoid false positives.
+            // A single "scripts/" is too common (skill dirs, npm packages, etc.).
+            if (countStashDirs(current) >= 2) {
+                return current;
+            }
+        }
+        if (depth >= BFS_MAX_DEPTH)
+            continue;
+        let children;
+        try {
+            children = fs.readdirSync(current, { withFileTypes: true });
+        }
+        catch {
+            continue;
+        }
+        for (const child of children) {
+            if (!child.isDirectory())
+                continue;
+            if (child.name === ".git" || child.name === "node_modules")
+                continue;
+            queue.push({ dir: path.join(current, child.name), depth: depth + 1 });
+        }
+    }
+    return undefined;
+}

package/dist/stash-providers/sync-from-ref.js

@@ -0,0 +1,45 @@
+/**
+ * Unified install-ref dispatcher.
+ *
+ * Replaces the historical `installRegistryRef()` entry point. Given an
+ * unparsed install ref, this resolves the right syncable provider and
+ * invokes its `sync()` method.
+ *
+ * Audit is intentionally NOT performed here; callers (`akmAdd`,
+ * `akmUpdate`) decide whether to run `auditInstallCandidate` on the
+ * synced `contentDir` because they own the `--trust` flag.
+ */
+import { UsageError } from "../errors";
+import { parseRegistryRef } from "../registry-resolve";
+import { detectStashRoot } from "./provider-utils";
+export async function syncFromRef(ref, options) {
+    const parsed = parseRegistryRef(ref);
+    if (parsed.source === "local") {
+        return syncLocalRef(parsed, options);
+    }
+    if (parsed.source === "npm") {
+        const { syncNpmRef } = await import("./npm");
+        return syncNpmRef(ref, options);
+    }
+    if (parsed.source === "git" || parsed.source === "github") {
+        const { syncRegistryGitRef } = await import("./git");
+        return syncRegistryGitRef(ref, options);
+    }
+    // Exhaustiveness — `parseRegistryRef` only emits the four sources above.
+    throw new UsageError(`No syncable provider for ref: ${ref} (source=${parsed.source})`);
+}
+function syncLocalRef(parsed, options) {
+    const stashRoot = detectStashRoot(parsed.sourcePath);
+    const syncedAt = (options?.now ?? new Date()).toISOString();
+    return {
+        id: parsed.id,
+        source: "local",
+        ref: parsed.ref,
+        artifactUrl: parsed.sourcePath,
+        contentDir: stashRoot,
+        cacheDir: parsed.sourcePath,
+        extractedDir: parsed.sourcePath,
+        writable: options?.writable,
+        syncedAt,
+    };
+}

package/dist/stash-providers/tar-utils.js

@@ -0,0 +1,151 @@
+/**
+ * Tar archive extraction and integrity verification utilities.
+ *
+ * These helpers are security-critical: they validate archive entries to
+ * prevent path traversal, run a post-extraction scan for symlink escapes,
+ * and verify integrity hashes (SRI or hex shasum) before extraction.
+ *
+ * Extracted from `registry-install.ts` and shared by all syncable
+ * providers that fetch tarballs (currently `NpmStashProvider` and the
+ * registry index builder).
+ */
+import { spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { isWithin } from "../common";
+import { warn } from "../warn";
+/**
+ * Verify an archive's integrity against a known hash. Throws and removes
+ * the archive when verification fails.
+ *
+ * Supports SRI hashes (sha256-/sha512-) and hex SHA-1 from npm.
+ * Skips verification for git/github sources (revisions are commit SHAs,
+ * not content hashes).
+ */
+export function verifyArchiveIntegrity(archivePath, expected, source) {
+    if (!expected)
+        return;
+    // For GitHub and git sources, resolvedRevision is a commit SHA, not a content hash.
+    // Content integrity cannot be verified from a commit hash, so skip verification.
+    if (source === "github" || source === "git")
+        return;
+    const fileBuffer = fs.readFileSync(archivePath);
+    // SRI hash format: sha256-<base64> or sha512-<base64>
+    if (expected.startsWith("sha256-") || expected.startsWith("sha512-")) {
+        const dashIndex = expected.indexOf("-");
+        const algorithm = expected.slice(0, dashIndex);
+        const expectedBase64 = expected.slice(dashIndex + 1);
+        const actualBase64 = createHash(algorithm).update(fileBuffer).digest("base64");
+        if (actualBase64 !== expectedBase64) {
+            fs.unlinkSync(archivePath);
+            throw new Error(`Integrity check failed for ${archivePath}: expected ${algorithm} digest ${expectedBase64}, got ${actualBase64}`);
+        }
+        return;
+    }
+    // Hex shasum (SHA-1 from npm)
+    if (/^[0-9a-f]{40}$/i.test(expected)) {
+        const actualHex = createHash("sha1").update(fileBuffer).digest("hex");
+        if (actualHex.toLowerCase() !== expected.toLowerCase()) {
+            fs.unlinkSync(archivePath);
+            throw new Error(`Integrity check failed for ${archivePath}: expected sha1 ${expected}, got ${actualHex}`);
+        }
+        return;
+    }
+    // Unrecognized format — warn and skip verification
+    warn("Unrecognized integrity format: %s — verification skipped", expected);
+}
+/**
+ * Extract a tar.gz archive into `destinationDir`, validating entries first
+ * (no absolute paths, no `..` traversal, no NUL bytes), invoking tar with
+ * `--no-same-owner --strip-components=1`, and finally scanning the extracted
+ * tree for symlinks that would escape the destination.
+ */
+export function extractTarGzSecure(archivePath, destinationDir) {
+    const listResult = spawnSync("tar", ["tzf", archivePath], { encoding: "utf8" });
+    if (listResult.status !== 0) {
+        const err = listResult.stderr?.trim() || listResult.error?.message || "unknown error";
+        throw new Error(`Failed to inspect archive ${archivePath}: ${err}`);
+    }
+    validateTarEntries(listResult.stdout);
+    fs.rmSync(destinationDir, { recursive: true, force: true });
+    fs.mkdirSync(destinationDir, { recursive: true });
+    const extractResult = spawnSync("tar", ["xzf", archivePath, "--no-same-owner", "--strip-components=1", "-C", destinationDir], { encoding: "utf8" });
+    if (extractResult.status !== 0) {
+        const err = extractResult.stderr?.trim() || extractResult.error?.message || "unknown error";
+        throw new Error(`Failed to extract archive ${archivePath}: ${err}`);
+    }
+    // Post-extraction scan: verify all extracted files are within destinationDir
+    // This mitigates TOCTOU between validateTarEntries (list) and tar extract.
+    scanExtractedFiles(destinationDir, destinationDir);
+}
+function scanExtractedFiles(dir, root) {
+    let entries;
+    try {
+        entries = fs.readdirSync(dir, { withFileTypes: true });
+    }
+    catch {
+        return;
+    }
+    for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name);
+        // Reject only entries whose name is exactly the parent-traversal segment
+        // (or `.`). Substring matches (`foo..bar`, `archive..2024.tar`) are
+        // legitimate filenames that the previous `entry.name.includes("..")`
+        // check rejected as false positives — flagged in PR #168 review.
+        if (entry.name === ".." || entry.name === ".") {
+            throw new Error(`Post-extraction scan: suspicious entry name: ${fullPath}`);
+        }
+        // Symlinks: resolve and confirm the target stays inside the destination.
+        if (entry.isSymbolicLink()) {
+            const target = fs.realpathSync(fullPath);
+            if (!isWithin(target, root)) {
+                throw new Error(`Post-extraction scan: symlink escapes destination directory: ${fullPath} -> ${target}`);
+            }
+        }
+        // Belt-and-suspenders: any regular entry whose resolved path lands outside
+        // the destination root is rejected, regardless of how its name looks. This
+        // catches anything the tar pre-validation missed.
+        if (!entry.isSymbolicLink() && !isWithin(fullPath, root)) {
+            throw new Error(`Post-extraction scan: entry escapes destination directory: ${fullPath}`);
+        }
+        if (entry.isDirectory()) {
+            scanExtractedFiles(fullPath, root);
+        }
+    }
+}
+/**
+ * Validate the line-oriented `tar tzf` listing for unsafe entries.
+ *
+ * Rejects:
+ *   - empty/NUL-containing entries
+ *   - absolute paths
+ *   - parent traversal (`..` / `../`)
+ *   - any entry that would still escape after `--strip-components=1`
+ */
+export function validateTarEntries(listOutput) {
+    const lines = listOutput.split(/\r?\n/).filter(Boolean);
+    for (const rawLine of lines) {
+        const entry = rawLine.trim();
+        if (!entry || entry.includes("\0")) {
+            throw new Error(`Archive contains an invalid entry: ${JSON.stringify(rawLine)}`);
+        }
+        if (entry.startsWith("/")) {
+            throw new Error(`Archive contains an absolute path entry: ${entry}`);
+        }
+        const normalized = path.posix.normalize(entry);
+        if (normalized === ".." || normalized.startsWith("../")) {
+            throw new Error(`Archive contains a path traversal entry: ${entry}`);
+        }
+        const parts = normalized.split("/").filter(Boolean);
+        const stripped = parts.slice(1).join("/");
+        if (!stripped)
+            continue;
+        const normalizedStripped = path.posix.normalize(stripped);
+        if (normalizedStripped === ".." ||
+            normalizedStripped.startsWith("../") ||
+            path.posix.isAbsolute(normalizedStripped)) {
+            throw new Error(`Archive contains an unsafe entry after strip-components: ${entry}`);
+        }
+    }
+}

package/dist/stash-providers/website.js

@@ -1,11 +1,11 @@
 import { createHash } from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
-import { fetchWithRetry } from "../common";
+import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap } from "../common";
 import { ConfigError, UsageError } from "../errors";
 import { getRegistryIndexCacheDir } from "../paths";
 import { registerStashProvider } from "../stash-provider-factory";
-import { isExpired, sanitizeString } from "./provider-utils";
+import { isDirectory, isExpired, sanitizeString } from "./provider-utils";
 /** Refresh website snapshots every 12 hours to balance freshness with scraping load. */
 const CACHE_REFRESH_INTERVAL_MS = 12 * 60 * 60 * 1000;
 /** Allow up to 7 days of stale snapshots when refresh fails so search remains available during outages. */
@@ -14,10 +14,31 @@ const CACHE_STALE_MS = 7 * 24 * 60 * 60 * 1000;
 const QUEUE_EXPANSION_FACTOR = 5;
 const MAX_PAGES_DEFAULT = 50;
 const MAX_DEPTH_DEFAULT = 3;
+/**
+ * Per-page body cap for website scraping. HTML pages this large are
+ * almost never useful as agent knowledge sources and a runaway server
+ * streaming tens of megabytes would blow memory with no upside.
+ */
+const WEBSITE_PAGE_BYTE_CAP = 5 * 1024 * 1024;
+/**
+ * Wall-clock cap for a full crawl (10 minutes). With per-request timeouts
+ * of 15s and a `maxPages` default of 50, an unresponsive site could
+ * otherwise stall `akm add` for 12.5 minutes with no feedback. Cap the
+ * whole crawl and return what we have when time runs out.
+ */
+const WEBSITE_CRAWL_WALL_CLOCK_MS = 10 * 60 * 1000;
+/**
+ * Website stash provider. Implements {@link SyncableStashProvider} (which
+ * extends LiveStashProvider) — scrapes pages into a local mirror so the FTS5
+ * indexer can walk them.
+ */
 class WebsiteStashProvider {
     type = "website";
+    kind = "syncable";
     name;
+    config;
     constructor(config) {
+        this.config = config;
         this.name = config.name ?? "website";
         validateWebsiteUrl(config.url ?? "");
     }
@@ -33,6 +54,40 @@ class WebsiteStashProvider {
     canShow(_ref) {
         return false;
     }
+    async sync(config, options) {
+        const cachePaths = await ensureWebsiteMirror(config, { requireStashDir: true, force: options?.force });
+        const syncedAt = (options?.now ?? new Date()).toISOString();
+        const url = config.url ?? "";
+        // #123 added "website" to the StashSource union, so we can use it directly.
+        return {
+            id: url,
+            source: "website",
+            ref: url,
+            artifactUrl: url,
+            contentDir: cachePaths.stashDir,
+            cacheDir: cachePaths.rootDir,
+            extractedDir: cachePaths.stashDir,
+            syncedAt,
+        };
+    }
+    getContentDir(config) {
+        const url = config.url ?? "";
+        return getCachePaths(url).stashDir;
+    }
+    async remove(config) {
+        const url = config.url;
+        if (!url)
+            return;
+        const paths = getCachePaths(url);
+        if (isDirectory(paths.rootDir)) {
+            try {
+                fs.rmSync(paths.rootDir, { recursive: true, force: true });
+            }
+            catch {
+                /* best-effort */
+            }
+        }
+    }
 }
 registerStashProvider("website", (config) => new WebsiteStashProvider(config));
 function getCachePaths(siteUrl) {
@@ -49,6 +104,7 @@ async function ensureWebsiteMirror(config, options) {
     const normalizedUrl = validateWebsiteUrl(rawUrl);
     const cachePaths = getCachePaths(normalizedUrl);
     const requireStashDir = options?.requireStashDir === true;
+    const force = options?.force === true;
     let mtime = 0;
     try {
         mtime = fs.statSync(cachePaths.manifestPath).mtimeMs;
@@ -56,7 +112,8 @@ async function ensureWebsiteMirror(config, options) {
     catch {
         /* no cached manifest */
     }
-    if (mtime &&
+    if (!force &&
+        mtime &&
         !isExpired(mtime, CACHE_REFRESH_INTERVAL_MS) &&
         (!requireStashDir || hasExtractedSite(cachePaths.stashDir))) {
         return cachePaths;
@@ -124,7 +181,13 @@ async function crawlWebsite(startUrl, options) {
     const queue = [{ url: start.toString(), depth: 0 }];
     const visited = new Set();
     const pages = [];
+    const deadline = Date.now() + WEBSITE_CRAWL_WALL_CLOCK_MS;
+    let stoppedAtDeadline = false;
     while (queue.length > 0 && pages.length < options.maxPages) {
+        if (Date.now() > deadline) {
+            stoppedAtDeadline = true;
+            break;
+        }
         const next = queue.shift();
         if (!next)
             break;
@@ -149,6 +212,9 @@ async function crawlWebsite(startUrl, options) {
             queue.push({ url: candidate, depth: next.depth + 1 });
         }
     }
+    if (stoppedAtDeadline) {
+        console.warn(`[akm] website crawl stopped at the ${WEBSITE_CRAWL_WALL_CLOCK_MS / 1000}s wall-clock cap with ${pages.length}/${options.maxPages} pages collected from ${startUrl}.`);
+    }
     return pages;
 }
 async function fetchWebsitePage(pageUrl) {
@@ -164,7 +230,17 @@ async function fetchWebsitePage(pageUrl) {
         throw new Error(`Failed to fetch website content (${response.status}) from ${pageUrl}`);
     }
     const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
-    const body = await response.text();
+    let body;
+    try {
+        body = await readBodyWithByteCap(response, WEBSITE_PAGE_BYTE_CAP);
+    }
+    catch (err) {
+        if (err instanceof ResponseTooLargeError) {
+            // Skip oversized pages rather than aborting the whole crawl.
+            return null;
+        }
+        throw err;
+    }
     const finalUrl = normalizeCrawlUrl(response.url || pageUrl) ?? pageUrl;
     if (contentType.includes("text/html") || contentType.includes("application/xhtml+xml") || looksLikeMarkup(body)) {
         const title = extractHtmlTitle(body) || new URL(finalUrl).hostname;