akm-cli 0.9.11 → 0.9.13

package/dist/setup/setup.js

@@ -27,7 +27,7 @@ import { listTopLevelConfigKeys } from "../core/config/config-schema.js";
 import { deepMergeConfig, isPlainObject } from "../core/config/deep-merge.js";
 import { ConfigError, UsageError } from "../core/errors.js";
 import { getConfigPath, getDefaultStashDir, isTransientStashPath } from "../core/paths.js";
-import { warn } from "../core/warn.js";
+import { warn, warnOnce } from "../core/warn.js";
 import { akmIndex } from "../indexer/indexer.js";
 import { detectAgentCliProfiles, pickDefaultAgentProfile } from "../integrations/agent/index.js";
 import { defaultProfileName } from "../integrations/harnesses/index.js";
@@ -51,32 +51,25 @@ import { stepStashDir } from "./steps/stashdir.js";
 import { stepScheduledTasks } from "./steps/tasks.js";
 // ── Setup sandbox guard ─────────────────────────────────────────────────────
 /**
- * Refuse to persist an explicit `--dir /tmp/...` as the default bundle path.
- * The OS may reap the directory at any time, leaving the next run with a
- * default bundle that points at a deleted path. Mirrors the
- * `assertInitSandbox` check in commands/init.ts, but
- * fires under all runtimes (not just `bun test`) because `akm setup --dir
- * /tmp/X` is a documented isolation pattern that has been observed to
- * silently clobber the host config (the 2026-05-23 setup-clobbers-user-config
- * incident).
- *
- * Escape hatch: set `AKM_FORCE_SETUP_TMP_STASH=1` to override. When the
- * escape hatch is on, `applyStashIsolationToEnv` below also pre-sets
- * `AKM_BUNDLE_DIR` so that the `getConfigDir` / `getCacheDir` isolation
- * rules fire and config + cache writes route into `$stashDir/.akm/`
- * instead of the user's host `~/.config/akm`.
+ * Warn when an explicit `--dir /tmp/...` is persisted as the default bundle
+ * path — `akm setup --dir $(mktemp -d)` resolves under this family on every
+ * macOS machine (`/var/folders/...`), and used to be refused outright with
+ * no discoverable remedy. The OS may reap the directory at any time, so the
+ * operator is told up front; `applyStashIsolationToEnv` below still pre-sets
+ * `AKM_BUNDLE_DIR` so the `getConfigDir` / `getCacheDir` isolation rules
+ * (`core/paths.ts`) route config + cache writes into `$stashDir/.akm/`
+ * instead of the user's host `~/.config/akm`, which is what actually
+ * protects the host config (the 2026-05-23 setup-clobbers-user-config
+ * incident this guard traces to).
  */
 export function assertSetupSandbox(stashDir, dirExplicitlyProvided) {
     if (!dirExplicitlyProvided)
         return;
-    if (process.env.AKM_FORCE_SETUP_TMP_STASH === "1")
-        return;
     if (!isTransientStashPath(stashDir))
         return;
-    throw new ConfigError(`refusing to run \`akm setup --dir ${stashDir}\`: the path is in a transient/sandbox directory family the OS may reap. ` +
-        "Persisting it as the default bundle would leave the next run pointing at a deleted path. " +
-        "Use a persistent directory, OR set AKM_FORCE_SETUP_TMP_STASH=1 if you intentionally want a sandbox setup " +
-        "(setup will also auto-isolate config + cache writes into $stashDir/.akm/ so the host config is preserved).", "SETUP_TMP_STASH_REFUSED");
+    warnOnce(`setup-tmp-stash:${stashDir}`, `\`akm setup --dir ${stashDir}\` targets a transient/sandbox directory the OS may reap at any time; ` +
+        `the next run would then point at a deleted bundle. Config and cache writes are isolated into ` +
+        `${stashDir}/.akm/ so the host config is unaffected, but treat this stash as disposable.`);
 }
 /**
  * Propagate the explicit `--dir <stashDir>` choice to the env so that the

package/dist/sources/include.js

@@ -4,6 +4,7 @@
 import fs from "node:fs";
 import path from "node:path";
 import { isWithin } from "../core/common.js";
+import { warnOnce } from "../core/warn.js";
 // ── Helpers ─────────────────────────────────────────────────────────────────
 /** Key to check in package.json for akm include configuration. */
 const INCLUDE_CONFIG_KEYS = ["akm"];
@@ -64,51 +65,180 @@ export function findNearestIncludeConfig(startDir, boundary) {
     }
     return undefined;
 }
+/**
+ * Does `entry` use glob syntax (`*`/`?`)? Everything else is a literal path,
+ * resolved and copied exactly as before — this only changes behavior for
+ * entries that actually contain a wildcard.
+ */
+function isIncludeEntryPattern(entry) {
+    return /[*?]/.test(entry);
+}
+/**
+ * Translate one `akm.include` entry to a regular expression matched against a
+ * `/`-joined path relative to `sourceDir`. Supports the subset package.json
+ * authors reach for in practice: `*` (any run of characters within one path
+ * segment), `?` (one character), and `**` (any run of characters across
+ * segments, including none) — e.g. `skills/*` or `skills/**\/*.md`. Not a
+ * full glob implementation (no brace expansion, character classes, or
+ * negation); those fall through to being treated as a literal, almost
+ * certainly non-matching path, same as an ordinary typo would.
+ */
+function includeEntryToRegExp(entry) {
+    const normalized = entry.split(path.sep).join("/");
+    let pattern = "";
+    for (let i = 0; i < normalized.length;) {
+        if (normalized.startsWith("**", i)) {
+            const consumesSlash = normalized[i + 2] === "/";
+            pattern += consumesSlash ? "(?:.*/)?" : ".*";
+            i += consumesSlash ? 3 : 2;
+            continue;
+        }
+        const char = normalized[i];
+        if (char === "*")
+            pattern += "[^/]*";
+        else if (char === "?")
+            pattern += "[^/]";
+        else
+            pattern += char.replace(/[.+^${}()|[\]\\]/g, "\\$&");
+        i += 1;
+    }
+    return new RegExp(`^${pattern}$`);
+}
+/** A symlink whose lstat succeeds but whose target does not exist — `copyPath` reports this case with its own specific warning. */
+function isBrokenSymlink(candidate) {
+    try {
+        return fs.lstatSync(candidate).isSymbolicLink();
+    }
+    catch {
+        return false;
+    }
+}
+/** Every path under `dir`, `/`-joined and relative to `base`, without following symlinked directories. */
+function listAllRelativePaths(dir, base) {
+    const out = [];
+    let entries;
+    try {
+        entries = fs.readdirSync(dir, { withFileTypes: true });
+    }
+    catch {
+        return out;
+    }
+    for (const entry of entries) {
+        if (entry.name === ".git")
+            continue;
+        const absolute = path.join(dir, entry.name);
+        out.push(path.relative(base, absolute).split(path.sep).join("/"));
+        // `isDirectory()` on a `Dirent` reflects lstat, so a symlinked directory
+        // is listed as an entry (and handled by copyPath's own symlink handling)
+        // but never recursed into here.
+        if (entry.isDirectory())
+            out.push(...listAllRelativePaths(absolute, base));
+    }
+    return out;
+}
+/**
+ * Resolve one `akm.include` entry to the absolute candidate paths it names.
+ * A literal entry resolves to itself, whether or not it exists on disk — the
+ * caller checks that (after re-verifying containment, which needs to see
+ * the literal candidate regardless). A pattern entry expands to every
+ * existing matching path under `sourceDir`, so matching nothing already
+ * naturally comes back as an empty array.
+ */
+function resolveIncludeEntry(entry, sourceDir) {
+    if (!isIncludeEntryPattern(entry))
+        return [path.resolve(sourceDir, entry)];
+    const pattern = includeEntryToRegExp(entry);
+    return listAllRelativePaths(sourceDir, sourceDir)
+        .filter((relative) => pattern.test(relative))
+        .map((relative) => path.join(sourceDir, relative));
+}
 /**
  * Copy each glob/path in `includeGlobs` from `sourceDir` to `destDir`.
  *
  * Uses `isWithin()` to prevent path-traversal attacks: any entry that escapes
- * `sourceDir` throws immediately rather than silently being skipped.
+ * `sourceDir` throws immediately — this is the operator's own config
+ * (`akm.include` in a package.json under a source they added), not
+ * discovered content, so a deliberate `../` here is treated as the mistake
+ * (or attack) it looks like, not skipped.
  *
- * @throws {Error} if an include path escapes `sourceDir` or does not exist on disk.
+ * A missing entry, or a pattern that matches nothing, is skipped with a
+ * warning instead of failing the whole install — one bad entry in a large
+ * `include` list should not leave zero assets installed.
  */
 export function copyIncludedPaths(includeGlobs, sourceDir, destDir) {
     for (const entry of includeGlobs) {
-        const resolvedSource = path.resolve(sourceDir, entry);
-        if (!isWithin(resolvedSource, sourceDir)) {
-            throw new Error(`Path in akm.include escapes the package root: ${entry}`);
-        }
-        if (!fs.existsSync(resolvedSource)) {
-            throw new Error(`Path in akm.include does not exist: ${entry}`);
-        }
-        if (path.basename(resolvedSource) === ".git") {
+        const resolvedPaths = resolveIncludeEntry(entry, sourceDir);
+        if (resolvedPaths.length === 0) {
+            warnOnce(`akm-include-missing:${sourceDir}:${entry}`, `[akm] Skipping "${entry}" in akm.include: no matching file or directory.`);
             continue;
         }
-        const relativePath = path.relative(sourceDir, resolvedSource);
-        if (!relativePath || relativePath === ".") {
-            copyDirectoryContents(sourceDir, destDir);
-            continue;
+        for (const resolvedSource of resolvedPaths) {
+            if (!isWithin(resolvedSource, sourceDir)) {
+                throw new Error(`Path in akm.include escapes the package root: ${entry}`);
+            }
+            if (!fs.existsSync(resolvedSource) && !isBrokenSymlink(resolvedSource)) {
+                warnOnce(`akm-include-missing:${sourceDir}:${entry}`, `[akm] Skipping "${entry}" in akm.include: no matching file or directory.`);
+                continue;
+            }
+            if (path.basename(resolvedSource) === ".git")
+                continue;
+            const relativePath = path.relative(sourceDir, resolvedSource);
+            if (!relativePath || relativePath === ".") {
+                copyDirectoryContents(sourceDir, destDir, sourceDir);
+                continue;
+            }
+            copyPath(resolvedSource, path.join(destDir, relativePath), sourceDir);
         }
-        copyPath(resolvedSource, path.join(destDir, relativePath));
     }
 }
 // ── Private helpers ─────────────────────────────────────────────────────────
-function copyDirectoryContents(sourceDir, destinationDir) {
+function copyDirectoryContents(sourceDir, destinationDir, containmentRoot) {
     for (const entry of fs.readdirSync(sourceDir, { withFileTypes: true })) {
         if (entry.name === ".git")
             continue;
-        copyPath(path.join(sourceDir, entry.name), path.join(destinationDir, entry.name));
+        copyPath(path.join(sourceDir, entry.name), path.join(destinationDir, entry.name), containmentRoot);
     }
 }
-function copyPath(sourcePath, destinationPath) {
+/**
+ * Copy one path. A symlink is followed and its TARGET's content copied
+ * (never re-materialized as a symlink) rather than refused outright — a
+ * published package with, say, `README.md -> docs/README.md` is ordinary,
+ * not an attack. The resolved target must still stay within
+ * `containmentRoot`: this is the same containment guarantee `isWithin`
+ * enforces on every literal `akm.include` entry, just applied to a target we
+ * discovered by following a link rather than one the operator typed — a
+ * symlink escaping the package (to `/etc/passwd`, say) is skipped with a
+ * warning, not copied.
+ */
+function copyPath(sourcePath, destinationPath, containmentRoot) {
     const stat = fs.lstatSync(sourcePath);
     if (stat.isSymbolicLink()) {
-        throw new Error(`Path in akm.include must not be a symlink: ${sourcePath}`);
+        let resolved;
+        try {
+            resolved = fs.realpathSync(sourcePath);
+        }
+        catch {
+            warnOnce(`akm-include-broken-symlink:${sourcePath}`, `[akm] Skipping broken symlink in akm.include: ${path.relative(containmentRoot, sourcePath)}`);
+            return;
+        }
+        if (!isWithin(resolved, containmentRoot)) {
+            warnOnce(`akm-include-symlink-escape:${sourcePath}`, `[akm] Skipping symlink in akm.include that points outside the package: ${path.relative(containmentRoot, sourcePath)}`);
+            return;
+        }
+        const resolvedStat = fs.statSync(resolved);
+        fs.mkdirSync(path.dirname(destinationPath), { recursive: true });
+        if (resolvedStat.isDirectory()) {
+            fs.mkdirSync(destinationPath, { recursive: true });
+            copyDirectoryContents(resolved, destinationPath, containmentRoot);
+            return;
+        }
+        fs.copyFileSync(resolved, destinationPath);
+        return;
     }
     fs.mkdirSync(path.dirname(destinationPath), { recursive: true });
     if (stat.isDirectory()) {
         fs.mkdirSync(destinationPath, { recursive: true });
-        copyDirectoryContents(sourcePath, destinationPath);
+        copyDirectoryContents(sourcePath, destinationPath, containmentRoot);
         return;
     }
     fs.copyFileSync(sourcePath, destinationPath);

package/dist/sources/providers/git-install.js

@@ -350,25 +350,27 @@ export function syncExistingWritableCheckout(parsed, resolved, contentRoot, sync
         targetRevision = fallbackTarget.stdout.trim();
     }
     const relation = gitRelation(repoDir, targetRevision);
-    if (relation.ahead > 0) {
-        throw new UsageError(`Writable Git install at ${root} has local commits that are not in the requested upstream revision; push or reconcile them before update.`);
-    }
-    assertNoIgnoredPathOverwrite(repoDir, targetRevision);
     const rootsToPreserve = [...new Set([root, ...requiredRoots.map((candidate) => path.resolve(candidate))])];
     for (const requiredRoot of rootsToPreserve) {
         if (!isWithin(requiredRoot, repoDir)) {
             throw new UsageError(`Configured Git component root ${requiredRoot} resolves outside ${repoDir}.`);
         }
-        const relative = path.relative(repoDir, requiredRoot).replaceAll(path.sep, "/");
-        if (!relative)
-            continue;
-        const tree = runGit(["-C", repoDir, "ls-tree", "-d", "-z", "--name-only", targetRevision, "--", relative]);
-        const names = tree.stdout.split("\0").filter(Boolean);
-        if (tree.status !== 0 || !names.includes(relative)) {
-            throw new UsageError(`Writable Git update would remove configured content root ${requiredRoot}; the existing checkout was left unchanged.`);
-        }
     }
     if (relation.behind > 0) {
+        if (relation.ahead > 0) {
+            throw new UsageError(`Writable Git install at ${root} has local commits that are not in the requested upstream revision; push or reconcile them before update.`);
+        }
+        assertNoIgnoredPathOverwrite(repoDir, targetRevision);
+        for (const requiredRoot of rootsToPreserve) {
+            const relative = path.relative(repoDir, requiredRoot).replaceAll(path.sep, "/");
+            if (!relative)
+                continue;
+            const tree = runGit(["-C", repoDir, "ls-tree", "-d", "-z", "--name-only", targetRevision, "--", relative]);
+            const names = tree.stdout.split("\0").filter(Boolean);
+            if (tree.status !== 0 || !names.includes(relative)) {
+                throw new UsageError(`Writable Git update would remove configured content root ${requiredRoot}; the existing checkout was left unchanged.`);
+            }
+        }
         const statusBeforeMerge = runGit(["-C", repoDir, "status", "--porcelain"]);
         if (statusBeforeMerge.status !== 0 || statusBeforeMerge.stdout.trim()) {
             throw new UsageError(`Writable Git install at ${root} changed while its update was prepared; local work was preserved.`);

package/dist/sources/providers/git-provider.js

@@ -127,10 +127,10 @@ function pullRepo(repoDir) {
         throw new UsageError(`Writable Git source at ${repoDir} has uncommitted changes; refusing to update it.`);
     }
     const relation = inspectGitUpstream(repoDir);
-    if (relation.ahead > 0) {
-        throw new UsageError(`Writable Git source at ${repoDir} has unpushed commits; refusing to update it.`);
-    }
     if (relation.behind > 0 && relation.upstream) {
+        if (relation.ahead > 0) {
+            throw new UsageError(`Writable Git source at ${repoDir} has unpushed commits; refusing to update it.`);
+        }
         const statusBeforeMerge = runGit(["-C", repoDir, "status", "--porcelain"]);
         if (statusBeforeMerge.status !== 0 || statusBeforeMerge.stdout.trim()) {
             throw new UsageError(`Writable Git source at ${repoDir} changed while its update was prepared; refusing to merge.`);

package/dist/sources/snapshot-fetchers/website-ingest.js

@@ -6,7 +6,7 @@ import fs from "node:fs";
 import path from "node:path";
 import { fetchWithRetry, isWithin, ResponseTooLargeError, readBodyWithByteCap, resolveStashDir, todayIso, } from "../../core/common.js";
 import { ConfigError, UsageError } from "../../core/errors.js";
-import { warn, warnVerbose } from "../../core/warn.js";
+import { warn, warnOnce, warnVerbose } from "../../core/warn.js";
 import { withFreshnessCache } from "../freshness.js";
 import { sanitizeString } from "../providers/provider-utils.js";
 import { getWebsiteCachePaths, normalizeSiteUrl, validateWebsiteInputUrl, validateWebsiteUrl, } from "../website-url.js";
@@ -324,19 +324,20 @@ async function scrapeWebsiteToStash(startUrl, stashDir, options) {
 export async function fetchWebsiteMarkdownSnapshot(rawUrl, options) {
     const normalizedUrl = validateWebsiteInputUrl(rawUrl, { allowPrivateHosts: options?.allowPrivateHosts });
     const parsedUrl = new URL(normalizedUrl);
+    const allowPrivateHosts = await resolveAllowPrivateStartHost(parsedUrl, options?.allowPrivateHosts);
     const stashDir = resolveFetcherStashDir(options?.stashDir);
     const context = {
         stashDir: stashDir ?? "",
         timeoutMs: options?.timeoutMs ?? 15_000,
         signal: options?.signal,
         ...(options?.resolveSecret ? { resolveSecret: options.resolveSecret } : {}),
-        ...(options?.allowPrivateHosts ? { allowPrivateHosts: true } : {}),
+        ...(allowPrivateHosts ? { allowPrivateHosts: true } : {}),
     };
     const snapshot = await dispatchSnapshotFetchers(parsedUrl, context, stashDir);
     if (snapshot)
         return websiteMarkdownSnapshotFromResult(snapshot);
     const fetchedResponse = await fetchWebsiteResponse(normalizedUrl, 0, {
-        allowPrivateHosts: options?.allowPrivateHosts,
+        allowPrivateHosts,
         signal: options?.signal,
     });
     const finalUrl = normalizeCrawlUrl(fetchedResponse.finalUrl) ?? normalizedUrl;
@@ -355,7 +356,7 @@ export async function fetchWebsiteMarkdownSnapshot(rawUrl, options) {
         }
     }
     const fetched = await websitePageFromResponse(fetchedResponse, normalizedUrl, {
-        allowPrivateHosts: options?.allowPrivateHosts,
+        allowPrivateHosts,
         signal: options?.signal,
     });
     if (!fetched)
@@ -465,9 +466,12 @@ async function resolveCrawlRobotsDecision(robots, normalizedUrl, rawUrl) {
 }
 /**
  * C-02/C-03: fail fast, before any page fetch, when the start URL itself is
- * off-limits. A 5xx robots.txt (RobotsPolicy caches `DISALLOW_ALL_RULES` for
- * that case) gets a distinct message calling out the server error, per spec
- * §4.6.
+ * off-limits per an actually-parsed robots.txt rule. A 5xx on robots.txt
+ * itself no longer reaches this as `disallowAll`: `loadRobotsTxt` reports it the same way a
+ * 4xx is reported (`ALLOW_ALL_RULES`), with its own warning, rather than
+ * treating the site's own transient error as a Disallow. The check below
+ * stays as defense-in-depth for a `RobotsPolicy` built over some other
+ * loader that does still report `disallowAll`.
  *
  * Checks both `start`'s (normalized) URL and `rawStartUrl` — the URL exactly
  * as the user supplied it in config, before `validateWebsiteUrl` ->
@@ -506,9 +510,37 @@ async function assertStartUrlAllowedByRobots(robots, start, rawStartUrl) {
             `source to bypass robots.txt.`);
     }
 }
+/**
+ * Decide whether to treat a fetch's start host — the URL an operator directly
+ * typed or configured (`akm bundle add`, `akm knowledge add`, a crawl's own
+ * start URL), never a link discovered while crawling — as private for this
+ * operation. `requested` (already true when the URL itself was
+ * loopback/private-literal, per `shouldAllowPrivateWebsiteUrlForTests`) wins
+ * outright. Otherwise, resolve the host: a name that only turns out to be
+ * private via DNS (a corporate wiki behind split-horizon DNS, a VPN hostname)
+ * would otherwise abort with no escape hatch even though the operator named
+ * this exact host. Callers scope the resulting bypass to same-origin work
+ * only — a link discovered mid-crawl is restricted to the start URL's origin
+ * before it ever reaches a guard, so this never extends trust to a host the
+ * fetched content merely points at.
+ */
+async function resolveAllowPrivateStartHost(start, requested) {
+    if (requested)
+        return true;
+    try {
+        await assertResolvedHostAllowed(start.hostname);
+        return false;
+    }
+    catch {
+        warnOnce(`website-private-start-host:${start.origin}`, `[akm] ${start.origin} is not a publicly routable host, but you added it as a source directly — proceeding. ` +
+            "Links to other hosts found in its content are still checked.");
+        return true;
+    }
+}
 async function crawlWebsite(startUrl, options) {
     const start = new URL(normalizeSiteUrl(startUrl));
     const allowedOrigin = start.origin;
+    const allowPrivateHosts = await resolveAllowPrivateStartHost(start, options.allowPrivateHosts);
     const queue = [
         { url: start.toString(), rawUrl: options.rawStartUrl ?? start.toString(), depth: 0, deferrals: 0 },
     ];
@@ -532,7 +564,7 @@ async function crawlWebsite(startUrl, options) {
     const crawlSignal = abortController.signal;
     const robots = options.respectRobots === false
         ? createAllowAllRobotsPolicy()
-        : createRobotsPolicy((robotsUrl) => loadRobotsTxt(robotsUrl, { allowPrivateHosts: options.allowPrivateHosts, signal: crawlSignal }));
+        : createRobotsPolicy((robotsUrl) => loadRobotsTxt(robotsUrl, { allowPrivateHosts, signal: crawlSignal }));
     await assertStartUrlAllowedByRobots(robots, start, options.rawStartUrl);
     // llms.txt fast path: an increasing number of doc sites publish a curated,
     // deduplicated link list at `/llms.txt` specifically for tools like this
@@ -545,7 +577,7 @@ async function crawlWebsite(startUrl, options) {
     // whole site's manifest.
     if (isOriginRootUrl(start)) {
         const manifest = await fetchLlmsManifest(start, robots, {
-            allowPrivateHosts: options.allowPrivateHosts,
+            allowPrivateHosts,
             signal: crawlSignal,
         });
         if (manifest) {
@@ -623,7 +655,7 @@ async function crawlWebsite(startUrl, options) {
         deferred.delete(normalized);
         fetchAttempts++;
         const fetched = await fetchWebsitePage(decision.fetchUrl, {
-            allowPrivateHosts: options.allowPrivateHosts,
+            allowPrivateHosts,
             robots,
             signal: crawlSignal,
         });
@@ -851,12 +883,18 @@ export async function loadRobotsTxt(robotsUrl, options) {
         }
         if (response.status >= 500 && response.status < 600) {
             await response.body?.cancel().catch(() => undefined);
-            // RFC 9309 §2.3.1.4: an unreachable robots.txt is a full disallow, not
-            // an allow-all. `fetchWithRetry` already retried this once, so a
-            // transient blip does not trip it.
-            warn("[akm] robots.txt at %s returned %d; treating the crawl as fully disallowed until it recovers. " +
-                "Set respectRobots: false on this website source to bypass robots.txt.", robotsUrl, response.status);
-            return { kind: "unreachable" };
+            // Guard-audit finding 12: RFC 9309 §2.3.1.4 recommends treating an
+            // unreachable robots.txt as a full disallow, but for a personal tool
+            // crawling a site the operator explicitly chose, a transient 5xx on
+            // the site's OWN robots.txt is not a site owner's Disallow — it is the
+            // site being flaky. `fetchWithRetry` already retried this once, so
+            // this is not a single blip; still, blocking the crawl outright over
+            // it serves the operator worse than proceeding unrestricted. Reported
+            // as "unavailable" (the same outcome a 4xx gets), which resolves to
+            // ALLOW_ALL_RULES rather than DISALLOW_ALL_RULES.
+            warn("[akm] robots.txt at %s returned %d; proceeding as if no robots.txt were published, rather than treating " +
+                "the crawl as fully disallowed. Set respectRobots: false on this website source to silence this warning.", robotsUrl, response.status);
+            return { kind: "unavailable" };
         }
         // 4xx (404 is the common, silent case), and any other non-2xx/5xx status.
         await response.body?.cancel().catch(() => undefined);

package/dist/sources/website-url.js

@@ -12,20 +12,28 @@
 import { createHash } from "node:crypto";
 import path from "node:path";
 import { ConfigError, UsageError } from "../core/errors.js";
+import { classifyNetworkHostname } from "../core/network-policy.js";
 import { getRegistryIndexCacheDir } from "../core/paths.js";
-import { assertWebsiteRequestUrl, isLoopbackWebsiteHostname, } from "./snapshot-fetchers/host-guard.js";
+import { warnOnce } from "../core/warn.js";
+import { assertWebsiteRequestUrl } from "./snapshot-fetchers/host-guard.js";
 export function shouldAllowPrivateWebsiteHostsForTests() {
     return process.env.BUN_TEST === "1" || process.env.NODE_ENV === "test";
 }
 export function shouldAllowPrivateWebsiteUrlForTests(rawUrl) {
-    if (!shouldAllowPrivateWebsiteHostsForTests())
-        return false;
+    let hostname;
     try {
-        return isLoopbackWebsiteHostname(new URL(rawUrl).hostname.toLowerCase());
+        hostname = new URL(rawUrl).hostname.toLowerCase();
     }
     catch {
         return false;
     }
+    if (classifyNetworkHostname(hostname) === "public")
+        return false;
+    if (!shouldAllowPrivateWebsiteHostsForTests()) {
+        warnOnce(`website-private-host:${hostname}`, `[akm] "${hostname}" is not a publicly routable host, but you added it as a source directly — proceeding. ` +
+            "Links discovered elsewhere are still checked.");
+    }
+    return true;
 }
 export function getWebsiteCachePaths(siteUrl, cacheRootOverride) {
     const key = createHash("sha256").update(normalizeSiteUrl(siteUrl)).digest("hex").slice(0, 16);

package/dist/storage/engines/sqlite-migrations.js

@@ -39,17 +39,28 @@ function inspectLedgerAgainst(db, registryIds) {
     const migrationIds = rows.map((row) => row.id);
     for (const [index, row] of rows.entries()) {
         const expectedId = registryIds[index];
+        // Every id this binary knows matched in order and the ledger carries more:
+        // the database was migrated by a newer akm. Nothing here is applicable —
+        // this binary's whole registry is already applied — so this is version
+        // skew, not divergence.
         if (!expectedId) {
-            return { status: "newer", migrationIds, detail: `unknown migration ID ${row.id}` };
+            return {
+                status: "newer",
+                migrationIds,
+                detail: `applied migration ID${rows.length - registryIds.length === 1 ? "" : "s"} ${migrationIds
+                    .slice(registryIds.length)
+                    .join(", ")} unknown to this akm`,
+            };
         }
+        // A mismatch at a position this binary has a migration for is divergence,
+        // whether or not the id is one this binary knows later: this binary's
+        // migration at `index` was never applied, and something else was.
         if (row.id !== expectedId) {
-            const knownLater = registryIds.includes(row.id);
             return {
-                status: knownLater ? "inconsistent" : "newer",
+                status: "inconsistent",
                 migrationIds,
-                detail: knownLater
-                    ? `migration ledger is not an exact ordered prefix at position ${index + 1}`
-                    : `unknown migration ID ${row.id}`,
+                detail: `migration ledger is not an exact ordered prefix at position ${index + 1} (found '${row.id}', expected '${expectedId}'). ` +
+                    `Applied, in order: [${migrationIds.join(", ")}]. This akm's expected order: [${registryIds.join(", ")}].`,
             };
         }
     }
@@ -62,13 +73,32 @@ export function inspectMigrationLedger(db, migrations) {
     assertMigrationRegistry(migrations);
     return inspectLedgerAgainst(db, migrations.map((migration) => migration.id));
 }
+/**
+ * Reject only a ledger this binary cannot reason about at all.
+ *
+ * A `newer` ledger — an exact ordered prefix of this binary's registry plus
+ * migrations a later akm added — is NOT rejected. Two akm versions sharing one
+ * data directory is a supported deployment (a bundled CLI alongside a newer
+ * global install), and refusing the open bricked the older one for every
+ * command while protecting nothing: its entire registry is already applied, so
+ * it has no pending migration to run. Callers that want to tell an operator
+ * about the skew read {@link MigrationLedgerState.status}.
+ *
+ * A `inconsistent` ledger is different: this binary has a migration that was
+ * never applied and something else was applied in its place, so running the
+ * pending set could conflict with schema it cannot see. That still refuses.
+ */
 export function assertMigrationLedger(db, migrations) {
     const state = inspectMigrationLedger(db, migrations);
-    if (state.status === "newer") {
-        throw new Error(`Refusing to open a database with a newer migration ledger: ${state.detail}.`);
-    }
     if (state.status === "inconsistent") {
-        throw new Error(`Refusing a database whose migrations are not an exact ordered prefix: ${state.detail}.`);
+        throw new Error(`Refusing a database whose migrations are not an exact ordered prefix: ${state.detail} ` +
+            "Applying this binary's missing migration now could run it against a schema a later migration already " +
+            "changed underneath it, which is a real risk of producing a wrong schema — not something akm can guess " +
+            "its way out of safely. This usually means the database was migrated by an incompatible akm build or " +
+            "fork, or schema_migrations was edited by hand. Restore this file from a backup taken before the " +
+            "divergence, or — if there is no backup and the data is not needed — delete it and let akm rebuild it " +
+            "from scratch (a derived index.db regenerates from your sources on the next 'akm index'; state.db loses " +
+            "durable history such as improve/proposal state and must be treated as a last resort).");
     }
     return state;
 }

package/dist/storage/like-pattern.js

@@ -0,0 +1,7 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/** Escape `%`, `_`, and `\\` so a LIKE pattern matches `text` literally; pair with `ESCAPE '\\'`. */
+export function escapeLikePattern(text) {
+    return text.replace(/[\\%_]/g, (ch) => `\\${ch}`);
+}

package/dist/storage/repositories/extract-sessions-repository.js

@@ -73,6 +73,29 @@ export function getLastExtractRunAt(db, harness) {
     const ms = Date.parse(row.last);
     return Number.isFinite(ms) ? ms : null;
 }
+/**
+ * #914: outcome counts from the `extract_sessions_seen` ledger since
+ * `sinceIso`, grouped by harness/outcome/skipReason/engine. This is what lets
+ * `akm health`'s `session-extraction` check derive its verdict from what
+ * standalone `akm proposal extract` actually recorded — the only write path
+ * the hook-driven SessionEnd extraction uses (`improve_runs` is never
+ * populated by it). `skipReason`/`engine` are read from `metadata_json` via
+ * `json_extract` and come back `null` both for a non-skip outcome and for a
+ * row a release before #912/#913 wrote (no key at that path) — callers must
+ * treat a `null` reason as "skipped, reason unknown", not as evidence of a
+ * specific cause.
+ */
+export function getExtractOutcomeCountsSince(db, sinceIso) {
+    return db
+        .prepare(`SELECT harness, outcome,
+              json_extract(metadata_json, '$.skipReason') AS skipReason,
+              json_extract(metadata_json, '$.engine') AS engine,
+              COUNT(*) AS count
+       FROM extract_sessions_seen
+       WHERE processed_at >= ?
+       GROUP BY harness, outcome, skipReason, engine`)
+        .all(sinceIso);
+}
 /**
  * Decide whether a session should be skipped because the extractor has already
  * processed BYTE-IDENTICAL content (#602). The skip authority is the content