akm-cli 0.9.0-beta.50 → 0.9.0-beta.51

package/dist/sources/website-ingest.js

@@ -3,6 +3,7 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 import { createHash } from "node:crypto";
 import fs from "node:fs";
+import { isIP } from "node:net";
 import path from "node:path";
 import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap, resolveStashDir } from "../core/common.js";
 import { ConfigError, UsageError } from "../core/errors.js";
@@ -31,6 +32,20 @@ const WEBSITE_PAGE_BYTE_CAP = 5 * 1024 * 1024;
  * whole crawl and return what we have when time runs out.
  */
 const WEBSITE_CRAWL_WALL_CLOCK_MS = 10 * 60 * 1000;
+const WEBSITE_MAX_REDIRECTS = 8;
+export function shouldAllowPrivateWebsiteHostsForTests() {
+    return process.env.BUN_TEST === "1" || process.env.NODE_ENV === "test";
+}
+export function shouldAllowPrivateWebsiteUrlForTests(rawUrl) {
+    if (!shouldAllowPrivateWebsiteHostsForTests())
+        return false;
+    try {
+        return isLoopbackWebsiteHostname(new URL(rawUrl).hostname.toLowerCase());
+    }
+    catch {
+        return false;
+    }
+}
 function resolveFetcherStashDir(explicitStashDir) {
     if (explicitStashDir)
         return explicitStashDir;
@@ -52,7 +67,7 @@ export function getWebsiteCachePaths(siteUrl) {
 }
 export async function ensureWebsiteMirror(config, options) {
     const rawUrl = config.url ?? "";
-    const normalizedUrl = validateWebsiteUrl(rawUrl);
+    const normalizedUrl = validateWebsiteUrl(rawUrl, { allowPrivateHosts: options?.allowPrivateHosts });
     const cachePaths = getWebsiteCachePaths(normalizedUrl);
     const requireStashDir = options?.requireStashDir === true;
     const force = options?.force === true;
@@ -74,6 +89,7 @@ export async function ensureWebsiteMirror(config, options) {
         await scrapeWebsiteToStash(normalizedUrl, cachePaths.stashDir, {
             maxPages: coercePositiveInt(config.options?.maxPages, MAX_PAGES_DEFAULT),
             maxDepth: coercePositiveInt(config.options?.maxDepth, MAX_DEPTH_DEFAULT),
+            allowPrivateHosts: options?.allowPrivateHosts,
         });
         fs.writeFileSync(cachePaths.manifestPath, `${JSON.stringify({ url: normalizedUrl, fetchedAt: new Date().toISOString() }, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
         return cachePaths;
@@ -126,7 +142,7 @@ async function scrapeWebsiteToStash(startUrl, stashDir, options) {
     }
 }
 export async function fetchWebsiteMarkdownSnapshot(rawUrl, options) {
-    const normalizedUrl = validateWebsiteInputUrl(rawUrl);
+    const normalizedUrl = validateWebsiteInputUrl(rawUrl, { allowPrivateHosts: options?.allowPrivateHosts });
     const parsedUrl = new URL(normalizedUrl);
     const stashDir = resolveFetcherStashDir(options?.stashDir);
     const context = {
@@ -147,7 +163,7 @@ export async function fetchWebsiteMarkdownSnapshot(rawUrl, options) {
             warn("[akm] wiki-fetcher %s threw on %s: %s", fetcher.name, normalizedUrl, error instanceof Error ? error.message : String(error));
         }
     }
-    const fetched = await fetchWebsitePage(normalizedUrl);
+    const fetched = await fetchWebsitePage(normalizedUrl, { allowPrivateHosts: options?.allowPrivateHosts });
     if (!fetched) {
         throw new UsageError(`No content could be fetched from ${normalizedUrl}`);
     }
@@ -189,7 +205,7 @@ async function crawlWebsite(startUrl, options) {
         if (!normalized || visited.has(normalized))
             continue;
         visited.add(normalized);
-        const fetched = await fetchWebsitePage(normalized);
+        const fetched = await fetchWebsitePage(normalized, { allowPrivateHosts: options.allowPrivateHosts });
         if (!fetched)
             continue;
         pages.push(fetched.page);
@@ -211,17 +227,8 @@ async function crawlWebsite(startUrl, options) {
     }
     return pages;
 }
-async function fetchWebsitePage(pageUrl) {
-    const parsedUrl = new URL(pageUrl);
-    if (parsedUrl.hostname.endsWith(".invalid")) {
-        throw new Error(`Refusing to fetch reserved invalid hostname: ${parsedUrl.hostname}`);
-    }
-    const response = await fetchWithRetry(pageUrl, {
-        headers: {
-            Accept: "text/html, text/markdown, text/plain;q=0.9, application/xhtml+xml;q=0.8",
-            "User-Agent": "akm-cli website provider",
-        },
-    }, { timeout: 15_000, retries: 1 });
+async function fetchWebsitePage(pageUrl, options) {
+    const response = await fetchWebsiteResponse(pageUrl, 0, options);
     if (!response.ok) {
         if (response.status === 404)
             return null;
@@ -238,6 +245,7 @@ async function fetchWebsitePage(pageUrl) {
         throw err;
     }
     const finalUrl = normalizeCrawlUrl(response.url || pageUrl) ?? pageUrl;
+    assertWebsiteRequestUrl(finalUrl, Error, options);
     if (contentType.includes("text/html") || contentType.includes("application/xhtml+xml") || looksLikeMarkup(body)) {
         const title = extractHtmlTitle(body) || new URL(finalUrl).hostname;
         return {
@@ -258,6 +266,29 @@ async function fetchWebsitePage(pageUrl) {
         links: [],
     };
 }
+async function fetchWebsiteResponse(pageUrl, redirectCount = 0, options) {
+    assertWebsiteRequestUrl(pageUrl, Error, options);
+    const response = await fetchWithRetry(pageUrl, {
+        headers: {
+            Accept: "text/html, text/markdown, text/plain;q=0.9, application/xhtml+xml;q=0.8",
+            "User-Agent": "akm-cli website provider",
+        },
+        redirect: "manual",
+    }, { timeout: 15_000, retries: 1 });
+    if (response.status >= 300 && response.status < 400) {
+        if (redirectCount >= WEBSITE_MAX_REDIRECTS) {
+            throw new Error(`Too many redirects while fetching ${pageUrl}`);
+        }
+        const location = response.headers.get("location");
+        if (!location) {
+            throw new Error(`Redirect response from ${pageUrl} did not include a Location header`);
+        }
+        const nextUrl = new URL(location, pageUrl).toString();
+        assertWebsiteRequestUrl(nextUrl, Error, options);
+        return fetchWebsiteResponse(nextUrl, redirectCount + 1, options);
+    }
+    return response;
+}
 function buildMarkdownSnapshot(page, slug, tags) {
     const title = sanitizeString(page.title, 200) || slug;
     const description = sanitizeString(`Snapshot of ${page.url}`, 500);
@@ -282,13 +313,13 @@ function buildMarkdownSnapshot(page, slug, tags) {
         "",
     ].join("\n");
 }
-export function validateWebsiteUrl(rawUrl) {
-    return validateWebsiteUrlWithError(rawUrl, ConfigError);
+export function validateWebsiteUrl(rawUrl, options) {
+    return validateWebsiteUrlWithError(rawUrl, ConfigError, options);
 }
-export function validateWebsiteInputUrl(rawUrl) {
-    return validateWebsiteUrlWithError(rawUrl, UsageError);
+export function validateWebsiteInputUrl(rawUrl, options) {
+    return validateWebsiteUrlWithError(rawUrl, UsageError, options);
 }
-function validateWebsiteUrlWithError(rawUrl, ErrorType) {
+function validateWebsiteUrlWithError(rawUrl, ErrorType, options) {
     if (!rawUrl) {
         throw new ErrorType("Website provider requires a URL");
     }
@@ -305,6 +336,7 @@ function validateWebsiteUrlWithError(rawUrl, ErrorType) {
     if (parsed.username || parsed.password) {
         throw new ErrorType("Website URL must not contain embedded credentials");
     }
+    assertWebsiteRequestUrl(parsed.toString(), ErrorType, options);
     parsed.hash = "";
     return normalizeSiteUrl(parsed.toString());
 }
@@ -503,6 +535,90 @@ function isAssetLikePath(pathname) {
 function isSafeLinkUrl(url) {
     return url.protocol === "http:" || url.protocol === "https:";
 }
+function assertWebsiteRequestUrl(rawUrl, ErrorType = Error, options) {
+    const parsedUrl = new URL(rawUrl);
+    const hostname = parsedUrl.hostname.toLowerCase();
+    if (hostname.endsWith(".invalid")) {
+        throw new ErrorType(`Refusing to fetch reserved invalid hostname: ${parsedUrl.hostname}`);
+    }
+    if (isForbiddenWebsiteHostname(hostname, options)) {
+        throw new ErrorType(`Refusing to fetch non-public website host: ${parsedUrl.hostname}`);
+    }
+}
+// WHATWG URL.hostname wraps IPv6 literals in brackets (e.g. "[::1]"), but
+// node:net's isIP() only recognizes the bare address form and returns 0 for
+// anything bracketed — silently skipping all IPv6 forbidden-host checks
+// below for every hostname parsed off a URL. Strip the brackets before any
+// isIP()/isForbiddenIpv6() call so those checks actually run.
+function stripIpv6Brackets(hostname) {
+    return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
+}
+function isForbiddenWebsiteHostname(hostname, options) {
+    if (options?.allowPrivateHosts === true)
+        return false;
+    if (hostname === "localhost" || hostname.endsWith(".localhost") || hostname === "metadata.google.internal") {
+        return true;
+    }
+    const bareHostname = stripIpv6Brackets(hostname);
+    const ipVersion = isIP(bareHostname);
+    if (ipVersion === 4)
+        return isForbiddenIpv4(bareHostname);
+    if (ipVersion === 6)
+        return isForbiddenIpv6(bareHostname);
+    return false;
+}
+function isLoopbackWebsiteHostname(hostname) {
+    if (hostname === "localhost" || hostname.endsWith(".localhost"))
+        return true;
+    const bareHostname = stripIpv6Brackets(hostname);
+    const ipVersion = isIP(bareHostname);
+    if (ipVersion === 4)
+        return bareHostname.startsWith("127.");
+    if (ipVersion === 6)
+        return bareHostname === "::1";
+    return false;
+}
+function isForbiddenIpv4(hostname) {
+    const parts = hostname.split(".").map((part) => Number.parseInt(part, 10));
+    if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255))
+        return true;
+    const [a, b] = parts;
+    return (a === 0 ||
+        a === 10 ||
+        a === 127 ||
+        (a === 169 && b === 254) ||
+        (a === 172 && b >= 16 && b <= 31) ||
+        (a === 192 && b === 168));
+}
+/**
+ * Extracts the embedded IPv4 address from an IPv4-mapped IPv6 literal
+ * (`::ffff:a.b.c.d` or its canonical hex form `::ffff:xxxx:yyyy`), or
+ * returns null if `hostname` isn't one.
+ */
+function extractIpv4MappedAddress(normalizedHostname) {
+    const match = normalizedHostname.match(/^::ffff:(?:(\d{1,3}(?:\.\d{1,3}){3})|([0-9a-f]{1,4}):([0-9a-f]{1,4}))$/);
+    if (!match)
+        return null;
+    if (match[1])
+        return match[1];
+    const high = Number.parseInt(match[2], 16);
+    const low = Number.parseInt(match[3], 16);
+    return `${(high >> 8) & 0xff}.${high & 0xff}.${(low >> 8) & 0xff}.${low & 0xff}`;
+}
+function isForbiddenIpv6(hostname) {
+    const normalized = hostname.toLowerCase();
+    const mappedIpv4 = extractIpv4MappedAddress(normalized);
+    if (mappedIpv4)
+        return isForbiddenIpv4(mappedIpv4);
+    return (normalized === "::" ||
+        normalized === "::1" ||
+        normalized.startsWith("fc") ||
+        normalized.startsWith("fd") ||
+        normalized.startsWith("fe8") ||
+        normalized.startsWith("fe9") ||
+        normalized.startsWith("fea") ||
+        normalized.startsWith("feb"));
+}
 function stripDangerousBlockTag(value, tagName) {
     const pattern = new RegExp(`<${tagName}\\b[^>]*>[\\s\\S]*?<\\/${tagName}\\s*>`, "gi");
     return value.replace(pattern, "");

package/docs/data-and-telemetry.md

@@ -63,7 +63,7 @@ Override: set `AKM_CACHE_DIR` or `XDG_CACHE_HOME`.
 
 | Path | Contents | Safe to delete? |
 |---|---|---|
-| `<stash>/` | All your asset files: agents, skills, commands, knowledge, workflows, memories, vaults, wikis, lessons | **No** — this is YOUR data |
+| `<stash>/` | All your asset files: agents, skills, commands, knowledge, workflows, memories, env files, secrets, wikis, lessons, facts | **No** — this is YOUR data |
 | `<stash>/.akm/` | Hidden AKM metadata (v0.7 proposals, legacy runs) | Caution — check for pending proposals first |
 
 Override: set `AKM_STASH_DIR` (or configure `stashDir` in `config.json`).
@@ -130,7 +130,7 @@ An append-only log of every mutating action you perform with AKM. Events are sto
 
 ### 2. Proposals Table
 
-The proposal queue: pending, accepted, and rejected improvement proposals for your stash assets. Generated by `akm reflect` and the improve loop.
+The proposal queue: pending, accepted, and rejected improvement proposals for your stash assets. Generated by `akm improve`, `akm propose`, and related proposal-producing flows.
 
 Contents:
 - Proposal UUID (primary key)

package/docs/migration/release-notes/0.9.0.md

@@ -0,0 +1,39 @@
+Migration notes for akm v0.9.0
+
+This is the final 0.9.0 cut. If you were already running a 0.9.0 beta, the
+main upgrade work is to align scripts, prompts, and stash layout with the
+released command surface.
+
+Key operator-facing changes:
+
+- Node.js is supported again for the published CLI. Install with Bun, Node.js
+  >= 20.12, or the prebuilt binary.
+- The legacy `vault` asset type and `akm vault ...` command family are gone.
+  Use `env:` for whole `.env` groups and `secret:` for a single sensitive
+  value.
+- `akm-migrate-storage` still ships and still performs the non-destructive
+  `vaults/` -> `env/` copy for older stashes. Run it before indexing if you are
+  upgrading from a stash that still stores `.env` files only under `vaults/`.
+- Proposal workflow is fully consolidated around `akm improve`, `akm propose`,
+  and `akm proposal ...`. Update any old `akm reflect`, `akm distill`,
+  `akm accept`, `akm reject`, or `akm proposals` usage.
+
+Primary public command family for 0.9.0:
+
+- `akm improve <ref> [--task "..."]`
+- `akm propose <type> <name> (--task "..." | --file <path>)`
+- `akm proposal list`
+- `akm proposal show <id>`
+- `akm proposal diff <id>`
+- `akm proposal accept <id>`
+- `akm proposal reject <id> --reason "..."`
+
+Recommended post-upgrade checks:
+
+- Run `akm help migrate 0.9.0` on every machine where the CLI is installed.
+- Run `akm-migrate-storage --yes` once if the stash ever used `vaults/`.
+- Rebuild the index with `akm index`.
+- Review agent instructions and docs for old `vault`, `reflect`, and `distill`
+  examples.
+
+Full changelog: https://github.com/itlackey/akm/blob/main/CHANGELOG.md

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "akm-cli",
-  "version": "0.9.0-beta.50",
+  "version": "0.9.0-beta.51",
   "type": "module",
   "description": "akm (Agent Knowledge Management) — A package manager for AI agent skills, commands, tools, and knowledge. Works with Claude Code, OpenCode, Cursor, and any AI coding assistant.",
   "keywords": [
@@ -46,11 +46,11 @@
     "docs/data-and-telemetry.md"
   ],
   "bin": {
-    "akm": "dist/cli.js",
-    "akm-migrate-storage": "dist/scripts/migrate-storage.js"
+    "akm": "dist/akm",
+    "akm-migrate-storage": "dist/akm-migrate-storage"
   },
   "scripts": {
-    "preinstall": "node -e \"var ua=process.env.npm_config_user_agent||'';var major=parseInt((process.versions.node||'0').split('.')[0],10);if(process.versions.bun||ua.startsWith('bun/')||process.env.BUN_INSTALL||major>=20){process.exit(0)}console.error('\\n  ERROR: akm-cli requires the Bun runtime (https://bun.sh), Node.js >= 20, or the prebuilt binary.\\n  Install options:\\n    1. Bun:    curl -fsSL https://bun.sh/install | bash  &&  bun install -g akm-cli\\n    2. Binary: curl -fsSL https://github.com/itlackey/akm/releases/latest/download/install.sh | bash\\n');process.exit(1)\"",
+    "preinstall": "node -e \"var ua=process.env.npm_config_user_agent||'';var v=(process.versions.node||'0').split('.').map(function(n){return parseInt(n,10)||0});var ok=v[0]>20||(v[0]===20&&(v[1]>12||(v[1]===12&&v[2]>=0)));if(process.versions.bun||ua.startsWith('bun/')||process.env.BUN_INSTALL||ok){process.exit(0)}console.error('\\n  ERROR: akm-cli requires the Bun runtime (https://bun.sh), Node.js >= 20.12, or the prebuilt binary.\\n  Install options:\\n    1. Bun:    curl -fsSL https://bun.sh/install | bash  &&  bun install -g akm-cli\\n    2. Node:   upgrade to Node.js 20.12 or newer (https://nodejs.org)\\n    3. Binary: curl -fsSL https://github.com/itlackey/akm/releases/latest/download/install.sh | bash\\n');process.exit(1)\"",
     "build": "rm -rf dist && bun scripts/gen-config-schema.ts &&bun run tsc --project ./tsconfig.build.json && bun scripts/copy-assets.ts && bun scripts/fix-esm-extensions.ts",
     "check": "bun run lint && bunx tsc --noEmit && bun run test:unit && bun run test:integration",
     "check:fast": "bun run lint && bunx tsc --noEmit && bun run test:unit",
@@ -58,9 +58,9 @@
     "sweep:tmp": "bun scripts/sweep-test-tmp.ts",
     "test": "bash scripts/test-unit.sh",
     "test:unit": "bash scripts/test-unit.sh",
-    "test:integration": "bun run sweep:tmp && bun test --parallel=1 --timeout=30000 ./tests/integration ./tests/commands ./tests/workflows",
-    "test:unit:shard": "bun run sweep:tmp && bun test --parallel=1 --timeout=30000 ./tests --path-ignore-patterns=tests/integration --shard=${SHARD:?set SHARD=k/N}",
-    "test:integration:shard": "bun run sweep:tmp && bun test --parallel=1 --timeout=30000 ./tests/integration ./tests/commands ./tests/workflows --shard=${SHARD:?set SHARD=k/N}",
+    "test:integration": "bun run sweep:tmp && bun test --isolate --timeout=30000 ./tests/integration ./tests/commands ./tests/workflows",
+    "test:unit:shard": "bun run sweep:tmp && bun test --isolate --timeout=30000 ./tests --path-ignore-patterns=tests/integration --shard=${SHARD:?set SHARD=k/N}",
+    "test:integration:shard": "bun run sweep:tmp && bun test --isolate --timeout=30000 ./tests/integration ./tests/commands ./tests/workflows --shard=${SHARD:?set SHARD=k/N}",
     "test:node-smoke": "bun scripts/node-smoke.ts",
     "test:node-compat": "AKM_NODE_COMPAT_TESTS=1 bun test --timeout=120000 tests/integration/node-compat.test.ts",
     "test:time": "bun scripts/test-timing-report.ts",
@@ -95,7 +95,7 @@
   },
   "engines": {
     "bun": ">=1.0.0",
-    "node": ">=20.0.0"
+    "node": ">=20.12.0"
   },
   "dependencies": {
     "@clack/prompts": "^1.3.0",