akm-cli 0.7.0 → 0.7.2

package/dist/{src/sources/providers/website.js → sources/website-ingest.js}

@@ -1,12 +1,11 @@
 import { createHash } from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
-import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap } from "../../core/common";
-import { ConfigError, UsageError } from "../../core/errors";
-import { getRegistryIndexCacheDir } from "../../core/paths";
-import { warn } from "../../core/warn";
-import { registerSourceProvider } from "../provider-factory";
-import { isExpired, sanitizeString } from "./provider-utils";
+import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap } from "../core/common";
+import { ConfigError, UsageError } from "../core/errors";
+import { getRegistryIndexCacheDir } from "../core/paths";
+import { warn } from "../core/warn";
+import { isExpired, sanitizeString } from "./providers/provider-utils";
 /** Refresh website snapshots every 12 hours to balance freshness with scraping load. */
 const CACHE_REFRESH_INTERVAL_MS = 12 * 60 * 60 * 1000;
 /** Allow up to 7 days of stale snapshots when refresh fails so search remains available during outages. */
@@ -28,36 +27,7 @@ const WEBSITE_PAGE_BYTE_CAP = 5 * 1024 * 1024;
  * whole crawl and return what we have when time runs out.
  */
 const WEBSITE_CRAWL_WALL_CLOCK_MS = 10 * 60 * 1000;
-/**
- * Website source provider — scrapes pages into a local mirror so the FTS5
- * indexer can walk them. Implements the v1 {@link SourceProvider} interface
- * (spec §2.1): `{ name, kind, init, path, sync }`.
- *
- * Reading is the indexer's job — this class doesn't implement `search` or
- * `show`.
- */
-class WebsiteSourceProvider {
-    kind = "website";
-    name;
-    #config;
-    #url;
-    constructor(config) {
-        this.#config = config;
-        this.name = config.name ?? "website";
-        this.#url = validateWebsiteUrl(config.url ?? "");
-    }
-    async init(_ctx) {
-        // URL validation already happens in the constructor; nothing else to do.
-    }
-    path() {
-        return getCachePaths(this.#url).stashDir;
-    }
-    async sync() {
-        await ensureWebsiteMirror(this.#config, { requireStashDir: true });
-    }
-}
-registerSourceProvider("website", (config) => new WebsiteSourceProvider(config));
-function getCachePaths(siteUrl) {
+export function getWebsiteCachePaths(siteUrl) {
     const key = createHash("sha256").update(normalizeSiteUrl(siteUrl)).digest("hex").slice(0, 16);
     const rootDir = path.join(getRegistryIndexCacheDir(), `website-${key}`);
     return {
@@ -66,10 +36,10 @@ function getCachePaths(siteUrl) {
         manifestPath: path.join(rootDir, "manifest.json"),
     };
 }
-async function ensureWebsiteMirror(config, options) {
+export async function ensureWebsiteMirror(config, options) {
     const rawUrl = config.url ?? "";
     const normalizedUrl = validateWebsiteUrl(rawUrl);
-    const cachePaths = getCachePaths(normalizedUrl);
+    const cachePaths = getWebsiteCachePaths(normalizedUrl);
     const requireStashDir = options?.requireStashDir === true;
     const force = options?.force === true;
     let mtime = 0;
@@ -106,7 +76,6 @@ function hasExtractedSite(stashDir) {
         const knowledgeDir = path.join(stashDir, "knowledge");
         if (!fs.statSync(stashDir).isDirectory() || !fs.statSync(knowledgeDir).isDirectory())
             return false;
-        // Check top-level and one level of subdirectories for .md files
         for (const entry of fs.readdirSync(knowledgeDir, { withFileTypes: true })) {
             if (entry.isFile() && entry.name.endsWith(".md"))
                 return true;
@@ -142,6 +111,22 @@ async function scrapeWebsiteToStash(startUrl, stashDir, options) {
         fs.writeFileSync(filePath, buildMarkdownSnapshot(page, slug), "utf8");
     }
 }
+export async function fetchWebsiteMarkdownSnapshot(rawUrl) {
+    const normalizedUrl = validateWebsiteInputUrl(rawUrl);
+    const fetched = await fetchWebsitePage(normalizedUrl);
+    if (!fetched) {
+        throw new UsageError(`No content could be fetched from ${normalizedUrl}`);
+    }
+    const preferredName = deriveImportPath(fetched.page.url);
+    const slug = preferredName.split("/").pop() ?? preferredName;
+    return {
+        url: fetched.page.url,
+        title: fetched.page.title,
+        markdown: fetched.page.markdown,
+        preferredName,
+        content: buildMarkdownSnapshot(fetched.page, slug || "website"),
+    };
+}
 async function crawlWebsite(startUrl, options) {
     const start = new URL(normalizeSiteUrl(startUrl));
     const allowedOrigin = start.origin;
@@ -199,10 +184,8 @@ async function fetchWebsitePage(pageUrl) {
         body = await readBodyWithByteCap(response, WEBSITE_PAGE_BYTE_CAP);
     }
     catch (err) {
-        if (err instanceof ResponseTooLargeError) {
-            // Skip oversized pages rather than aborting the whole crawl.
+        if (err instanceof ResponseTooLargeError)
             return null;
-        }
         throw err;
     }
     const finalUrl = normalizeCrawlUrl(response.url || pageUrl) ?? pageUrl;
@@ -250,10 +233,10 @@ function buildMarkdownSnapshot(page, slug) {
         "",
     ].join("\n");
 }
-function validateWebsiteUrl(rawUrl) {
+export function validateWebsiteUrl(rawUrl) {
     return validateWebsiteUrlWithError(rawUrl, ConfigError);
 }
-function validateWebsiteInputUrl(rawUrl) {
+export function validateWebsiteInputUrl(rawUrl) {
     return validateWebsiteUrlWithError(rawUrl, UsageError);
 }
 function validateWebsiteUrlWithError(rawUrl, ErrorType) {
@@ -299,10 +282,6 @@ function normalizeCrawlUrl(rawUrl) {
         return null;
     }
 }
-/** Convert a page URL into a relative file path preserving the URL hierarchy.
- *  e.g. https://example.com/docs/guide → docs/guide
- *       https://example.com/           → index
- */
 function urlToRelativePath(rawUrl) {
     const parsed = new URL(rawUrl);
     const segments = parsed.pathname
@@ -318,6 +297,17 @@ function urlToRelativePath(rawUrl) {
     }
     return segments.length > 0 ? segments.join("/") : "index";
 }
+function deriveImportPath(rawUrl) {
+    const parsed = new URL(rawUrl);
+    const relativePath = urlToRelativePath(rawUrl);
+    if (relativePath !== "index")
+        return relativePath;
+    const host = slugifySegment(parsed.hostname) || "website";
+    if (!parsed.search)
+        return host;
+    const querySuffix = slugifySegment(parsed.search.slice(1));
+    return querySuffix ? `${host}-${querySuffix}` : host;
+}
 function slugifySegment(value) {
     return sanitizeString(value, 200)
         .toLowerCase()
@@ -459,8 +449,6 @@ function decodeHtmlEntities(value) {
     });
 }
 function isAssetLikePath(pathname) {
-    // Keep this list intentionally conservative so docs paths are still crawled
-    // unless they clearly point at static assets/binaries.
     return /\.(css|js|json|png|jpe?g|gif|svg|ico|webp|pdf|zip|tar|gz|mp4|mp3|woff2?)$/i.test(pathname);
 }
 function isSafeLinkUrl(url) {
@@ -480,4 +468,3 @@ function safeCodePointToString(value) {
         return undefined;
     }
 }
-export { ensureWebsiteMirror, getCachePaths, validateWebsiteInputUrl, validateWebsiteUrl, WebsiteSourceProvider };

package/docs/README.md

@@ -21,6 +21,13 @@
 - [Configuration](configuration.md) -- Providers, settings, and Ollama setup
 - [Filesystem](technical/filesystem.md) -- Directory layout and `.stash.json` schema
 
+## Official Ecosystem Repositories
+
+- [itlackey/akm-stash](https://github.com/itlackey/akm-stash) -- the official onboarding stash with ready-made assets you can install with `akm add`
+- [itlackey/akm-registry](https://github.com/itlackey/akm-registry) -- the official registry index that powers built-in discovery
+- [itlackey/akm-plugins](https://github.com/itlackey/akm-plugins) -- optional integrations for tools like OpenCode
+- [itlackey/akm-bench](https://github.com/itlackey/akm-bench) -- the standalone benchmark and evaluation repo for akm
+
 ## Internals
 
 - [Search](technical/search.md) -- Hybrid search architecture and scoring

package/docs/migration/release-notes/0.7.0.md

@@ -150,6 +150,20 @@ This is the surface 0.7.0 commits to maintain through 0.8.x / 0.9.x.
 Renaming or removing any of these commands at 1.0 GA would be a major
 version bump.
 
+## Post-0.7.0 additive surfaces
+
+These landed after the 0.7.0 cut without changing the existing command grammar:
+
+- `akm import <url>` now fetches one HTTP/HTTPS URL, converts it to markdown,
+  and writes it into `knowledge/` using a URL-path-derived default name.
+- `akm wiki stash <name> <url>` now fetches one HTTP/HTTPS URL, converts it to
+  markdown, and writes it into `wikis/<name>/raw/`.
+- Both flows are one-shot ingest only: they do not register a persistent
+  website source and they do not crawl linked pages.
+- Website source sync (`akm add <url> --provider website`) and one-shot URL
+  ingest now share the same `src/sources/website-ingest.ts` module for URL
+  validation, fetch/convert, and mirror generation behavior.
+
 ## Pre-prod hardening (PR #275)
 
 PR #275 batched five issues plus the bench tmp follow-up that operators

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "akm-cli",
-  "version": "0.7.0",
+  "version": "0.7.2",
   "type": "module",
   "description": "akm (Agent Kit Manager) — A package manager for AI agent skills, commands, tools, and knowledge. Works with Claude Code, OpenCode, Cursor, and any AI coding assistant.",
   "keywords": [
@@ -42,10 +42,13 @@
     "akm": "dist/cli.js"
   },
   "scripts": {
-    "build": "rm -rf dist && bun run tsc --project ./tsconfig.json --outDir dist",
+    "build": "rm -rf dist && bun run tsc --project ./tsconfig.build.json",
     "check": "bun run lint && bunx tsc --noEmit && bun test ./tests",
     "check:changed": "bun test tests/output-baseline.test.ts tests/e2e.test.ts tests/stash-search.test.ts && bun run lint && bunx tsc --noEmit",
     "test": "bun test ./tests",
+    "lint:devto-posts": "bun scripts/lint-devto-posts.ts",
+    "lint:devto-posts:fix": "bun scripts/lint-devto-posts.ts --fix",
+    "publish:devto": "npx -y @sinedied/devto-cli push \"docs/posts/**/*.md\" --token \"$DEVTO_TOKEN\" --repo \"$GITHUB_REPOSITORY\" --branch \"${GITHUB_REF_NAME:-main}\" --reconcile",
     "release:check": "./tests/release-check.sh",
     "lint": "bunx biome check src/ tests/",
     "lint:fix": "bunx biome check --write src/ tests/",
@@ -57,9 +60,9 @@
     "access": "public"
   },
   "devDependencies": {
-    "@biomejs/biome": "^2.4.6",
-    "@types/node": "^22.0.0",
-    "bun-types": "^1.3.10",
+    "@biomejs/biome": "^2.4.14",
+    "@types/node": "^22.19.17",
+    "bun-types": "^1.3.13",
     "typescript": "^5.9.3"
   },
   "optionalDependencies": {
@@ -70,9 +73,9 @@
     "bun": ">=1.0.0"
   },
   "dependencies": {
-    "@clack/prompts": "^1.1.0",
-    "citty": "^0.2.1",
+    "@clack/prompts": "^1.3.0",
+    "citty": "^0.2.2",
     "dotenv": "^17.4.2",
-    "yaml": "^2.8.2"
+    "yaml": "^2.8.4"
   }
 }

package/dist/src/llm/memory-infer.js

@@ -1,86 +0,0 @@
-/**
- * LLM helper for the `akm index` memory-inference pass (#201).
- *
- * Splits a single memory body into a list of atomic facts. The pass itself
- * (in `src/indexer/memory-inference.ts`) is responsible for deciding which
- * memories are pending, persisting the resulting atomic memories with the
- * correct frontmatter (`inferred: true`, `source: <parent-ref>`), and
- * marking the parent as processed for idempotency.
- *
- * This module is intentionally tiny and stateless so tests can stub it via
- * `mock.module("../src/llm/memory-infer", ...)` without hitting a network.
- *
- * Locked v1 contract (#208): the LLM connection always comes from the
- * shared `akm.llm` block — never from a per-pass override. Callers obtain
- * the connection via `resolveIndexPassLLM("memory", config)` and pass it
- * straight through.
- */
-import { toErrorMessage } from "../core/common";
-import { warn } from "../core/warn";
-import { chatCompletion, parseJsonResponse } from "./client";
-/** Hard cap on body chars sent to the model — pragmatic and matches `runLlmEnrich`. */
-const MAX_BODY_CHARS = 4000;
-/** Hard cap on the number of atomic facts returned per memory. */
-const MAX_FACTS_PER_MEMORY = 16;
-/** Hard timeout for the LLM call. The index run must not hang on a misbehaving endpoint. */
-const LLM_TIMEOUT_MS = 30_000;
-const SYSTEM_PROMPT = "You split a developer memory into atomic, self-contained facts. " +
-    "Return only valid JSON. No prose, no markdown fences.";
-const USER_PROMPT_PREFIX = `Split the memory below into a JSON array of short, self-contained atomic facts.
-
-Rules:
-- Output ONLY a JSON object: {"facts": ["fact one", "fact two", ...]}.
-- Each fact is a single complete sentence, decontextualized so it stands alone.
-- Drop pleasantries, meta-commentary, and timestamps.
-- Preserve technical specifics (names, versions, identifiers) verbatim.
-- If the memory is already a single atomic fact, return it as the only entry.
-- Limit to at most ${MAX_FACTS_PER_MEMORY} facts.
-
-Memory:
-`;
-/**
- * Split a single memory body into atomic facts via the configured LLM.
- *
- * Returns `[]` on any failure (timeout, invalid JSON, empty response). Errors
- * are logged via `warn()` but never thrown — a failed split for one memory
- * must not abort the rest of the index pass.
- */
-export async function splitMemoryIntoAtomicFacts(llmConfig, body) {
-    const trimmedBody = body.trim();
-    if (!trimmedBody)
-        return [];
-    const userPrompt = `${USER_PROMPT_PREFIX}${trimmedBody.slice(0, MAX_BODY_CHARS)}`;
-    let timeoutHandle;
-    try {
-        const raw = await Promise.race([
-            chatCompletion(llmConfig, [
-                { role: "system", content: SYSTEM_PROMPT },
-                { role: "user", content: userPrompt },
-            ], { maxTokens: 768, temperature: 0.1 }),
-            new Promise((_, reject) => {
-                timeoutHandle = setTimeout(() => reject(new Error("memory inference timed out")), LLM_TIMEOUT_MS);
-            }),
-        ]);
-        if (!raw)
-            return [];
-        const parsed = parseJsonResponse(raw);
-        if (!parsed || !Array.isArray(parsed.facts)) {
-            warn("memory inference: invalid JSON response from LLM; skipping memory.");
-            return [];
-        }
-        const facts = parsed.facts
-            .filter((f) => typeof f === "string")
-            .map((f) => f.trim())
-            .filter((f) => f.length > 0)
-            .slice(0, MAX_FACTS_PER_MEMORY);
-        return facts;
-    }
-    catch (err) {
-        warn(`memory inference failed: ${toErrorMessage(err)}`);
-        return [];
-    }
-    finally {
-        if (timeoutHandle !== undefined)
-            clearTimeout(timeoutHandle);
-    }
-}

package/dist/tests/add-website-source.test.js

@@ -1,119 +0,0 @@
-import { afterAll, afterEach, describe, expect, test } from "bun:test";
-import { spawn } from "node:child_process";
-import fs from "node:fs";
-import os from "node:os";
-import path from "node:path";
-const CLI = path.join(__dirname, "..", "src", "cli.ts");
-const tempDirs = [];
-const servers = [];
-const CLI_TIMEOUT_MS = 30_000;
-const TEST_TIMEOUT_MS = 60_000;
-function makeTempDir(prefix) {
-    const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
-    tempDirs.push(dir);
-    return dir;
-}
-function createWorkingStash() {
-    const dir = makeTempDir("akm-add-website-stash-");
-    for (const sub of ["skills", "commands", "agents", "knowledge", "scripts"]) {
-        fs.mkdirSync(path.join(dir, sub), { recursive: true });
-    }
-    return dir;
-}
-function serveWebsite() {
-    const server = Bun.serve({
-        port: 0,
-        fetch(request) {
-            const url = new URL(request.url);
-            if (url.pathname === "/") {
-                return new Response("<html><head><title>Example Docs</title></head><body><h1>Example Docs</h1><p>Welcome to the docs.</p><a href='/getting-started'>Getting started</a></body></html>", {
-                    headers: { "Content-Type": "text/html; charset=utf-8" },
-                });
-            }
-            if (url.pathname === "/getting-started") {
-                return new Response("<html><body><h1>Getting started</h1><p>Run setup first.</p></body></html>", {
-                    headers: { "Content-Type": "text/html; charset=utf-8" },
-                });
-            }
-            return new Response("not found", { status: 404 });
-        },
-    });
-    servers.push(server);
-    return `http://127.0.0.1:${server.port}`;
-}
-afterEach(() => {
-    for (const server of servers.splice(0)) {
-        server.stop(true);
-    }
-    for (const dir of tempDirs.splice(0)) {
-        fs.rmSync(dir, { recursive: true, force: true });
-    }
-});
-afterAll(() => {
-    for (const server of servers.splice(0)) {
-        server.stop(true);
-    }
-});
-describe("akm add website", () => {
-    test("adds a website stash source, caches markdown, and indexes it", async () => {
-        const stashDir = createWorkingStash();
-        const xdgCache = makeTempDir("akm-add-website-cache-");
-        const xdgConfig = makeTempDir("akm-add-website-config-");
-        const websiteUrl = serveWebsite();
-        const configDir = path.join(xdgConfig, "akm");
-        fs.mkdirSync(configDir, { recursive: true });
-        fs.writeFileSync(path.join(configDir, "config.json"), `${JSON.stringify({ semanticSearchMode: "off" }, null, 2)}\n`);
-        const child = spawn("bun", [CLI, "add", websiteUrl, "--name", "docs-site", "--format=json"], {
-            stdio: ["ignore", "pipe", "pipe"],
-            env: {
-                ...process.env,
-                AKM_STASH_DIR: stashDir,
-                XDG_CACHE_HOME: xdgCache,
-                XDG_CONFIG_HOME: xdgConfig,
-            },
-        });
-        let stdout = "";
-        let stderr = "";
-        child.stdout.on("data", (chunk) => {
-            stdout += String(chunk);
-        });
-        child.stderr.on("data", (chunk) => {
-            stderr += String(chunk);
-        });
-        const exitCode = await new Promise((resolve, reject) => {
-            const timer = setTimeout(() => {
-                child.kill("SIGKILL");
-                reject(new Error("CLI website add timed out"));
-            }, CLI_TIMEOUT_MS);
-            child.on("error", (err) => {
-                clearTimeout(timer);
-                reject(err);
-            });
-            child.on("close", (code) => {
-                clearTimeout(timer);
-                resolve(code ?? 1);
-            });
-        });
-        expect(exitCode).toBe(0);
-        expect(stderr.trim()).toBe("");
-        const parsed = JSON.parse(stdout.trim());
-        const normalizedWebsiteUrl = `${websiteUrl}/`;
-        expect(parsed.sourceAdded).toBeDefined();
-        expect(parsed.sourceAdded?.type).toBe("website");
-        expect(parsed.sourceAdded?.url).toBe(normalizedWebsiteUrl);
-        expect(parsed.sourceAdded?.name).toBe("docs-site");
-        expect(parsed.index?.totalEntries).toBeGreaterThanOrEqual(2);
-        const configPath = path.join(xdgConfig, "akm", "config.json");
-        const config = JSON.parse(fs.readFileSync(configPath, "utf8"));
-        expect(config.sources).toContainEqual({
-            type: "website",
-            url: normalizedWebsiteUrl,
-            name: "docs-site",
-        });
-        expect(parsed.sourceAdded?.stashRoot).toBeDefined();
-        const knowledgeFiles = fs.readdirSync(path.join(parsed.sourceAdded?.stashRoot, "knowledge")).sort();
-        expect(knowledgeFiles).toEqual(["getting-started.md", "index.md"]);
-        const homeDoc = fs.readFileSync(path.join(parsed.sourceAdded?.stashRoot, "knowledge", "index.md"), "utf8");
-        expect(homeDoc).toContain("Example Docs");
-    }, { timeout: TEST_TIMEOUT_MS });
-});

package/dist/tests/agent/agent-config-loader.test.js

@@ -1,70 +0,0 @@
-/**
- * Integration test: the AkmConfig loader propagates the `agent` block
- * through `loadConfig()` (and through the on-disk JSONC parser, exercising
- * the `pickKnownKeys` path).
- *
- * The acceptance criterion "config schema accepts an optional agent block"
- * lives at the loader boundary, not just the parser; this test pins the
- * end-to-end shape.
- */
-import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-import fs from "node:fs";
-import os from "node:os";
-import path from "node:path";
-let tmpHome;
-let originalHome;
-let originalXdg;
-beforeEach(() => {
-    tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "akm-agent-cfg-"));
-    originalHome = process.env.HOME;
-    originalXdg = process.env.XDG_CONFIG_HOME;
-    process.env.HOME = tmpHome;
-    process.env.XDG_CONFIG_HOME = path.join(tmpHome, ".config");
-});
-afterEach(() => {
-    if (originalHome === undefined)
-        delete process.env.HOME;
-    else
-        process.env.HOME = originalHome;
-    if (originalXdg === undefined)
-        delete process.env.XDG_CONFIG_HOME;
-    else
-        process.env.XDG_CONFIG_HOME = originalXdg;
-    fs.rmSync(tmpHome, { recursive: true, force: true });
-});
-describe("AkmConfig loader — agent block", () => {
-    test("loads agent.default + agent.profiles from disk", async () => {
-        const { getConfigPath, loadUserConfig, resetConfigCache } = await import("../../src/core/config");
-        const cfgPath = getConfigPath();
-        fs.mkdirSync(path.dirname(cfgPath), { recursive: true });
-        fs.writeFileSync(cfgPath, JSON.stringify({
-            semanticSearchMode: "auto",
-            agent: {
-                default: "claude",
-                timeoutMs: 45000,
-                profiles: {
-                    claude: { args: ["--print"] },
-                    rover: { bin: "rover-cli", parseOutput: "json" },
-                },
-                // Unknown key — must not throw at load.
-                mystery: 1,
-            },
-        }, null, 2));
-        resetConfigCache();
-        const cfg = loadUserConfig();
-        expect(cfg.agent?.default).toBe("claude");
-        expect(cfg.agent?.timeoutMs).toBe(45000);
-        expect(cfg.agent?.profiles?.claude?.args).toEqual(["--print"]);
-        expect(cfg.agent?.profiles?.rover?.bin).toBe("rover-cli");
-        expect(cfg.agent?.profiles?.rover?.parseOutput).toBe("json");
-    });
-    test("agent block absent → cfg.agent is undefined → requireAgentProfile throws", async () => {
-        const { loadUserConfig, resetConfigCache } = await import("../../src/core/config");
-        const { requireAgentProfile } = await import("../../src/integrations/agent/config");
-        const { ConfigError } = await import("../../src/core/errors");
-        resetConfigCache();
-        const cfg = loadUserConfig();
-        expect(cfg.agent).toBeUndefined();
-        expect(() => requireAgentProfile(cfg.agent)).toThrow(ConfigError);
-    });
-});