akm-cli 0.7.1 → 0.7.2

package/dist/sources/website-ingest.js

@@ -0,0 +1,470 @@
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap } from "../core/common";
+import { ConfigError, UsageError } from "../core/errors";
+import { getRegistryIndexCacheDir } from "../core/paths";
+import { warn } from "../core/warn";
+import { isExpired, sanitizeString } from "./providers/provider-utils";
+/** Refresh website snapshots every 12 hours to balance freshness with scraping load. */
+const CACHE_REFRESH_INTERVAL_MS = 12 * 60 * 60 * 1000;
+/** Allow up to 7 days of stale snapshots when refresh fails so search remains available during outages. */
+const CACHE_STALE_MS = 7 * 24 * 60 * 60 * 1000;
+/** Allow limited breadth-first expansion without letting the crawl queue grow unbounded. */
+const QUEUE_EXPANSION_FACTOR = 5;
+const MAX_PAGES_DEFAULT = 50;
+const MAX_DEPTH_DEFAULT = 3;
+/**
+ * Per-page body cap for website scraping. HTML pages this large are
+ * almost never useful as agent knowledge sources and a runaway server
+ * streaming tens of megabytes would blow memory with no upside.
+ */
+const WEBSITE_PAGE_BYTE_CAP = 5 * 1024 * 1024;
+/**
+ * Wall-clock cap for a full crawl (10 minutes). With per-request timeouts
+ * of 15s and a `maxPages` default of 50, an unresponsive site could
+ * otherwise stall `akm add` for 12.5 minutes with no feedback. Cap the
+ * whole crawl and return what we have when time runs out.
+ */
+const WEBSITE_CRAWL_WALL_CLOCK_MS = 10 * 60 * 1000;
+export function getWebsiteCachePaths(siteUrl) {
+    const key = createHash("sha256").update(normalizeSiteUrl(siteUrl)).digest("hex").slice(0, 16);
+    const rootDir = path.join(getRegistryIndexCacheDir(), `website-${key}`);
+    return {
+        rootDir,
+        stashDir: path.join(rootDir, "stash"),
+        manifestPath: path.join(rootDir, "manifest.json"),
+    };
+}
+export async function ensureWebsiteMirror(config, options) {
+    const rawUrl = config.url ?? "";
+    const normalizedUrl = validateWebsiteUrl(rawUrl);
+    const cachePaths = getWebsiteCachePaths(normalizedUrl);
+    const requireStashDir = options?.requireStashDir === true;
+    const force = options?.force === true;
+    let mtime = 0;
+    try {
+        mtime = fs.statSync(cachePaths.manifestPath).mtimeMs;
+    }
+    catch {
+        /* no cached manifest */
+    }
+    if (!force &&
+        mtime &&
+        !isExpired(mtime, CACHE_REFRESH_INTERVAL_MS) &&
+        (!requireStashDir || hasExtractedSite(cachePaths.stashDir))) {
+        return cachePaths;
+    }
+    try {
+        fs.mkdirSync(cachePaths.rootDir, { recursive: true });
+        await scrapeWebsiteToStash(normalizedUrl, cachePaths.stashDir, {
+            maxPages: coercePositiveInt(config.options?.maxPages, MAX_PAGES_DEFAULT),
+            maxDepth: coercePositiveInt(config.options?.maxDepth, MAX_DEPTH_DEFAULT),
+        });
+        fs.writeFileSync(cachePaths.manifestPath, `${JSON.stringify({ url: normalizedUrl, fetchedAt: new Date().toISOString() }, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
+        return cachePaths;
+    }
+    catch (err) {
+        if (mtime && !isExpired(mtime, CACHE_STALE_MS) && (!requireStashDir || hasExtractedSite(cachePaths.stashDir))) {
+            return cachePaths;
+        }
+        throw err;
+    }
+}
+function hasExtractedSite(stashDir) {
+    try {
+        const knowledgeDir = path.join(stashDir, "knowledge");
+        if (!fs.statSync(stashDir).isDirectory() || !fs.statSync(knowledgeDir).isDirectory())
+            return false;
+        for (const entry of fs.readdirSync(knowledgeDir, { withFileTypes: true })) {
+            if (entry.isFile() && entry.name.endsWith(".md"))
+                return true;
+            if (entry.isDirectory()) {
+                const subEntries = fs.readdirSync(path.join(knowledgeDir, entry.name));
+                if (subEntries.some((e) => e.endsWith(".md")))
+                    return true;
+            }
+        }
+        return false;
+    }
+    catch {
+        return false;
+    }
+}
+async function scrapeWebsiteToStash(startUrl, stashDir, options) {
+    const pages = await crawlWebsite(startUrl, options);
+    if (pages.length === 0) {
+        throw new Error(`No content could be scraped from ${startUrl}`);
+    }
+    fs.rmSync(stashDir, { recursive: true, force: true });
+    const knowledgeDir = path.join(stashDir, "knowledge");
+    fs.mkdirSync(knowledgeDir, { recursive: true });
+    const usedPaths = new Set();
+    for (const page of pages) {
+        const relPath = urlToRelativePath(page.url);
+        const uniquePath = uniqueSlug(relPath, usedPaths);
+        const filePath = path.join(knowledgeDir, `${uniquePath}.md`);
+        const dir = path.dirname(filePath);
+        if (dir !== knowledgeDir)
+            fs.mkdirSync(dir, { recursive: true });
+        const slug = uniquePath.split("/").pop() ?? "index";
+        fs.writeFileSync(filePath, buildMarkdownSnapshot(page, slug), "utf8");
+    }
+}
+export async function fetchWebsiteMarkdownSnapshot(rawUrl) {
+    const normalizedUrl = validateWebsiteInputUrl(rawUrl);
+    const fetched = await fetchWebsitePage(normalizedUrl);
+    if (!fetched) {
+        throw new UsageError(`No content could be fetched from ${normalizedUrl}`);
+    }
+    const preferredName = deriveImportPath(fetched.page.url);
+    const slug = preferredName.split("/").pop() ?? preferredName;
+    return {
+        url: fetched.page.url,
+        title: fetched.page.title,
+        markdown: fetched.page.markdown,
+        preferredName,
+        content: buildMarkdownSnapshot(fetched.page, slug || "website"),
+    };
+}
+async function crawlWebsite(startUrl, options) {
+    const start = new URL(normalizeSiteUrl(startUrl));
+    const allowedOrigin = start.origin;
+    const queue = [{ url: start.toString(), depth: 0 }];
+    const visited = new Set();
+    const pages = [];
+    const deadline = Date.now() + WEBSITE_CRAWL_WALL_CLOCK_MS;
+    while (queue.length > 0 && pages.length < options.maxPages) {
+        if (Date.now() > deadline)
+            break;
+        const next = queue.shift();
+        if (!next)
+            break;
+        const normalized = normalizeCrawlUrl(next.url);
+        if (!normalized || visited.has(normalized))
+            continue;
+        visited.add(normalized);
+        const fetched = await fetchWebsitePage(normalized);
+        if (!fetched)
+            continue;
+        pages.push(fetched.page);
+        if (next.depth >= options.maxDepth)
+            continue;
+        for (const link of fetched.links) {
+            if (queue.length + pages.length >= options.maxPages * QUEUE_EXPANSION_FACTOR)
+                break;
+            if (link.origin !== allowedOrigin)
+                continue;
+            const candidate = normalizeCrawlUrl(link.toString());
+            if (!candidate || visited.has(candidate) || isAssetLikePath(link.pathname))
+                continue;
+            queue.push({ url: candidate, depth: next.depth + 1 });
+        }
+    }
+    if (Date.now() > deadline) {
+        warn("[akm] website crawl stopped at the %ds wall-clock cap with %d/%d pages collected from %s.", WEBSITE_CRAWL_WALL_CLOCK_MS / 1000, pages.length, options.maxPages, startUrl);
+    }
+    return pages;
+}
+async function fetchWebsitePage(pageUrl) {
+    const response = await fetchWithRetry(pageUrl, {
+        headers: {
+            Accept: "text/html, text/markdown, text/plain;q=0.9, application/xhtml+xml;q=0.8",
+            "User-Agent": "akm-cli website provider",
+        },
+    }, { timeout: 15_000, retries: 1 });
+    if (!response.ok) {
+        if (response.status === 404)
+            return null;
+        throw new Error(`Failed to fetch website content (${response.status}) from ${pageUrl}`);
+    }
+    const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
+    let body;
+    try {
+        body = await readBodyWithByteCap(response, WEBSITE_PAGE_BYTE_CAP);
+    }
+    catch (err) {
+        if (err instanceof ResponseTooLargeError)
+            return null;
+        throw err;
+    }
+    const finalUrl = normalizeCrawlUrl(response.url || pageUrl) ?? pageUrl;
+    if (contentType.includes("text/html") || contentType.includes("application/xhtml+xml") || looksLikeMarkup(body)) {
+        const title = extractHtmlTitle(body) || new URL(finalUrl).hostname;
+        return {
+            page: {
+                url: finalUrl,
+                title,
+                markdown: htmlToMarkdown(body, finalUrl),
+            },
+            links: extractSameDocumentLinks(body, finalUrl),
+        };
+    }
+    return {
+        page: {
+            url: finalUrl,
+            title: extractTextTitle(body) || new URL(finalUrl).hostname,
+            markdown: body.trim(),
+        },
+        links: [],
+    };
+}
+function buildMarkdownSnapshot(page, slug) {
+    const title = sanitizeString(page.title, 200) || slug;
+    const description = sanitizeString(`Snapshot of ${page.url}`, 500);
+    const host = sanitizeString(new URL(page.url).hostname, 120);
+    const content = page.markdown.trim() || `Source: ${page.url}`;
+    return [
+        "---",
+        `name: ${JSON.stringify(slug)}`,
+        `description: ${JSON.stringify(description)}`,
+        `sourceUrl: ${JSON.stringify(page.url)}`,
+        `title: ${JSON.stringify(title)}`,
+        "tags:",
+        `  - ${JSON.stringify("website")}`,
+        `  - ${JSON.stringify(host)}`,
+        "---",
+        "",
+        `# ${title}`,
+        "",
+        `Source: ${page.url}`,
+        "",
+        content,
+        "",
+    ].join("\n");
+}
+export function validateWebsiteUrl(rawUrl) {
+    return validateWebsiteUrlWithError(rawUrl, ConfigError);
+}
+export function validateWebsiteInputUrl(rawUrl) {
+    return validateWebsiteUrlWithError(rawUrl, UsageError);
+}
+function validateWebsiteUrlWithError(rawUrl, ErrorType) {
+    if (!rawUrl) {
+        throw new ErrorType("Website provider requires a URL");
+    }
+    let parsed;
+    try {
+        parsed = new URL(rawUrl);
+    }
+    catch {
+        throw new ErrorType(`Website URL is not valid: "${rawUrl}"`);
+    }
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+        throw new ErrorType(`Website URL must use http:// or https://, got "${parsed.protocol}" in "${rawUrl}"`);
+    }
+    if (parsed.username || parsed.password) {
+        throw new ErrorType("Website URL must not contain embedded credentials");
+    }
+    parsed.hash = "";
+    return normalizeSiteUrl(parsed.toString());
+}
+function normalizeSiteUrl(rawUrl) {
+    const parsed = new URL(rawUrl);
+    parsed.hash = "";
+    if (parsed.pathname !== "/" && parsed.pathname.endsWith("/")) {
+        parsed.pathname = parsed.pathname.replace(/\/+$/, "");
+    }
+    return parsed.toString();
+}
+function normalizeCrawlUrl(rawUrl) {
+    try {
+        const parsed = new URL(rawUrl);
+        if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
+            return null;
+        parsed.hash = "";
+        if (parsed.pathname !== "/" && parsed.pathname.endsWith("/")) {
+            parsed.pathname = parsed.pathname.replace(/\/+$/, "");
+        }
+        return parsed.toString();
+    }
+    catch {
+        return null;
+    }
+}
+function urlToRelativePath(rawUrl) {
+    const parsed = new URL(rawUrl);
+    const segments = parsed.pathname
+        .split("/")
+        .filter(Boolean)
+        .map((segment) => slugifySegment(segment))
+        .filter(Boolean);
+    if (parsed.search) {
+        const querySuffix = slugifySegment(parsed.search.slice(1));
+        if (querySuffix && segments.length > 0) {
+            segments[segments.length - 1] = `${segments[segments.length - 1]}_${querySuffix}`;
+        }
+    }
+    return segments.length > 0 ? segments.join("/") : "index";
+}
+function deriveImportPath(rawUrl) {
+    const parsed = new URL(rawUrl);
+    const relativePath = urlToRelativePath(rawUrl);
+    if (relativePath !== "index")
+        return relativePath;
+    const host = slugifySegment(parsed.hostname) || "website";
+    if (!parsed.search)
+        return host;
+    const querySuffix = slugifySegment(parsed.search.slice(1));
+    return querySuffix ? `${host}-${querySuffix}` : host;
+}
+function slugifySegment(value) {
+    return sanitizeString(value, 200)
+        .toLowerCase()
+        .replace(/[^a-z0-9._-]+/g, "-")
+        .replace(/^-+|-+$/g, "");
+}
+function uniqueSlug(base, used) {
+    const seed = base || "website";
+    let candidate = seed;
+    let i = 2;
+    while (used.has(candidate)) {
+        candidate = `${seed}-${i}`;
+        i += 1;
+    }
+    used.add(candidate);
+    return candidate;
+}
+function coercePositiveInt(value, fallback) {
+    if (typeof value === "number" && Number.isInteger(value) && value > 0)
+        return value;
+    if (typeof value === "string") {
+        const parsed = Number.parseInt(value, 10);
+        if (Number.isInteger(parsed) && parsed > 0)
+            return parsed;
+    }
+    return fallback;
+}
+function looksLikeMarkup(body) {
+    return /<html[\s>]|<body[\s>]|<\/[a-z][\w:-]*>/i.test(body);
+}
+function extractHtmlTitle(html) {
+    const title = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1];
+    if (title)
+        return decodeHtmlEntities(stripTags(title)).trim();
+    const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i)?.[1];
+    if (h1)
+        return decodeHtmlEntities(stripTags(h1)).trim();
+    return undefined;
+}
+function extractTextTitle(text) {
+    for (const line of text.split(/\r?\n/)) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        if (trimmed.startsWith("#"))
+            return trimmed.replace(/^#+\s*/, "");
+        return trimmed.slice(0, 120);
+    }
+    return undefined;
+}
+function extractSameDocumentLinks(html, pageUrl) {
+    const links = [];
+    const hrefPattern = /<a\b[^>]*href\s*=\s*(['"])(.*?)\1[^>]*>/gi;
+    for (const match of html.matchAll(hrefPattern)) {
+        const href = match[2]?.trim();
+        if (!href || href.startsWith("#"))
+            continue;
+        try {
+            const resolved = new URL(href, pageUrl);
+            if (!isSafeLinkUrl(resolved))
+                continue;
+            links.push(resolved);
+        }
+        catch {
+            /* ignore malformed links */
+        }
+    }
+    return links;
+}
+function htmlToMarkdown(html, pageUrl) {
+    let text = html;
+    text = stripDangerousBlockTag(text, "script");
+    text = stripDangerousBlockTag(text, "style");
+    text = stripDangerousBlockTag(text, "noscript");
+    text = stripDangerousBlockTag(text, "template");
+    text = text.replace(/<pre\b[^>]*><code\b[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (_match, code) => {
+        const decoded = decodeHtmlEntities(stripTags(code)).trim();
+        return decoded ? `\n\n\`\`\`\n${decoded}\n\`\`\`\n\n` : "\n\n";
+    });
+    text = text.replace(/<code\b[^>]*>([\s\S]*?)<\/code>/gi, (_match, code) => {
+        const decoded = decodeHtmlEntities(stripTags(code)).trim();
+        return decoded ? `\`${decoded}\`` : "";
+    });
+    text = text.replace(/<a\b[^>]*href\s*=\s*(['"])(.*?)\1[^>]*>([\s\S]*?)<\/a>/gi, (_match, _q, href, body) => {
+        const label = decodeHtmlEntities(stripTags(body)).trim();
+        if (!label)
+            return "";
+        try {
+            const resolved = new URL(href, pageUrl);
+            if (!isSafeLinkUrl(resolved))
+                return label;
+            return `[${label}](${resolved})`;
+        }
+        catch {
+            return label;
+        }
+    });
+    text = text.replace(/<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi, (_match, level, body) => {
+        const heading = decodeHtmlEntities(stripTags(body)).trim();
+        return heading ? `\n\n${"#".repeat(Number(level))} ${heading}\n\n` : "\n\n";
+    });
+    text = text.replace(/<li\b[^>]*>([\s\S]*?)<\/li>/gi, (_match, body) => {
+        const item = decodeHtmlEntities(stripTags(body)).trim();
+        return item ? `\n- ${item}` : "";
+    });
+    text = text.replace(/<(p|div|section|article|main|header|footer|blockquote|table|tr)\b[^>]*>/gi, "\n\n");
+    text = text.replace(/<\/(p|div|section|article|main|header|footer|blockquote|table|tr)>/gi, "\n\n");
+    text = text.replace(/<br\s*\/?>/gi, "\n");
+    text = text.replace(/<\/?(ul|ol)\b[^>]*>/gi, "\n");
+    text = decodeHtmlEntities(stripTags(text));
+    text = text
+        .replace(/\r/g, "")
+        .replace(/[ \t]+\n/g, "\n")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+    return text;
+}
+function stripTags(value) {
+    return value.replace(/<[^>]+>/g, " ");
+}
+function decodeHtmlEntities(value) {
+    const namedEntities = {
+        nbsp: " ",
+        amp: "&",
+        lt: "<",
+        gt: ">",
+        quot: '"',
+        apos: "'",
+    };
+    return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity) => {
+        const normalized = String(entity).toLowerCase();
+        if (normalized.startsWith("#x")) {
+            return safeCodePointToString(Number.parseInt(normalized.slice(2), 16)) ?? match;
+        }
+        if (normalized.startsWith("#")) {
+            return safeCodePointToString(Number.parseInt(normalized.slice(1), 10)) ?? match;
+        }
+        return namedEntities[normalized] ?? match;
+    });
+}
+function isAssetLikePath(pathname) {
+    return /\.(css|js|json|png|jpe?g|gif|svg|ico|webp|pdf|zip|tar|gz|mp4|mp3|woff2?)$/i.test(pathname);
+}
+function isSafeLinkUrl(url) {
+    return url.protocol === "http:" || url.protocol === "https:";
+}
+function stripDangerousBlockTag(value, tagName) {
+    const pattern = new RegExp(`<${tagName}\\b[^>]*>[\\s\\S]*?<\\/${tagName}\\s*>`, "gi");
+    return value.replace(pattern, "");
+}
+function safeCodePointToString(value) {
+    if (!Number.isFinite(value) || value < 0 || value > 0x10ffff)
+        return undefined;
+    try {
+        return String.fromCodePoint(value);
+    }
+    catch {
+        return undefined;
+    }
+}

package/docs/README.md

@@ -21,6 +21,13 @@
 - [Configuration](configuration.md) -- Providers, settings, and Ollama setup
 - [Filesystem](technical/filesystem.md) -- Directory layout and `.stash.json` schema
 
+## Official Ecosystem Repositories
+
+- [itlackey/akm-stash](https://github.com/itlackey/akm-stash) -- the official onboarding stash with ready-made assets you can install with `akm add`
+- [itlackey/akm-registry](https://github.com/itlackey/akm-registry) -- the official registry index that powers built-in discovery
+- [itlackey/akm-plugins](https://github.com/itlackey/akm-plugins) -- optional integrations for tools like OpenCode
+- [itlackey/akm-bench](https://github.com/itlackey/akm-bench) -- the standalone benchmark and evaluation repo for akm
+
 ## Internals
 
 - [Search](technical/search.md) -- Hybrid search architecture and scoring

package/docs/migration/release-notes/0.7.0.md

@@ -150,6 +150,20 @@ This is the surface 0.7.0 commits to maintain through 0.8.x / 0.9.x.
 Renaming or removing any of these commands at 1.0 GA would be a major
 version bump.
 
+## Post-0.7.0 additive surfaces
+
+These landed after the 0.7.0 cut without changing the existing command grammar:
+
+- `akm import <url>` now fetches one HTTP/HTTPS URL, converts it to markdown,
+  and writes it into `knowledge/` using a URL-path-derived default name.
+- `akm wiki stash <name> <url>` now fetches one HTTP/HTTPS URL, converts it to
+  markdown, and writes it into `wikis/<name>/raw/`.
+- Both flows are one-shot ingest only: they do not register a persistent
+  website source and they do not crawl linked pages.
+- Website source sync (`akm add <url> --provider website`) and one-shot URL
+  ingest now share the same `src/sources/website-ingest.ts` module for URL
+  validation, fetch/convert, and mirror generation behavior.
+
 ## Pre-prod hardening (PR #275)
 
 PR #275 batched five issues plus the bench tmp follow-up that operators

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "akm-cli",
-  "version": "0.7.1",
+  "version": "0.7.2",
   "type": "module",
   "description": "akm (Agent Kit Manager) — A package manager for AI agent skills, commands, tools, and knowledge. Works with Claude Code, OpenCode, Cursor, and any AI coding assistant.",
   "keywords": [
@@ -46,6 +46,9 @@
     "check": "bun run lint && bunx tsc --noEmit && bun test ./tests",
     "check:changed": "bun test tests/output-baseline.test.ts tests/e2e.test.ts tests/stash-search.test.ts && bun run lint && bunx tsc --noEmit",
     "test": "bun test ./tests",
+    "lint:devto-posts": "bun scripts/lint-devto-posts.ts",
+    "lint:devto-posts:fix": "bun scripts/lint-devto-posts.ts --fix",
+    "publish:devto": "npx -y @sinedied/devto-cli push \"docs/posts/**/*.md\" --token \"$DEVTO_TOKEN\" --repo \"$GITHUB_REPOSITORY\" --branch \"${GITHUB_REF_NAME:-main}\" --reconcile",
     "release:check": "./tests/release-check.sh",
     "lint": "bunx biome check src/ tests/",
     "lint:fix": "bunx biome check --write src/ tests/",