akm-cli 0.7.0 → 0.7.2

package/dist/{src/llm → llm}/client.js

@@ -51,8 +51,9 @@ export async function chatCompletion(config, messages, options) {
             messages,
             temperature: options?.temperature ?? config.temperature ?? 0.3,
             max_tokens: options?.maxTokens ?? config.maxTokens ?? 512,
+            ...config.extraParams,
         }),
-    });
+    }, 30_000, options?.signal);
     if (!response.ok) {
         const rawBody = await response.text().catch(() => "");
         const safeBody = redactErrorBody(rawBody);
@@ -65,6 +66,7 @@ export async function chatCompletion(config, messages, options) {
 export function stripJsonFences(raw) {
     return raw
         .trim()
+        .replace(/<think>[\s\S]*?<\/think>/gi, "")
         .replace(/^```(?:json)?\s*\n?/i, "")
         .replace(/\n?```\s*$/i, "")
         .trim();
@@ -78,6 +80,64 @@ export function parseJsonResponse(raw) {
         return undefined;
     }
 }
+/**
+ * Best-effort recovery for providers that wrap JSON in extra prose or fenced
+ * blocks. Extracts the first balanced top-level object/array and parses it.
+ */
+export function parseEmbeddedJsonResponse(raw) {
+    const direct = parseJsonResponse(raw);
+    if (direct !== undefined)
+        return direct;
+    const text = stripJsonFences(raw);
+    let arrayFallback;
+    for (let start = 0; start < text.length; start++) {
+        const opener = text[start];
+        if (opener !== "{" && opener !== "[")
+            continue;
+        const closer = opener === "{" ? "}" : "]";
+        let depth = 0;
+        let inString = false;
+        let escaped = false;
+        for (let i = start; i < text.length; i++) {
+            const ch = text[i];
+            if (inString) {
+                if (escaped) {
+                    escaped = false;
+                }
+                else if (ch === "\\") {
+                    escaped = true;
+                }
+                else if (ch === '"') {
+                    inString = false;
+                }
+                continue;
+            }
+            if (ch === '"') {
+                inString = true;
+                continue;
+            }
+            if (ch === opener)
+                depth += 1;
+            if (ch === closer) {
+                depth -= 1;
+                if (depth === 0) {
+                    try {
+                        const parsed = JSON.parse(text.slice(start, i + 1));
+                        if (!Array.isArray(parsed)) {
+                            return parsed;
+                        }
+                        arrayFallback ??= parsed;
+                        break;
+                    }
+                    catch {
+                        break;
+                    }
+                }
+            }
+        }
+    }
+    return arrayFallback;
+}
 // ── Availability check ──────────────────────────────────────────────────────
 /**
  * Check if the LLM endpoint is reachable.

package/dist/{src/llm → llm}/embedder.js

@@ -47,14 +47,14 @@ export function resetLocalEmbedder() {
  * Results are cached in an LRU cache (max ~100 entries) keyed by query text
  * and embedding config. Repeated identical queries return the cached vector.
  */
-export async function embed(text, embeddingConfig) {
+export async function embed(text, embeddingConfig, signal) {
     const key = embedCacheKey(text, embeddingConfig);
     const cached = getCachedEmbedding(key);
     if (cached)
         return cached;
     const result = embeddingConfig && hasRemoteEndpoint(embeddingConfig)
-        ? await new RemoteEmbedder(embeddingConfig).embed(text)
-        : await localEmbedder.embedWithModel(text, embeddingConfig?.localModel);
+        ? await new RemoteEmbedder(embeddingConfig).embed(text, signal)
+        : await localEmbedder.embed(text, signal);
     setCachedEmbedding(key, result);
     return result;
 }
@@ -63,16 +63,19 @@ export async function embed(text, embeddingConfig) {
  * Uses the OpenAI-compatible batch API for remote endpoints (batches of 100).
  * Falls back to sequential embedding for the local transformer pipeline.
  */
-export async function embedBatch(texts, embeddingConfig) {
+export async function embedBatch(texts, embeddingConfig, signal) {
     if (texts.length === 0)
         return [];
     if (embeddingConfig && hasRemoteEndpoint(embeddingConfig)) {
-        return new RemoteEmbedder(embeddingConfig).embedBatch(texts);
+        return new RemoteEmbedder(embeddingConfig).embedBatch(texts, signal);
     }
     // Local transformer: process sequentially (pipeline handles one at a time)
     const localModel = embeddingConfig?.localModel;
     const results = [];
     for (const text of texts) {
+        if (signal?.aborted) {
+            throw signal.reason instanceof Error ? signal.reason : new Error("embedding interrupted");
+        }
         results.push(await localEmbedder.embedWithModel(text, localModel));
     }
     return results;

package/dist/{src/llm → llm}/embedders/local.js

@@ -42,14 +42,20 @@ export class LocalEmbedder {
         this.pipelinePromise = undefined;
         this.pipelineModelName = undefined;
     }
-    async embed(text) {
+    async embed(text, signal) {
+        if (signal?.aborted) {
+            throw signal.reason instanceof Error ? signal.reason : new Error("embedding interrupted");
+        }
         return this.embedWithModel(text, this.defaultModel);
     }
-    async embedBatch(texts) {
+    async embedBatch(texts, signal) {
         if (texts.length === 0)
             return [];
         const results = [];
         for (const text of texts) {
+            if (signal?.aborted) {
+                throw signal.reason instanceof Error ? signal.reason : new Error("embedding interrupted");
+            }
             results.push(await this.embedWithModel(text, this.defaultModel));
         }
         return results;

package/dist/{src/llm → llm}/embedders/remote.js

@@ -15,7 +15,7 @@ export class RemoteEmbedder {
     constructor(config) {
         this.config = config;
     }
-    async embed(text) {
+    async embed(text, signal) {
         const headers = this.buildHeaders();
         const body = {
             input: text,
@@ -32,6 +32,7 @@ export class RemoteEmbedder {
             method: "POST",
             headers,
             body: JSON.stringify(body),
+            signal,
         });
         if (!response.ok) {
             const errBody = await response.text().catch(() => "");
@@ -43,7 +44,7 @@ export class RemoteEmbedder {
         }
         return l2Normalize(json.data[0].embedding);
     }
-    async embedBatch(texts) {
+    async embedBatch(texts, signal) {
         if (texts.length === 0)
             return [];
         const results = [];
@@ -66,6 +67,7 @@ export class RemoteEmbedder {
                 method: "POST",
                 headers,
                 body: JSON.stringify(body),
+                signal,
             });
             if (!response.ok) {
                 const respBody = await response.text().catch(() => "");

package/dist/{src/llm → llm}/graph-extract.js

@@ -19,7 +19,7 @@
  */
 import { toErrorMessage } from "../core/common";
 import { warn } from "../core/warn";
-import { chatCompletion, parseJsonResponse } from "./client";
+import { chatCompletion, parseEmbeddedJsonResponse } from "./client";
 /** Hard cap on body chars sent to the model. */
 const MAX_BODY_CHARS = 4000;
 /** Hard cap on entities returned per asset — guards against runaway LLM output. */
@@ -49,7 +49,7 @@ Asset body:
  * JSON, empty response). Errors are logged via `warn()` but never thrown — a
  * failed extraction for one asset must not abort the rest of the index pass.
  */
-export async function extractGraphFromBody(llmConfig, body) {
+export async function extractGraphFromBody(llmConfig, body, signal) {
     const empty = { entities: [], relations: [] };
     const trimmedBody = body.trim();
     if (!trimmedBody)
@@ -61,14 +61,14 @@ export async function extractGraphFromBody(llmConfig, body) {
             chatCompletion(llmConfig, [
                 { role: "system", content: SYSTEM_PROMPT },
                 { role: "user", content: userPrompt },
-            ], { maxTokens: 1024, temperature: 0.1 }),
+            ], { maxTokens: 1024, temperature: 0.1, signal }),
             new Promise((_, reject) => {
                 timeoutHandle = setTimeout(() => reject(new Error("graph extraction timed out")), LLM_TIMEOUT_MS);
             }),
         ]);
         if (!raw)
             return empty;
-        const parsed = parseJsonResponse(raw);
+        const parsed = parseEmbeddedJsonResponse(raw);
         if (!parsed) {
             warn("graph extraction: invalid JSON response from LLM; skipping asset.");
             return empty;

package/dist/llm/memory-infer.js

@@ -0,0 +1,114 @@
+/**
+ * LLM helper for the `akm index` memory-inference pass (#201).
+ *
+ * Compresses a single memory body into one higher-signal derived memory. The
+ * pass itself (in `src/indexer/memory-inference.ts`) is responsible for
+ * deciding which memories are pending, persisting the derived memory with the
+ * correct frontmatter (`inferred: true`, `source: <parent-ref>`), and marking
+ * the parent as processed for idempotency.
+ *
+ * This module is intentionally tiny and stateless so tests can stub it via
+ * `mock.module("../src/llm/memory-infer", ...)` without hitting a network.
+ *
+ * Locked v1 contract (#208): the LLM connection always comes from the
+ * shared `akm.llm` block — never from a per-pass override. Callers obtain
+ * the connection via `resolveIndexPassLLM("memory", config)` and pass it
+ * straight through.
+ */
+import { toErrorMessage } from "../core/common";
+import { warn } from "../core/warn";
+import { chatCompletion, parseEmbeddedJsonResponse } from "./client";
+/** Hard cap on body chars sent to the model — pragmatic and matches `runLlmEnrich`. */
+const MAX_BODY_CHARS = 4000;
+/** Hard timeout for the LLM call. The index run must not hang on a misbehaving endpoint. */
+const LLM_TIMEOUT_MS = 30_000;
+const SYSTEM_PROMPT = "You compress a developer memory into one high-signal derived memory for later retrieval. " +
+    "Return only valid JSON. No prose outside the JSON object. No markdown fences.";
+const USER_PROMPT_PREFIX = `Compress the memory below into one concise, information-dense derived memory.
+
+Rules:
+- Output ONLY a JSON object with exactly these keys: {"title": string, "description": string, "tags": string[], "searchHints": string[], "content": string}.
+- ` +
+    '"title"' +
+    ` is a short, descriptive title for the derived memory.
+- ` +
+    '"description"' +
+    ` is one sentence explaining why this derived memory matters.
+- ` +
+    '"tags"' +
+    ` contains 3-8 specific keywords.
+- ` +
+    '"searchHints"' +
+    ` contains 3-6 natural-language retrieval phrases.
+- ` +
+    '"content"' +
+    ` must be compact markdown that preserves the reusable insight, root cause, fix, constraints, and applicability conditions when present.
+- Prefer 2-4 short sections with informative headings over long prose.
+- Omit timestamps, verification-only metrics, pleasantries, and session-specific chatter unless they are essential to applying the insight later.
+- Preserve technical specifics (names, versions, identifiers, selectors, file paths, config keys) verbatim.
+
+Memory:
+`;
+/**
+ * Compress a single memory body into one derived memory via the configured LLM.
+ *
+ * Returns `undefined` on any failure (timeout, invalid JSON, empty response).
+ * Errors
+ * are logged via `warn()` but never thrown — a failed split for one memory
+ * must not abort the rest of the index pass.
+ */
+export async function compressMemoryToDerivedMemory(llmConfig, body, signal) {
+    const trimmedBody = body.trim();
+    if (!trimmedBody)
+        return undefined;
+    const userPrompt = `${USER_PROMPT_PREFIX}${trimmedBody.slice(0, MAX_BODY_CHARS)}`;
+    let timeoutHandle;
+    try {
+        const raw = await Promise.race([
+            chatCompletion(llmConfig, [
+                { role: "system", content: SYSTEM_PROMPT },
+                { role: "user", content: userPrompt },
+            ], { maxTokens: 768, temperature: 0.1, signal }),
+            new Promise((_, reject) => {
+                timeoutHandle = setTimeout(() => reject(new Error("memory inference timed out")), LLM_TIMEOUT_MS);
+            }),
+        ]);
+        if (!raw)
+            return undefined;
+        const parsed = parseEmbeddedJsonResponse(raw);
+        if (!parsed) {
+            warn("memory inference: invalid JSON response from LLM; skipping memory.");
+            return undefined;
+        }
+        const title = typeof parsed.title === "string" ? parsed.title.trim() : "";
+        const description = typeof parsed.description === "string" ? parsed.description.trim() : "";
+        const content = typeof parsed.content === "string" ? parsed.content.trim() : "";
+        const tags = Array.isArray(parsed.tags)
+            ? parsed.tags
+                .filter((t) => typeof t === "string")
+                .map((t) => t.trim())
+                .filter(Boolean)
+                .slice(0, 8)
+            : [];
+        const searchHints = Array.isArray(parsed.searchHints)
+            ? parsed.searchHints
+                .filter((h) => typeof h === "string")
+                .map((h) => h.trim())
+                .filter(Boolean)
+                .slice(0, 6)
+            : [];
+        if (!title || !description || !content || tags.length === 0 || searchHints.length === 0) {
+            warn("memory inference: incomplete derived memory payload from LLM; skipping memory.");
+            return undefined;
+        }
+        return { title, description, tags, searchHints, content };
+    }
+    catch (err) {
+        warn(`memory inference failed: ${toErrorMessage(err)}`);
+        return undefined;
+    }
+    finally {
+        if (timeoutHandle !== undefined)
+            clearTimeout(timeoutHandle);
+    }
+}

package/dist/{src/llm → llm}/metadata-enhance.js

@@ -11,7 +11,7 @@ const SYSTEM_PROMPT = `You are a metadata generator for a developer asset regist
  * Use an LLM to enhance a stash entry's metadata: improve description,
  * generate searchHints, and suggest tags.
  */
-export async function enhanceMetadata(config, entry, fileContent) {
+export async function enhanceMetadata(config, entry, fileContent, signal) {
     const contextParts = [`Name: ${entry.name}`, `Type: ${entry.type}`];
     if (entry.description)
         contextParts.push(`Current description: ${entry.description}`);
@@ -33,7 +33,7 @@ Return ONLY the JSON object, no explanation.`;
     const raw = await chatCompletion(config, [
         { role: "system", content: SYSTEM_PROMPT },
         { role: "user", content: userPrompt },
-    ]);
+    ], { signal });
     const parsed = parseJsonResponse(raw);
     if (!parsed)
         return {};

package/dist/{src/output → output}/cli-hints.js

@@ -133,6 +133,7 @@ akm remember "Deployment needs VPN access"     # Record a memory in your stash
 akm remember --name release-retro < notes.md   # Save multiline memory from stdin
 akm import ./docs/auth-flow.md                 # Import a file as knowledge
 akm import - --name scratch-notes < notes.md   # Import stdin as a knowledge doc
+akm import https://example.com/docs/auth       # Fetch one URL and import it as knowledge
 akm workflow create ship-release               # Create a workflow asset in the stash
 akm workflow validate workflows/foo.md         # Validate a workflow file or ref; lists every error
 akm workflow next workflow:ship-release        # Start or resume the next workflow step
@@ -160,6 +161,7 @@ akm wiki show research                         # Path, description, counts, last
 akm wiki pages research                        # Page refs + descriptions (excludes schema/index/log; includes raw/)
 akm wiki search research "attention"           # Scoped search (equivalent to --type wiki --wiki research)
 akm wiki stash research ./paper.md             # Copy source into raw/<slug>.md (never overwrites)
+akm wiki stash research https://example.com/paper # Fetch one URL into raw/<slug>.md
 echo "..." | akm wiki stash research -         # stdin form
 akm wiki lint research                         # Structural checks: orphans, broken xrefs, uncited raws, stale index
 akm wiki ingest research                       # Print the ingest workflow for this wiki (no action)

package/dist/{src/setup → setup}/setup.js

@@ -19,15 +19,22 @@ import { probeLlmCapabilities } from "../llm/client";
 import { checkEmbeddingAvailability, DEFAULT_LOCAL_MODEL, isTransformersAvailable } from "../llm/embedder";
 import { detectAgentPlatforms, detectOllama } from "./detect";
 import { createSetupContext, runSetupSteps } from "./steps";
-// ── Constants ───────────────────────────────────────────────────────────────
 /**
  * Recommended GitHub repositories shown during setup.
- *
- * Currently empty — populating from the akm-registry at runtime is a
- * separate feature. The wizard prompt infrastructure is retained for that
- * future use.
  */
-const RECOMMENDED_GITHUB_REPOS = [];
+const RECOMMENDED_GITHUB_REPOS = [
+    {
+        url: "https://github.com/itlackey/akm-stash",
+        name: "itlackey/akm-stash",
+        hint: "official onboarding stash",
+        defaultSelected: true,
+    },
+    {
+        url: "https://github.com/andrewyng/context-hub",
+        name: "andrewyng/context-hub",
+        hint: "optional community prompt and context stash",
+    },
+];
 // Approximate first-download sizes used in the setup note.
 // LOCAL_MODEL_APPROX_SIZE_MB tracks the default local model (DEFAULT_LOCAL_MODEL).
 const LOCAL_MODEL_APPROX_SIZE_MB = 130;
@@ -466,7 +473,7 @@ export async function stepLlm(current, ollamaEndpoint, ollamaChatModels) {
     }
     return llm;
 }
-async function stepRegistries(current) {
+export async function stepRegistries(current) {
     const defaults = DEFAULT_CONFIG.registries ?? [];
     const currentRegistries = current.registries ?? defaults;
     const defaultUrls = new Set(defaults.map((r) => r.url));
@@ -508,40 +515,43 @@ async function stepRegistries(current) {
  * @internal Exported for testing only.
  */
 export async function stepAddSources(current) {
-    const stashes = [...(current.sources ?? current.stashes ?? [])];
-    if (stashes.length > 0) {
-        p.log.info(`You have ${stashes.length} existing stash source(s).`);
+    const sources = [...(current.sources ?? current.stashes ?? [])];
+    if (sources.length > 0) {
+        p.log.info(`You have ${sources.length} existing stash source(s).`);
     }
     // ── Recommended GitHub repos ───────────────────────────────────────────
     // Skip the prompt entirely when there are no recommendations to show.
     // The infrastructure is retained for a future registry-driven version.
     if (RECOMMENDED_GITHUB_REPOS.length > 0) {
-        const existingUrls = new Set(stashes.map((s) => s.url));
+        const existingUrls = new Set(sources.map((s) => s.url));
         const repoOptions = RECOMMENDED_GITHUB_REPOS.map((r) => ({
             value: r.url,
             label: r.name,
             hint: existingUrls.has(r.url) ? `${r.hint} (already added)` : r.hint,
         }));
+        const initialValues = sources.length > 0
+            ? repoOptions.filter((o) => existingUrls.has(o.value)).map((o) => o.value)
+            : RECOMMENDED_GITHUB_REPOS.filter((r) => r.defaultSelected).map((r) => r.url);
         const selectedRepos = await prompt(() => p.multiselect({
             message: "Recommended GitHub repositories — toggle to add or remove:",
             options: repoOptions,
-            initialValues: repoOptions.filter((o) => existingUrls.has(o.value)).map((o) => o.value),
+            initialValues,
             required: false,
         }));
         // Add newly selected repos
         for (const url of selectedRepos) {
             if (!existingUrls.has(url)) {
                 const rec = RECOMMENDED_GITHUB_REPOS.find((r) => r.url === url);
-                stashes.push({ type: "git", url, name: rec?.name });
+                sources.push({ type: "git", url, name: rec?.name });
                 existingUrls.add(url);
             }
         }
         // Remove deselected repos that were previously configured
         for (const rec of RECOMMENDED_GITHUB_REPOS) {
             if (existingUrls.has(rec.url) && !selectedRepos.includes(rec.url)) {
-                const idx = stashes.findIndex((s) => s.url === rec.url);
+                const idx = sources.findIndex((s) => s.url === rec.url);
                 if (idx !== -1) {
-                    stashes.splice(idx, 1);
+                    sources.splice(idx, 1);
                     existingUrls.delete(rec.url);
                     p.log.info(`Removed ${rec.name}.`);
                 }
@@ -583,8 +593,8 @@ export async function stepAddSources(current) {
             const entry = { type: "git", url: url.trim() };
             if (name.trim())
                 entry.name = name.trim();
-            if (!stashes.some((s) => s.url === entry.url)) {
-                stashes.push(entry);
+            if (!sources.some((s) => s.url === entry.url)) {
+                sources.push(entry);
             }
             else {
                 p.log.warn("This URL is already configured.");
@@ -611,15 +621,15 @@ export async function stepAddSources(current) {
             const entry = { type: "filesystem", path: resolved };
             if (name.trim())
                 entry.name = name.trim();
-            if (!stashes.some((s) => s.path === entry.path)) {
-                stashes.push(entry);
+            if (!sources.some((s) => s.path === entry.path)) {
+                sources.push(entry);
             }
             else {
                 p.log.warn("This path is already configured.");
             }
         }
     }
-    return stashes;
+    return sources;
 }
 async function stepAgentPlatforms(current) {
     const platforms = detectAgentPlatforms();

package/dist/sources/providers/website.js

@@ -0,0 +1,27 @@
+import { registerSourceProvider } from "../provider-factory";
+import { ensureWebsiteMirror, getWebsiteCachePaths, validateWebsiteUrl } from "../website-ingest";
+/**
+ * Website source provider — thin adapter over the shared website ingest module.
+ */
+class WebsiteSourceProvider {
+    kind = "website";
+    name;
+    #config;
+    #url;
+    constructor(config) {
+        this.#config = config;
+        this.name = config.name ?? "website";
+        this.#url = validateWebsiteUrl(config.url ?? "");
+    }
+    async init(_ctx) {
+        // URL validation already happens in the constructor; nothing else to do.
+    }
+    path() {
+        return getWebsiteCachePaths(this.#url).stashDir;
+    }
+    async sync() {
+        await ensureWebsiteMirror(this.#config, { requireStashDir: true });
+    }
+}
+registerSourceProvider("website", (config) => new WebsiteSourceProvider(config));
+export { WebsiteSourceProvider };