akm-cli 0.9.0-beta.46 → 0.9.0-beta.47

package/dist/assets/wiki/ingest-workflow-template.md

@@ -6,20 +6,27 @@ Schema: {{SCHEMA_PATH}}
 Follow these steps. akm commands handle the invariants; use your native
 Read/Write/Edit tools for page edits.
 
+This workflow is for ingesting sources that are ALREADY present under
+`{{WIKI_DIR}}/raw/`. Do not ask the user for a source unless the raw queue is
+empty and the caller explicitly asked for interactive ingest.
+
 1. **Read the schema.** Open `{{SCHEMA_PATH}}`. It defines the voice, page
    kinds, contradiction policy, and any wiki-specific conventions. Do not
    skip this step even on familiar wikis — the schema may have changed.
 
-2. **File the source under `raw/`.**
+2. **Discover the pending raw queue.**
    ```sh
-   akm wiki stash {{WIKI_NAME}} <path-or-url-to-source>
-   # or: cat <source> | akm wiki stash {{WIKI_NAME}} -
+   akm wiki lint {{WIKI_NAME}}
    ```
-   Returns `{ slug, path, ref }`. The raw copy is immutable — never edit it.
+   Focus on `uncited-raw` findings: those raw files exist under `raw/` but are
+   not yet cited by any authored page. Treat each `uncited-raw` finding as a
+   pending ingest item. If there are no `uncited-raw` findings, exit cleanly
+   after a final `akm index` + `akm wiki lint {{WIKI_NAME}}` verification.
 
-3. **Find related existing pages.**
+3. **For each pending raw file, read the source and find related pages.**
+   Open the raw file directly from `{{WIKI_DIR}}/raw/`, then search:
    ```sh
-   akm wiki search {{WIKI_NAME}} "<key terms from the source>"
+   akm wiki search {{WIKI_NAME}} "<key terms from the raw source>"
    ```
    Read the top hits with `akm show wiki:{{WIKI_NAME}}/<page>`. Use
    `akm show wiki:{{WIKI_NAME}}/<page> toc` for large pages.
@@ -39,8 +46,8 @@ Read/Write/Edit tools for page edits.
 6. **Update xrefs both ways.** If page A now xrefs page B, page B must xref
    page A. `akm wiki lint {{WIKI_NAME}}` will flag violations.
 
-7. **Append to `log.md`.** One entry per ingest: date, source slug, one-line
-   summary, refs to created/edited pages. Newest at the top.
+7. **Append to `log.md`.** One entry per ingested raw source: date, raw slug,
+   one-line summary, refs to created/edited pages. Newest at the top.
 
 8. **Regenerate the index + verify.**
    ```sh
@@ -50,5 +57,5 @@ Read/Write/Edit tools for page edits.
    Resolve any lint findings before calling the ingest done.
 
 That's it. `akm` never calls an LLM — reasoning is your job; it just owns
-the invariants (raw immutability, unique slugs, ref validation, index
-regeneration, structural lint).
+the invariants (raw immutability, ref validation, index regeneration,
+structural lint).

package/dist/indexer/search/semantic-status.js

@@ -4,8 +4,12 @@
 import fs from "node:fs";
 import { writeFileAtomic } from "../../core/common.js";
 import { getCacheDir, getSemanticStatusPath } from "../../core/paths.js";
+import { DETERMINISTIC_EMBED_MODEL_ID, isDeterministicEmbedEnabled } from "../../llm/embedders/deterministic.js";
 import { DEFAULT_LOCAL_MODEL } from "../../llm/embedders/local.js";
 export function deriveSemanticProviderFingerprint(embedding) {
+    if (isDeterministicEmbedEnabled()) {
+        return `deterministic:${DETERMINISTIC_EMBED_MODEL_ID}`;
+    }
     if (embedding?.endpoint) {
         return `remote:${embedding.endpoint}|${embedding.model}|${embedding.dimension ?? "default"}`;
     }

package/dist/llm/embedder.js

@@ -2,6 +2,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 import { embedCacheKey, getCachedEmbedding, setCachedEmbedding } from "./embedders/cache.js";
+import { DETERMINISTIC_EMBED_MODEL_ID, deterministicEmbed, isDeterministicEmbedEnabled, } from "./embedders/deterministic.js";
 import { DEFAULT_LOCAL_MODEL, isTransformersAvailable, LocalEmbedder } from "./embedders/local.js";
 import { hasRemoteEndpoint, RemoteEmbedder } from "./embedders/remote.js";
 // ── Re-exports (public API) ─────────────────────────────────────────────────
@@ -39,6 +40,10 @@ export function resetLocalEmbedder() {
  * and embedding config. Repeated identical queries return the cached vector.
  */
 export async function embed(text, embeddingConfig, signal) {
+    // Deterministic mode (env-gated, test/bench only): model-free, stable.
+    if (isDeterministicEmbedEnabled()) {
+        return deterministicEmbed(text);
+    }
     const key = embedCacheKey(text, embeddingConfig);
     const cached = getCachedEmbedding(key);
     if (cached)
@@ -58,6 +63,10 @@ export async function embed(text, embeddingConfig, signal) {
 export async function embedBatch(texts, embeddingConfig, signal) {
     if (texts.length === 0)
         return [];
+    // Deterministic mode (env-gated, test/bench only): model-free, stable.
+    if (isDeterministicEmbedEnabled()) {
+        return texts.map((t) => deterministicEmbed(t));
+    }
     if (embeddingConfig && hasRemoteEndpoint(embeddingConfig)) {
         return new RemoteEmbedder(embeddingConfig).embedBatch(texts, signal);
     }
@@ -95,6 +104,8 @@ export { cosineSimilarity } from "./embedders/types.js";
  *   - No config: use `DEFAULT_LOCAL_MODEL` (the shared singleton model).
  */
 export function resolveEmbeddingModelId(embeddingConfig) {
+    if (isDeterministicEmbedEnabled())
+        return DETERMINISTIC_EMBED_MODEL_ID;
     if (!embeddingConfig)
         return DEFAULT_LOCAL_MODEL;
     if (hasRemoteEndpoint(embeddingConfig))
@@ -106,6 +117,10 @@ export function resolveEmbeddingModelId(embeddingConfig) {
  * Check whether embedding is available with a detailed reason on failure.
  */
 export async function checkEmbeddingAvailability(embeddingConfig) {
+    // Deterministic mode (env-gated): always available — no model, no network.
+    if (isDeterministicEmbedEnabled()) {
+        return { available: true };
+    }
     if (embeddingConfig && hasRemoteEndpoint(embeddingConfig)) {
         try {
             await new RemoteEmbedder(embeddingConfig).embed("test");

package/dist/llm/embedders/deterministic.js

@@ -0,0 +1,66 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/** Env var that switches the whole embedding facade into deterministic mode. */
+export const DETERMINISTIC_EMBED_ENV = "AKM_EMBED_DETERMINISTIC";
+/**
+ * Vector width. Matches the default local model (`bge-small`, 384 dims) so the
+ * index DB's embedding column and sqlite-vec table dimensions line up without
+ * any extra config.
+ */
+export const DETERMINISTIC_EMBED_DIM = 384;
+/**
+ * Stable model id reported for deterministic mode. Used as the embedding
+ * `model_id` and folded into the provider fingerprint so a deterministic index
+ * is never confused with a real-model index (and vice versa).
+ */
+export const DETERMINISTIC_EMBED_MODEL_ID = "akm-deterministic-hash-v1";
+/** True when deterministic embedding is enabled via env. */
+export function isDeterministicEmbedEnabled() {
+    return process.env[DETERMINISTIC_EMBED_ENV] === "1";
+}
+/** FNV-1a 32-bit hash. Platform- and version-stable. */
+function fnv1a(str) {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < str.length; i++) {
+        h ^= str.charCodeAt(i);
+        // 32-bit FNV prime multiply via shifts to stay in uint32.
+        h = (h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24))) >>> 0;
+    }
+    return h >>> 0;
+}
+/** Lowercase, split on non-alphanumeric, drop empties. */
+function tokenize(text) {
+    return text
+        .toLowerCase()
+        .split(/[^a-z0-9]+/)
+        .filter((t) => t.length > 0);
+}
+/**
+ * Deterministically embed `text` into a unit-length vector of width `dim`
+ * using feature hashing. Empty / token-less input returns a fixed unit
+ * vector so cosine similarity never sees a zero vector (NaN guard).
+ */
+export function deterministicEmbed(text, dim = DETERMINISTIC_EMBED_DIM) {
+    const vec = new Array(dim).fill(0);
+    const tokens = tokenize(text);
+    for (const tok of tokens) {
+        const h = fnv1a(tok);
+        const idx = h % dim;
+        // Use a higher bit for the sign so it is independent of the bucket index.
+        const sign = (h >>> 16) & 1 ? 1 : -1;
+        vec[idx] += sign;
+    }
+    let norm = 0;
+    for (const v of vec)
+        norm += v * v;
+    norm = Math.sqrt(norm);
+    if (norm === 0) {
+        // No usable tokens — return a fixed, stable unit vector.
+        vec[0] = 1;
+        return vec;
+    }
+    for (let i = 0; i < dim; i++)
+        vec[i] /= norm;
+    return vec;
+}

package/dist/sources/wiki-fetchers/youtube.js

@@ -19,6 +19,10 @@ function extractVideoId(url) {
         const id = url.pathname.slice("/shorts/".length).split("/")[0]?.trim();
         return id || null;
     }
+    if (url.pathname.startsWith("/embed/")) {
+        const id = url.pathname.slice("/embed/".length).split("/")[0]?.trim();
+        return id || null;
+    }
     return null;
 }
 function canonicalWatchUrl(videoId) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "akm-cli",
-  "version": "0.9.0-beta.46",
+  "version": "0.9.0-beta.47",
   "type": "module",
   "description": "akm (Agent Knowledge Management) — A package manager for AI agent skills, commands, tools, and knowledge. Works with Claude Code, OpenCode, Cursor, and any AI coding assistant.",
   "keywords": [