bc-observatory 0.1.1 → 0.2.0

package/README.md

@@ -1,7 +1,7 @@
 # bc-observatory (MCP server)
 
 [BC Observatory](https://waldo1001.github.io/waldo.BCObservatory/) for agents: an agent-first, cross-referenced knowledge base of
-Microsoft Dynamics 365 Business Central (Learn topic hubs, AL objects extracted from the code for BC28-30, country localizations,
+Microsoft Dynamics 365 Business Central (Learn topic hubs, AL objects extracted from the code for BC28-30 with history back to BC23, country localizations,
 Microsoft 365 roadmap features, videos and community posts), every claim with a trust tier and evidence.
 
 ```bash
@@ -16,4 +16,11 @@ The server downloads the site's search index (cached by hash in `~/.cache/bc-obs
 markdown. `BC_OBSERVATORY_LOCAL=<checkout of waldo1001/waldo.BCObservatory>` reads everything from disk instead;
 `BC_OBSERVATORY_SITE` points at another deployment.
 
+`search` is hybrid: keywords plus meaning, so "who is allowed to see what" finds the permission sets. Meaning comes from
+static embeddings ([model2vec](https://github.com/MinishLab/model2vec) `minishlab/potion-base-8M`, MIT) computed locally in
+plain JavaScript; the 30 MB model is downloaded once on the first search, pinned by revision and checked by SHA-256. Pass
+`mode: "keyword"` for exact terms such as an object name. `BC_OBSERVATORY_EMBEDDINGS=0` turns it off, and
+`BC_OBSERVATORY_MODEL_DIR=<dir with config.json, tokenizer.json, model.safetensors>` uses a model on disk (offline). Without a
+model, search is keyword-only and says so.
+
 MIT. Content: see the repository's CONTENT-NOTICE.md (Microsoft content attributed; community content derived only).

package/dist/embed.js

@@ -0,0 +1,133 @@
+export const MAX_TOKENS = 512;
+/** tokenizer.json (WordPiece), model.safetensors (one F32 tensor) and config.json into a model. */
+export function parseModel(tokenizerJson, safetensors, config = {}) {
+    const t = JSON.parse(tokenizerJson);
+    if (t.model.type !== "WordPiece")
+        throw new Error(`unsupported tokenizer model ${t.model.type}`);
+    const vocab = new Map(Object.entries(t.model.vocab));
+    const view = new DataView(safetensors.buffer, safetensors.byteOffset, safetensors.byteLength);
+    const headerLen = Number(view.getBigUint64(0, true));
+    const header = JSON.parse(new TextDecoder().decode(safetensors.subarray(8, 8 + headerLen)));
+    const name = Object.keys(header).find((k) => k !== "__metadata__");
+    if (!name)
+        throw new Error("safetensors file has no tensor");
+    const { dtype, shape, data_offsets: [start, end] } = header[name];
+    if (dtype !== "F32" || shape.length !== 2)
+        throw new Error(`unsupported tensor ${name}: ${dtype} ${shape.join("x")}`);
+    // copy into an aligned buffer: the data starts at 8 + headerLen, which need not be a multiple of 4
+    const bytes = safetensors.slice(8 + headerLen + start, 8 + headerLen + end);
+    const matrix = new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
+    if (matrix.length !== shape[0] * shape[1])
+        throw new Error("tensor size does not match its shape");
+    const unk = t.model.unk_token !== undefined ? (vocab.get(t.model.unk_token) ?? null) : null;
+    return { dims: shape[1], vocab, unk, matrix, normalize: config.normalize !== false, maxWordChars: t.model.max_input_chars_per_word ?? 100, prefix: t.model.continuing_subword_prefix ?? "##" };
+}
+const CJK = [[0x4e00, 0x9fff], [0x3400, 0x4dbf], [0x20000, 0x2a6df], [0x2a700, 0x2b73f], [0x2b740, 0x2b81f], [0x2b820, 0x2ceaf], [0xf900, 0xfaff], [0x2f800, 0x2fa1f]];
+const isCjk = (cp) => CJK.some(([a, b]) => cp >= a && cp <= b);
+const isPunct = (ch, cp) => (cp >= 33 && cp <= 47) || (cp >= 58 && cp <= 64) || (cp >= 91 && cp <= 96) || (cp >= 123 && cp <= 126) || /\p{P}/u.test(ch);
+/** BertNormalizer: clean text, CJK ideographs spaced out, accents stripped, lowercased. */
+export function normalize(text) {
+    let out = "";
+    for (const ch of text) {
+        const cp = ch.codePointAt(0);
+        if (cp === 0 || cp === 0xfffd)
+            continue;
+        if (ch === "\t" || ch === "\n" || ch === "\r" || /\s/u.test(ch)) {
+            out += " ";
+            continue;
+        }
+        if (/[\p{Cc}\p{Cf}]/u.test(ch))
+            continue;
+        out += isCjk(cp) ? ` ${ch} ` : ch;
+    }
+    return out.normalize("NFD").replace(/\p{Mn}/gu, "").toLowerCase();
+}
+/** BertPreTokenizer: words split on whitespace, every punctuation character a word of its own. */
+export function preTokenize(text) {
+    const words = [];
+    let cur = "";
+    for (const ch of text) {
+        if (/\s/u.test(ch)) {
+            if (cur)
+                words.push(cur);
+            cur = "";
+            continue;
+        }
+        if (isPunct(ch, ch.codePointAt(0))) {
+            if (cur)
+                words.push(cur);
+            words.push(ch);
+            cur = "";
+            continue;
+        }
+        cur += ch;
+    }
+    if (cur)
+        words.push(cur);
+    return words;
+}
+/** Token ids of a text: no special tokens, no [UNK], at most MAX_TOKENS. */
+export function tokenize(m, text) {
+    const ids = [];
+    for (const word of preTokenize(normalize(text))) {
+        const chars = [...word];
+        if (chars.length > m.maxWordChars)
+            continue; // the whole word would be [UNK], which is dropped
+        const pieces = [];
+        let start = 0, bad = false;
+        while (start < chars.length) {
+            let end = chars.length, found;
+            while (start < end) {
+                const sub = (start > 0 ? m.prefix : "") + chars.slice(start, end).join("");
+                found = m.vocab.get(sub);
+                if (found !== undefined)
+                    break;
+                end--;
+            }
+            if (found === undefined) {
+                bad = true;
+                break;
+            }
+            pieces.push(found);
+            start = end;
+        }
+        if (!bad)
+            for (const id of pieces)
+                if (id !== m.unk)
+                    ids.push(id);
+        if (ids.length >= MAX_TOKENS)
+            return ids.slice(0, MAX_TOKENS);
+    }
+    return ids;
+}
+/** The text's vector: mean of its tokens' rows, L2-normalized (all zeros for a text with no known token). */
+export function embed(m, text) {
+    const v = new Float32Array(m.dims);
+    const ids = tokenize(m, text);
+    if (!ids.length)
+        return v;
+    for (const id of ids) {
+        const off = id * m.dims;
+        for (let d = 0; d < m.dims; d++)
+            v[d] += m.matrix[off + d];
+    }
+    for (let d = 0; d < m.dims; d++)
+        v[d] /= ids.length;
+    if (m.normalize) {
+        let n = 0;
+        for (let d = 0; d < m.dims; d++)
+            n += v[d] * v[d];
+        n = Math.sqrt(n);
+        if (n > 0)
+            for (let d = 0; d < m.dims; d++)
+                v[d] /= n;
+    }
+    return v;
+}
+/** Dot product: the cosine similarity of two normalized vectors. */
+export function dot(a, b, bOffset = 0) {
+    let s = 0;
+    for (let d = 0; d < a.length; d++)
+        s += a[d] * b[bOffset + d];
+    return s;
+}

package/dist/server.js

@@ -8,6 +8,11 @@
  * BC_OBSERVATORY_SITE to point at another deployment.
  *
  * Tools: search, ls, cat, get_object, diff_object, localization, whats_new, blog_footprint, feedback.
+ *
+ * Search is hybrid (D63): MiniSearch keywords fused with static embeddings (model2vec potion-base-8M, MIT) by
+ * reciprocal rank, so "client" finds Customer. The model (30 MB) is downloaded once on the first search, pinned by
+ * revision and checked by SHA-256, into ~/.cache/bc-observatory/models/. BC_OBSERVATORY_EMBEDDINGS=0 turns it off;
+ * BC_OBSERVATORY_MODEL_DIR uses a model on disk (offline, tests). Without a model, search is keyword-only.
  */
 import { createHash } from "node:crypto";
 import { existsSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
@@ -18,7 +23,8 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import MiniSearch from "minisearch";
 import { z } from "zod";
-const VERSION = "0.1.1";
+import { dot, embed, parseModel } from "./embed.js";
+const VERSION = "0.2.0";
 const SITE = (process.env.BC_OBSERVATORY_SITE ?? "https://waldo1001.github.io/waldo.BCObservatory/").replace(/\/?$/, "/");
 const LOCAL = process.env.BC_OBSERVATORY_LOCAL ? resolve(process.env.BC_OBSERVATORY_LOCAL) : null;
 const REPO = "https://github.com/waldo1001/waldo.BCObservatory";
@@ -85,15 +91,94 @@ async function loadIndex() {
 const url = (path) => `${SITE}${path}/`;
 const line = (r) => `- ${r.title} [${r.type}${r.tier ? `, ${r.tier}` : ""}${r.date ? `, ${r.date}` : ""}] path=${r.path}\n  ${r.summary}`;
 const text = (t) => ({ content: [{ type: "text", text: t }] });
+// ---------------------------------------------------------------------------------------------- embeddings (D63)
+const MODEL = {
+    repo: "minishlab/potion-base-8M", revision: "bf8b056651a2c21b8d2565580b8569da283cab23",
+    sha256: {
+        "config.json": "2a6ac0e9aaa356a68a5688070db78fc3a464fefe85d2f06a1905ce3718687553",
+        "tokenizer.json": "e67e803f624fb4d67dea1c730d06e1067e1b14d830e2c2202569e3ef0f70bb50",
+        "model.safetensors": "f65d0f325faadc1e121c319e2faa41170d3fa07d8c89abd48ca5358d9a223de2",
+    },
+};
+const MODEL_DIR = process.env.BC_OBSERVATORY_MODEL_DIR ? resolve(process.env.BC_OBSERVATORY_MODEL_DIR) : null;
+// a local checkout is for offline use and tests: no download there unless a model directory is named
+const EMBEDDINGS = process.env.BC_OBSERVATORY_EMBEDDINGS !== "0" && (!LOCAL || !!MODEL_DIR);
+let model = null;
+let modelError = EMBEDDINGS ? null : "turned off";
+let vectors = null;
+async function loadModel() {
+    if (model || modelError)
+        return model;
+    try {
+        let dir = MODEL_DIR;
+        if (!dir) {
+            dir = join(process.env.XDG_CACHE_HOME ?? join(homedir(), ".cache"), "bc-observatory", "models", `potion-base-8M@${MODEL.revision.slice(0, 12)}`);
+            mkdirSync(dir, { recursive: true });
+            for (const [file, sha] of Object.entries(MODEL.sha256)) {
+                const p = join(dir, file);
+                if (existsSync(p))
+                    continue;
+                const res = await fetch(`https://huggingface.co/${MODEL.repo}/resolve/${MODEL.revision}/${file}`, { headers: { "user-agent": `bc-observatory-mcp/${VERSION}` } });
+                if (!res.ok)
+                    throw new Error(`HTTP ${res.status} for ${file}`);
+                const buf = Buffer.from(await res.arrayBuffer());
+                if (createHash("sha256").update(buf).digest("hex") !== sha)
+                    throw new Error(`${file} does not match its pinned SHA-256`);
+                writeFileSync(p, buf);
+            }
+        }
+        model = parseModel(readFileSync(join(dir, "tokenizer.json"), "utf8"), new Uint8Array(readFileSync(join(dir, "model.safetensors"))), JSON.parse(readFileSync(join(dir, "config.json"), "utf8")));
+    }
+    catch (e) {
+        modelError = String(e.message).slice(0, 160);
+    }
+    return model;
+}
+/** One vector per page (title, summary, tags), rebuilt when the index reloads: 22k pages take about 0.3 s. */
+function pageVectors(m) {
+    if (vectors && vectors.at === loadedAt)
+        return vectors.data;
+    const data = new Float32Array(pages.length * m.dims);
+    pages.forEach((p, i) => data.set(embed(m, [p.title, p.summary, ...(p.tags ?? [])].filter(Boolean).join(". ")), i * m.dims));
+    vectors = { at: loadedAt, data };
+    return data;
+}
+/** Reciprocal rank fusion of ranked lists of paths (k = 60). */
+export function fuse(lists, k = 60) {
+    const score = new Map();
+    for (const list of lists)
+        list.forEach((p, i) => score.set(p, (score.get(p) ?? 0) + 1 / (k + i + 1)));
+    return [...score].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).map(([p]) => p);
+}
 // ---------------------------------------------------------------------------------------------- tools
 export async function toolSearch(a) {
     await loadIndex();
-    const hits = search.search(a.query, {
-        filter: (r) => (!a.type || r.type === a.type) && (!a.tier || r.tier === a.tier) && (!a.system || r.system === a.system),
-    }).slice(0, a.limit ?? 10);
+    const keep = (r) => (!a.type || r.type === a.type) && (!a.tier || r.tier === a.tier) && (!a.system || r.system === a.system);
+    const limit = a.limit ?? 10, mode = a.mode ?? "hybrid", POOL = 50;
+    const keyword = mode === "semantic" ? [] : search.search(a.query, { filter: (r) => keep(r) }).slice(0, POOL).map((r) => r.path);
+    let semantic = [], note = "";
+    if (mode !== "keyword") {
+        const m = await loadModel();
+        if (m) {
+            const q = embed(m, a.query), data = pageVectors(m);
+            if (q.some((x) => x !== 0)) {
+                const scored = [];
+                pages.forEach((p, i) => { if (keep(p))
+                    scored.push([dot(q, data, i * m.dims), i]); });
+                semantic = scored.sort((x, y) => y[0] - x[0]).slice(0, POOL).map(([, i]) => pages[i].path);
+            }
+        }
+        else if (mode === "semantic")
+            note = ` (semantic search unavailable: ${modelError}; keyword results instead)`;
+        if (!m && mode === "semantic")
+            semantic = search.search(a.query, { filter: (r) => keep(r) }).slice(0, POOL).map((r) => r.path);
+    }
+    const byPath = new Map(pages.map((p) => [p.path, p]));
+    const hits = fuse([keyword, semantic].filter((l) => l.length)).slice(0, limit).map((p) => byPath.get(p)).filter(Boolean);
     if (!hits.length)
         return `No pages match "${a.query}". Try fewer words, or ls("") to browse sections.`;
-    return `${hits.length} results (read one with cat(path)):\n${hits.map(line).join("\n")}`;
+    const how = semantic.length && keyword.length ? "keyword + semantic" : semantic.length ? "semantic" : "keyword";
+    return `${hits.length} results, ${how}${note} (read one with cat(path)):\n${hits.map(line).join("\n")}`;
 }
 export async function toolLs(a) {
     await loadIndex();
@@ -135,7 +220,7 @@ export async function toolDiffObject(a) {
         diff = JSON.parse(await dataFile(`code/diffs/version/${a.from}__${a.to}.json`));
     }
     catch {
-        return `No version diff ${a.from} -> ${a.to}. Diffs exist between consecutive snapshot versions (28 -> 29, 29 -> 30).`;
+        return `No version diff ${a.from} -> ${a.to}. Diffs exist between consecutive majors from BC23 to BC30 (23 -> 24, ..., 29 -> 30); a changed member lists only what differs, as [from, to].`;
     }
     const o = diff.objects.find((x) => x.key === key);
     if (!o)
@@ -174,8 +259,8 @@ export function createServer() {
     const server = new McpServer({ name: "bc-observatory", version: VERSION }, {
         instructions: "BC Observatory: an agent-first knowledge base of Microsoft Dynamics 365 Business Central (Learn hubs, AL objects from the code for BC28-30, localizations, roadmap features, videos, community posts). Every page carries a trust tier (official = Microsoft, community = everyone else) and a review state: say which tier a claim comes from. Never invent AL object ids or version numbers: look them up with get_object. Start with search(), read pages with cat(path), browse with ls(path).",
     });
-    server.registerTool("search", { title: "Search the knowledge base", description: "Full-text search over every page (title, summary, tags). Filter by type (topic, feature, object, localization, video, post), tier (official, community) or galaxy system (finance, sales, development, ...).",
-        inputSchema: { query: z.string(), type: z.string().optional(), tier: z.string().optional(), system: z.string().optional(), limit: z.number().int().min(1).max(50).optional() } }, async (a) => text(await toolSearch(a)));
+    server.registerTool("search", { title: "Search the knowledge base", description: "Search every page (title, summary, tags). Hybrid by default: keywords plus meaning, so a synonym or a paraphrase still finds the page; mode 'keyword' for exact terms such as an object name, 'semantic' for meaning only. Filter by type (topic, feature, object, localization, video, post), tier (official, community) or galaxy system (finance, sales, development, ...).",
+        inputSchema: { query: z.string(), type: z.string().optional(), tier: z.string().optional(), system: z.string().optional(), limit: z.number().int().min(1).max(50).optional(), mode: z.enum(["hybrid", "keyword", "semantic"]).optional() } }, async (a) => text(await toolSearch(a)));
     server.registerTool("ls", { title: "List pages", description: "Browse the page tree, e.g. ls('objects/table') or ls('localizations'); empty path lists the sections.", inputSchema: { path: z.string().optional() } }, async (a) => text(await toolLs(a)));
     server.registerTool("cat", { title: "Read a page", description: "The full markdown of a page (frontmatter with evidence and links, then the body), by its path from search or ls.", inputSchema: { path: z.string() } }, async (a) => text(await toolCat(a)));
     server.registerTool("get_object", { title: "Look up an AL object", description: "An AL object of W1 or a Microsoft first-party app by type and id or exact name, e.g. ('table', '18') or ('codeunit', 'Sales-Post'): fields, keys, events, public procedures, obsolete state, versions, countries that replace it, Learn pages.",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bc-observatory",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "MCP server for BC Observatory: an agent-first, cross-referenced knowledge base of Microsoft Dynamics 365 Business Central (Learn, code, roadmap, videos, community blogs).",
   "type": "module",
   "bin": {