bc-observatory 0.2.0 → 0.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bc-observatory",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "MCP server for BC Observatory: an agent-first, cross-referenced knowledge base of Microsoft Dynamics 365 Business Central (Learn, code, roadmap, videos, community blogs).",
   "type": "module",
   "bin": {
@@ -28,12 +28,15 @@
     "model-context-protocol"
   ],
   "scripts": {
-    "build": "tsc -p tsconfig.json && chmod +x dist/server.js",
+    "build": "tsc -p tsconfig.json --noEmit && tsx build.ts",
     "start": "node dist/server.js"
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.32.1",
-    "minisearch": "^7.2.0",
     "zod": "^4.6.5"
+  },
+  "devDependencies": {
+    "@bc-observatory/search": "0.1.0",
+    "esbuild": "^0.28.2"
   }
 }

package/dist/embed.js

@@ -1,133 +0,0 @@
-export const MAX_TOKENS = 512;
-/** tokenizer.json (WordPiece), model.safetensors (one F32 tensor) and config.json into a model. */
-export function parseModel(tokenizerJson, safetensors, config = {}) {
-    const t = JSON.parse(tokenizerJson);
-    if (t.model.type !== "WordPiece")
-        throw new Error(`unsupported tokenizer model ${t.model.type}`);
-    const vocab = new Map(Object.entries(t.model.vocab));
-    const view = new DataView(safetensors.buffer, safetensors.byteOffset, safetensors.byteLength);
-    const headerLen = Number(view.getBigUint64(0, true));
-    const header = JSON.parse(new TextDecoder().decode(safetensors.subarray(8, 8 + headerLen)));
-    const name = Object.keys(header).find((k) => k !== "__metadata__");
-    if (!name)
-        throw new Error("safetensors file has no tensor");
-    const { dtype, shape, data_offsets: [start, end] } = header[name];
-    if (dtype !== "F32" || shape.length !== 2)
-        throw new Error(`unsupported tensor ${name}: ${dtype} ${shape.join("x")}`);
-    // copy into an aligned buffer: the data starts at 8 + headerLen, which need not be a multiple of 4
-    const bytes = safetensors.slice(8 + headerLen + start, 8 + headerLen + end);
-    const matrix = new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
-    if (matrix.length !== shape[0] * shape[1])
-        throw new Error("tensor size does not match its shape");
-    const unk = t.model.unk_token !== undefined ? (vocab.get(t.model.unk_token) ?? null) : null;
-    return { dims: shape[1], vocab, unk, matrix, normalize: config.normalize !== false, maxWordChars: t.model.max_input_chars_per_word ?? 100, prefix: t.model.continuing_subword_prefix ?? "##" };
-}
-const CJK = [[0x4e00, 0x9fff], [0x3400, 0x4dbf], [0x20000, 0x2a6df], [0x2a700, 0x2b73f], [0x2b740, 0x2b81f], [0x2b820, 0x2ceaf], [0xf900, 0xfaff], [0x2f800, 0x2fa1f]];
-const isCjk = (cp) => CJK.some(([a, b]) => cp >= a && cp <= b);
-const isPunct = (ch, cp) => (cp >= 33 && cp <= 47) || (cp >= 58 && cp <= 64) || (cp >= 91 && cp <= 96) || (cp >= 123 && cp <= 126) || /\p{P}/u.test(ch);
-/** BertNormalizer: clean text, CJK ideographs spaced out, accents stripped, lowercased. */
-export function normalize(text) {
-    let out = "";
-    for (const ch of text) {
-        const cp = ch.codePointAt(0);
-        if (cp === 0 || cp === 0xfffd)
-            continue;
-        if (ch === "\t" || ch === "\n" || ch === "\r" || /\s/u.test(ch)) {
-            out += " ";
-            continue;
-        }
-        if (/[\p{Cc}\p{Cf}]/u.test(ch))
-            continue;
-        out += isCjk(cp) ? ` ${ch} ` : ch;
-    }
-    return out.normalize("NFD").replace(/\p{Mn}/gu, "").toLowerCase();
-}
-/** BertPreTokenizer: words split on whitespace, every punctuation character a word of its own. */
-export function preTokenize(text) {
-    const words = [];
-    let cur = "";
-    for (const ch of text) {
-        if (/\s/u.test(ch)) {
-            if (cur)
-                words.push(cur);
-            cur = "";
-            continue;
-        }
-        if (isPunct(ch, ch.codePointAt(0))) {
-            if (cur)
-                words.push(cur);
-            words.push(ch);
-            cur = "";
-            continue;
-        }
-        cur += ch;
-    }
-    if (cur)
-        words.push(cur);
-    return words;
-}
-/** Token ids of a text: no special tokens, no [UNK], at most MAX_TOKENS. */
-export function tokenize(m, text) {
-    const ids = [];
-    for (const word of preTokenize(normalize(text))) {
-        const chars = [...word];
-        if (chars.length > m.maxWordChars)
-            continue; // the whole word would be [UNK], which is dropped
-        const pieces = [];
-        let start = 0, bad = false;
-        while (start < chars.length) {
-            let end = chars.length, found;
-            while (start < end) {
-                const sub = (start > 0 ? m.prefix : "") + chars.slice(start, end).join("");
-                found = m.vocab.get(sub);
-                if (found !== undefined)
-                    break;
-                end--;
-            }
-            if (found === undefined) {
-                bad = true;
-                break;
-            }
-            pieces.push(found);
-            start = end;
-        }
-        if (!bad)
-            for (const id of pieces)
-                if (id !== m.unk)
-                    ids.push(id);
-        if (ids.length >= MAX_TOKENS)
-            return ids.slice(0, MAX_TOKENS);
-    }
-    return ids;
-}
-/** The text's vector: mean of its tokens' rows, L2-normalized (all zeros for a text with no known token). */
-export function embed(m, text) {
-    const v = new Float32Array(m.dims);
-    const ids = tokenize(m, text);
-    if (!ids.length)
-        return v;
-    for (const id of ids) {
-        const off = id * m.dims;
-        for (let d = 0; d < m.dims; d++)
-            v[d] += m.matrix[off + d];
-    }
-    for (let d = 0; d < m.dims; d++)
-        v[d] /= ids.length;
-    if (m.normalize) {
-        let n = 0;
-        for (let d = 0; d < m.dims; d++)
-            n += v[d] * v[d];
-        n = Math.sqrt(n);
-        if (n > 0)
-            for (let d = 0; d < m.dims; d++)
-                v[d] /= n;
-    }
-    return v;
-}
-/** Dot product: the cosine similarity of two normalized vectors. */
-export function dot(a, b, bOffset = 0) {
-    let s = 0;
-    for (let d = 0; d < a.length; d++)
-        s += a[d] * b[bOffset + d];
-    return s;
-}