@abhishekmcp/notes 0.2.0 → 0.4.0

package/dist/tokenizer.js

@@ -0,0 +1,140 @@
+/**
+ * Hand-rolled BERT-uncased WordPiece tokenizer — pure JS, no dependencies.
+ * Mirrors google-research/bert `tokenization.py` (BasicTokenizer +
+ * WordpieceTokenizer) closely enough to feed all-MiniLM-L6-v2. We hand-roll it
+ * for the same reason we hand-roll the frontmatter parser: avoid a native /
+ * audit-heavy dependency (@huggingface/tokenizers is a Rust binding).
+ */
+const SPECIAL = { unk: "[UNK]", cls: "[CLS]", sep: "[SEP]", pad: "[PAD]" };
+/** True if a Unicode code point is a CJK character (BERT `_is_chinese_char`). */
+function isCjk(cp) {
+    return ((cp >= 0x4e00 && cp <= 0x9fff) ||
+        (cp >= 0x3400 && cp <= 0x4dbf) ||
+        (cp >= 0x20000 && cp <= 0x2a6df) ||
+        (cp >= 0x2a700 && cp <= 0x2b73f) ||
+        (cp >= 0x2b740 && cp <= 0x2b81f) ||
+        (cp >= 0x2b820 && cp <= 0x2ceaf) ||
+        (cp >= 0xf900 && cp <= 0xfaff) ||
+        (cp >= 0x2f800 && cp <= 0x2fa1f));
+}
+/** BERT `_is_punctuation`: all non-alphanumeric ASCII + any Unicode P* category. */
+function isPunct(ch) {
+    const cp = ch.codePointAt(0);
+    if ((cp >= 33 && cp <= 47) ||
+        (cp >= 58 && cp <= 64) ||
+        (cp >= 91 && cp <= 96) ||
+        (cp >= 123 && cp <= 126)) {
+        return true;
+    }
+    return /\p{P}/u.test(ch);
+}
+export class WordPieceTokenizer {
+    vocab;
+    unkId;
+    clsId;
+    sepId;
+    padId;
+    constructor(vocabText) {
+        this.vocab = new Map();
+        const lines = vocabText.split(/\r?\n/);
+        let i = 0;
+        for (const line of lines) {
+            // vocab.txt is one token per line; the line number is the id. Do not trim
+            // away a token that is itself whitespace-like — but blank trailing lines
+            // (from the split) must not consume ids, so stop at the first empty tail.
+            const tok = line.replace(/\r$/, "");
+            if (tok === "" && i >= lines.length - 2)
+                break;
+            this.vocab.set(tok, i);
+            i++;
+        }
+        this.unkId = this.vocab.get(SPECIAL.unk);
+        this.clsId = this.vocab.get(SPECIAL.cls);
+        this.sepId = this.vocab.get(SPECIAL.sep);
+        this.padId = this.vocab.get(SPECIAL.pad) ?? 0;
+        if (this.unkId === undefined || this.clsId === undefined || this.sepId === undefined) {
+            throw new Error("vocab.txt is missing required special tokens ([UNK]/[CLS]/[SEP]).");
+        }
+    }
+    /** Normalize: strip control chars, pad CJK, lowercase, remove accents (NFD/Mn). */
+    normalize(text) {
+        let out = "";
+        for (const ch of text) {
+            const cp = ch.codePointAt(0);
+            if (cp === 0 || cp === 0xfffd)
+                continue; // invalid
+            if (cp !== 9 && cp !== 10 && cp !== 13 && cp < 32)
+                continue; // control chars
+            out += isCjk(cp) ? ` ${ch} ` : ch;
+        }
+        return out
+            .toLowerCase()
+            .normalize("NFD")
+            .replace(/\p{Mn}/gu, "");
+    }
+    /** Whitespace + punctuation split into basic tokens (BERT BasicTokenizer). */
+    basicTokenize(text) {
+        const tokens = [];
+        for (const word of this.normalize(text).split(/\s+/)) {
+            if (!word)
+                continue;
+            let cur = "";
+            for (const ch of word) {
+                if (isPunct(ch)) {
+                    if (cur) {
+                        tokens.push(cur);
+                        cur = "";
+                    }
+                    tokens.push(ch);
+                }
+                else {
+                    cur += ch;
+                }
+            }
+            if (cur)
+                tokens.push(cur);
+        }
+        return tokens;
+    }
+    /** Greedy longest-match-first WordPiece split of a single word. */
+    wordpiece(word) {
+        if (word.length > 200)
+            return [SPECIAL.unk]; // BERT max_input_chars_per_word
+        const out = [];
+        let start = 0;
+        while (start < word.length) {
+            let end = word.length;
+            let found = null;
+            while (start < end) {
+                const sub = (start > 0 ? "##" : "") + word.slice(start, end);
+                if (this.vocab.has(sub)) {
+                    found = sub;
+                    break;
+                }
+                end--;
+            }
+            if (found === null)
+                return [SPECIAL.unk]; // any unmatchable piece → whole word UNK
+            out.push(found);
+            start = end;
+        }
+        return out;
+    }
+    /**
+     * Encode text into `[CLS] … [SEP]` token ids (truncated to maxLen) plus an
+     * all-ones attention mask. We embed one text at a time (batch 1), so no
+     * padding is needed.
+     */
+    encode(text, maxLen = 256) {
+        const pieceIds = [];
+        for (const tok of this.basicTokenize(text)) {
+            for (const piece of this.wordpiece(tok)) {
+                pieceIds.push(this.vocab.get(piece) ?? this.unkId);
+            }
+        }
+        const body = pieceIds.slice(0, Math.max(0, maxLen - 2));
+        const inputIds = [this.clsId, ...body, this.sepId];
+        return { inputIds, attentionMask: inputIds.map(() => 1) };
+    }
+}
+//# sourceMappingURL=tokenizer.js.map

package/dist/tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAW,CAAC;AAEpF,iFAAiF;AACjF,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,CACL,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;QAC9B,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;QAC9B,CAAC,EAAE,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,CAAC;QAChC,CAAC,EAAE,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,CAAC;QAChC,CAAC,EAAE,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,CAAC;QAChC,CAAC,EAAE,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,CAAC;QAChC,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;QAC9B,CAAC,EAAE,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,CAAC,CACjC,CAAC;AACJ,CAAC;AAED,oFAAoF;AACpF,SAAS,OAAO,CAAC,EAAU;IACzB,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC;IAC9B,IACE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACtB,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACtB,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACtB,CAAC,EAAE,IAAI,GAAG,IAAI,EAAE,IAAI,GAAG,CAAC,EACxB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,OAAO,kBAAkB;IACrB,KAAK,CAAsB;IAC1B,KAAK,CAAS;IACd,KAAK,CAAS;IACd,KAAK,CAAS;IACd,KAAK,CAAS;IAEvB,YAAY,SAAiB;QAC3B,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,0EAA0E;YAC1E,yEAAyE;YACzE,0EAA0E;YAC1E,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACpC,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM;YAC/C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YACvB,CAAC,EAAE,CAAC;QACN,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAE,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAE,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAE,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YACrF,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;QACvF,CAAC;IACH,CAAC;IAED,mFAAmF;IAC3E,SAAS,CAAC,IAAY;QAC5B,IAAI,GAAG,GAAG,EAAE,CAAC;QACb,KAAK,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;YACtB,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC;YAC9B,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,MAAM;gBAAE,SAAS,CAAC,UAAU;YACnD,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE;gBAAE,SAAS,CAAC,gBAAgB;YAC7E,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACpC,CAAC;QACD,OAAO,GAAG;aACP,WAAW,EAAE;aACb,SAAS,CAAC,KAAK,CAAC;aAChB,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC7B,CAAC;IAED,8EAA8E;IACtE,aAAa,CAAC,IAAY;QAChC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YACrD,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,IAAI,GAAG,GAAG,EAAE,CAAC;YACb,KAAK,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;gBACtB,IAAI,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;oBAChB,IAAI,GAAG,EAAE,CAAC;wBACR,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACjB,GAAG,GAAG,EAAE,CAAC;oBACX,CAAC;oBACD,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAClB,CAAC;qBAAM,CAAC;oBACN,GAAG,IAAI,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC;YACD,IAAI,GAAG;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,mEAAmE;IAC3D,SAAS,CAAC,IAAY;QAC5B,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG;YAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,gCAAgC;QAC7E,MAAM,GAAG,GAAa,EAAE,CAAC;QACzB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC3B,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC;YACtB,IAAI,KAAK,GAAkB,IAAI,CAAC;YAChC,OAAO,KAAK,GAAG,GAAG,EAAE,CAAC;gBACnB,MAAM,GAAG,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;gBAC7D,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACxB,KAAK,GAAG,GAAG,CAAC;oBACZ,MAAM;gBACR,CAAC;gBACD,GAAG,EAAE,CAAC;YACR,CAAC;YACD,IAAI,KAAK,KAAK,IAAI;gBAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,yCAAyC;YACnF,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,KAAK,GAAG,GAAG,CAAC;QACd,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,IAAY,EAAE,MAAM,GAAG,GAAG;QAC/B,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3C,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;QACD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QACnD,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5D,CAAC;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@abhishekmcp/notes",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "MCP server for local markdown notes — ranked search, tags, todos, and a wiki-link knowledge graph, from any MCP client.",
   "type": "module",
   "bin": {
@@ -44,6 +44,7 @@
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.29.0",
     "minisearch": "^7.1.0",
+    "onnxruntime-web": "^1.27.0",
     "zod": "^3.23.8"
   },
   "devDependencies": {