grepmax 0.26.4 → 0.26.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"name": "grepmax",
|
|
11
11
|
"source": "./plugins/grepmax",
|
|
12
12
|
"description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
|
|
13
|
-
"version": "0.26.
|
|
13
|
+
"version": "0.26.5",
|
|
14
14
|
"author": {
|
|
15
15
|
"name": "Robert Owens",
|
|
16
16
|
"email": "robowens@me.com"
|
|
@@ -43,6 +43,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
43
43
|
};
|
|
44
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
45
|
exports.TreeSitterChunker = void 0;
|
|
46
|
+
exports.repairLoneSurrogates = repairLoneSurrogates;
|
|
46
47
|
exports.formatChunkText = formatChunkText;
|
|
47
48
|
exports.buildAnchorChunk = buildAnchorChunk;
|
|
48
49
|
const fs = __importStar(require("node:fs"));
|
|
@@ -71,6 +72,19 @@ function resolveTreeSitterWasmLocator() {
|
|
|
71
72
|
return path.join(__dirname, "..", "..", "..", "node_modules", "web-tree-sitter", "tree-sitter.wasm");
|
|
72
73
|
}
|
|
73
74
|
}
|
|
75
|
+
// Char-stride chunk splitting can slice through a UTF-16 surrogate pair
|
|
76
|
+
// (e.g. PDF-extracted math italics like 𝑃), leaving a lone surrogate. That
|
|
77
|
+
// poisons the whole embed batch — the HF fast tokenizer rejects non-well-
|
|
78
|
+
// formed input — and puts invalid UTF-8 into the vector table's content
|
|
79
|
+
// column. Repair to U+FFFD at the chunk emit boundary. (Regex stand-in for
|
|
80
|
+
// String.prototype.toWellFormed, which the ES6 lib target doesn't declare.)
|
|
81
|
+
const LONE_SURROGATE_TEST = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/;
|
|
82
|
+
const LONE_SURROGATE_ALL = new RegExp(LONE_SURROGATE_TEST.source, "g");
|
|
83
|
+
function repairLoneSurrogates(s) {
|
|
84
|
+
if (!LONE_SURROGATE_TEST.test(s))
|
|
85
|
+
return s;
|
|
86
|
+
return s.replace(LONE_SURROGATE_ALL, "�");
|
|
87
|
+
}
|
|
74
88
|
function isDocLikeFile(filePath) {
|
|
75
89
|
const ext = path.extname(filePath).toLowerCase();
|
|
76
90
|
return [".md", ".mdx", ".txt", ".rst", ".adoc"].includes(ext);
|
|
@@ -249,6 +263,9 @@ class TreeSitterChunker {
|
|
|
249
263
|
}
|
|
250
264
|
// Split chunks if too big
|
|
251
265
|
result.chunks = result.chunks.flatMap((c) => this.splitIfTooBig(c));
|
|
266
|
+
for (const c of result.chunks) {
|
|
267
|
+
c.content = repairLoneSurrogates(c.content);
|
|
268
|
+
}
|
|
252
269
|
// Post-process: extract property function names from chunks.
|
|
253
270
|
// Catches resolver patterns like `posTableTurnReport: withAuth(scope, async (args) => {`
|
|
254
271
|
const PROP_FN_RE = /^\s*(\w+)\s*:\s*(?:async\s+)?(?:function\b|\(|[A-Za-z_]\w*\()/gm;
|
|
@@ -13,6 +13,7 @@ GPU operations.
|
|
|
13
13
|
import asyncio
|
|
14
14
|
import logging
|
|
15
15
|
import os
|
|
16
|
+
import re
|
|
16
17
|
import signal
|
|
17
18
|
import socket
|
|
18
19
|
import time
|
|
@@ -90,12 +91,22 @@ def is_port_in_use(port: int) -> bool:
|
|
|
90
91
|
return s.connect_ex(("127.0.0.1", port)) == 0
|
|
91
92
|
|
|
92
93
|
|
|
94
|
+
# JSON can carry lone UTF-16 surrogates (a client slicing mid-codepoint);
|
|
95
|
+
# the HF fast tokenizer rejects them with a TypeError that fails the whole
|
|
96
|
+
# batch. Replace with U+FFFD so one bad text can't take down a batch.
|
|
97
|
+
_LONE_SURROGATE = re.compile("[\ud800-\udfff]")
|
|
98
|
+
|
|
99
|
+
|
|
93
100
|
def embed_texts(texts: list[str]) -> mx.array:
|
|
94
101
|
"""Tokenize, forward pass, L2 normalize.
|
|
95
102
|
|
|
96
103
|
mlx_embeddings model already does mean pooling internally —
|
|
97
104
|
last_hidden_state is (batch, dim), not (batch, seq, dim).
|
|
98
105
|
"""
|
|
106
|
+
texts = [
|
|
107
|
+
_LONE_SURROGATE.sub("�", t) if _LONE_SURROGATE.search(t) else t
|
|
108
|
+
for t in texts
|
|
109
|
+
]
|
|
99
110
|
encoded = tokenizer(
|
|
100
111
|
texts, padding=True, truncation=True, max_length=256, return_tensors="np"
|
|
101
112
|
)
|
package/package.json
CHANGED