embarsy-qdrant-mcp 0.1.0 → 0.1.2

package/dist/bin/index.js

@@ -60,7 +60,8 @@ async function main() {
     const started = Date.now();
     const result = await indexRepo(rootAbs, cfg, { onProgress: (m) => process.stdout.write(m + "\n") });
     const secs = ((Date.now() - started) / 1000).toFixed(1);
-    process.stdout.write(`\nDone in ${secs}s — ${result.indexed} indexed, ${result.skipped} unchanged, ${result.removed} removed; ${result.chunks} chunks upserted into "${cfg.collection}".\n`);
+    const minified = result.skippedMinified > 0 ? `, ${result.skippedMinified} skipped (minified)` : "";
+    process.stdout.write(`\nDone in ${secs}s — ${result.indexed} indexed, ${result.skipped} unchanged${minified}, ${result.removed} removed; ${result.chunks} chunks upserted into "${cfg.collection}".\n`);
 }
 main().catch((err) => {
     process.stderr.write(`\nembarsy-index failed: ${err instanceof Error ? err.message : String(err)}\n`);

package/dist/chunker.js

@@ -1,61 +1,51 @@
 /**
- * Line-oriented chunking with a character budget and overlap. Chunks prefer to start at a
- * "definition-ish" line (function/class/etc.) when the current chunk is already sizeable, so
- * chunks tend to align with logical code boundaries. Pure-JS, no native parser.
+ * Character-window chunking with a HARD size cap, so no single chunk can ever exceed the
+ * embedding model's context — not even a minified/bundled file that is one enormous line.
+ *
+ * Windows prefer to end at a newline (cleaner code boundaries), but a line longer than the cap
+ * is split by character count instead of being embedded whole. The old line-based chunker sent
+ * a one-line 800 KB JSON as a single chunk, which tokenised past 32k and crashed llama-server.
  */
-const BOUNDARY = /^\s*(export\s+)?(async\s+)?(function|class|struct|enum|interface|trait|impl|def|func|fn|type|module|namespace|public|private|protected|static)\b/;
 export function chunkFile(content, cfg) {
-    const lines = content.split(/\r?\n/);
-    const chunks = [];
     const maxChars = Math.max(400, cfg.chunkMaxChars);
-    const overlapChars = Math.max(0, Math.min(cfg.chunkOverlapChars, maxChars - 100));
-    let startIdx = 0; // 0-based line index where the current chunk starts
-    let curChars = 0;
-    let i = 0;
-    const flush = (endIdxExclusive) => {
-        if (endIdxExclusive <= startIdx)
-            return;
-        const slice = lines.slice(startIdx, endIdxExclusive);
-        const text = slice.join("\n").trim();
-        if (text.length > 0) {
-            chunks.push({ startLine: startIdx + 1, endLine: endIdxExclusive, text: slice.join("\n") });
+    const overlap = Math.max(0, Math.min(cfg.chunkOverlapChars, Math.floor(maxChars / 2)));
+    if (content.trim().length === 0)
+        return [];
+    // Line-start offsets → 1-based line number for any character offset (binary search).
+    const lineStarts = [0];
+    for (let i = 0; i < content.length; i++) {
+        if (content.charCodeAt(i) === 10 /* \n */)
+            lineStarts.push(i + 1);
+    }
+    const lineAt = (offset) => {
+        let lo = 0, hi = lineStarts.length - 1;
+        while (lo < hi) {
+            const mid = (lo + hi + 1) >> 1;
+            if (lineStarts[mid] <= offset)
+                lo = mid;
+            else
+                hi = mid - 1;
         }
+        return lo + 1;
     };
-    while (i < lines.length) {
-        const line = lines[i] ?? "";
-        const lineChars = line.length + 1;
-        // Break BEFORE a definition line if the current chunk already has real content.
-        if (i > startIdx && curChars >= maxChars * 0.5 && BOUNDARY.test(line)) {
-            flush(i);
-            startIdx = backfillOverlap(lines, i, overlapChars);
-            curChars = charsBetween(lines, startIdx, i);
+    const chunks = [];
+    let pos = 0;
+    while (pos < content.length) {
+        let end = Math.min(content.length, pos + maxChars);
+        if (end < content.length) {
+            // Snap to the last newline in the back half of the window for a tidy cut; a window with
+            // no newline (one giant line) falls through and is hard-cut at maxChars.
+            const nl = content.lastIndexOf("\n", end);
+            if (nl > pos + Math.floor(maxChars / 2))
+                end = nl;
         }
-        curChars += lineChars;
-        i++;
-        if (curChars >= maxChars) {
-            flush(i);
-            startIdx = backfillOverlap(lines, i, overlapChars);
-            curChars = charsBetween(lines, startIdx, i);
+        const text = content.slice(pos, end);
+        if (text.trim().length > 0) {
+            chunks.push({ startLine: lineAt(pos), endLine: lineAt(Math.max(pos, end - 1)), text });
         }
+        if (end >= content.length)
+            break;
+        pos = Math.max(pos + 1, end - overlap);
     }
-    flush(lines.length);
     return chunks;
 }
-/** Walk back from `idx` to include ~overlapChars of trailing context; returns new start index. */
-function backfillOverlap(lines, idx, overlapChars) {
-    if (overlapChars <= 0)
-        return idx;
-    let chars = 0;
-    let j = idx;
-    while (j > 0 && chars < overlapChars) {
-        j--;
-        chars += (lines[j]?.length ?? 0) + 1;
-    }
-    return j;
-}
-function charsBetween(lines, from, to) {
-    let c = 0;
-    for (let k = from; k < to; k++)
-        c += (lines[k]?.length ?? 0) + 1;
-    return c;
-}

package/dist/config.js

@@ -17,8 +17,9 @@ export function loadConfig(overrides = {}) {
         collection: env.QDRANT_COLLECTION_NAME ?? env.QDRANT_COLLECTION ?? "",
         chunkMaxChars: intEnv(env.EMBARSY_CHUNK_CHARS, 1500),
         chunkOverlapChars: intEnv(env.EMBARSY_CHUNK_OVERLAP, 200),
-        embedBatch: intEnv(env.EMBARSY_EMBED_BATCH, 64),
+        embedBatch: intEnv(env.EMBARSY_EMBED_BATCH, 32),
         maxFileBytes: intEnv(env.EMBARSY_MAX_FILE_BYTES, 1_000_000),
+        maxLineChars: intEnv(env.EMBARSY_MAX_LINE_CHARS, 5000),
         ...overrides,
     };
 }

package/dist/embeddings.js

@@ -4,20 +4,27 @@ import { requestJSON } from "./http.js";
  * This is the piece other tools get wrong: we send a bearer API key and use the standard
  * POST /v1/embeddings shape, so Embarsy authenticates the call and counts it in Monitoring.
  */
+/** Absolute per-input cap (defense in depth on top of chunking) so no single text can overflow
+ *  the model's context and crash the embedding backend, whatever produced it. */
+const MAX_EMBED_CHARS = 8000;
 export async function embedBatch(texts, cfg) {
     if (texts.length === 0)
         return [];
+    const input = texts.map((t) => (t.length > MAX_EMBED_CHARS ? t.slice(0, MAX_EMBED_CHARS) : t));
     const json = await requestJSON(`${cfg.openaiBaseUrl}/embeddings`, {
         method: "POST",
         headers: {
             "Content-Type": "application/json",
             Authorization: `Bearer ${cfg.openaiApiKey}`,
         },
-        body: JSON.stringify({ model: cfg.embeddingModel, input: texts }),
-    }, { label: "embeddings" });
+        body: JSON.stringify({ model: cfg.embeddingModel, input }),
+    }, 
+    // Extra retries: if the embedding backend hiccups (e.g. Ollama restarts llama-server), ride
+    // out the restart instead of aborting the whole index.
+    { label: "embeddings", retries: 4 });
     const data = json?.data;
-    if (!Array.isArray(data) || data.length !== texts.length) {
-        throw new Error(`Unexpected embeddings response (got ${Array.isArray(data) ? data.length : "no"} vectors for ${texts.length} inputs).`);
+    if (!Array.isArray(data) || data.length !== input.length) {
+        throw new Error(`Unexpected embeddings response (got ${Array.isArray(data) ? data.length : "no"} vectors for ${input.length} inputs).`);
     }
     // Respect the `index` field so ordering is guaranteed to match the input.
     const ordered = [...data].sort((a, b) => (a.index ?? 0) - (b.index ?? 0));

package/dist/http.js

@@ -26,7 +26,7 @@ export async function requestJSON(url, init, opts = {}) {
                 throw e;
         }
         if (attempt < retries)
-            await sleep(300 * (attempt + 1));
+            await sleep(Math.min(5000, 400 * 2 ** attempt)); // exp backoff, capped 5s
     }
     throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
 }

package/dist/indexer.js

@@ -14,6 +14,21 @@ function looksBinary(s) {
     }
     return false;
 }
+/** Longest line length — a line far longer than any hand-written source line flags a minified
+ *  / bundled / single-blob file (a poor embedding candidate that also stresses the model). */
+function maxLineLength(s) {
+    let max = 0, cur = 0;
+    for (let i = 0; i < s.length; i++) {
+        if (s.charCodeAt(i) === 10) {
+            if (cur > max)
+                max = cur;
+            cur = 0;
+        }
+        else
+            cur++;
+    }
+    return cur > max ? cur : max;
+}
 /** Full/incremental index of a directory into the configured Qdrant collection. */
 export async function indexRepo(root, cfg, opts = {}) {
     const log = opts.onProgress ?? (() => { });
@@ -23,7 +38,7 @@ export async function indexRepo(root, cfg, opts = {}) {
     log(`Found ${files.length} indexable files in ${rootAbs}`);
     const existing = await scrollFileHashes(cfg);
     const seen = new Set();
-    const result = { files: files.length, indexed: 0, skipped: 0, removed: 0, chunks: 0 };
+    const result = { files: files.length, indexed: 0, skipped: 0, removed: 0, chunks: 0, skippedMinified: 0 };
     // Pending chunk buffer, flushed in embedding batches across files.
     let pending = [];
     const flush = async () => {
@@ -50,6 +65,11 @@ export async function indexRepo(root, cfg, opts = {}) {
         }
         if (looksBinary(content))
             continue;
+        if (maxLineLength(content) > cfg.maxLineChars) {
+            result.skippedMinified++;
+            log(`  skipped (minified/generated): ${file.rel}`);
+            continue;
+        }
         const hash = sha256(content);
         if (existing.get(file.rel) === hash) {
             result.skipped++;

package/dist/qdrant.js

@@ -20,7 +20,14 @@ export async function ensureCollection(cfg) {
     await requestJSON(base(cfg), {
         method: "PUT",
         headers: headers(cfg),
-        body: JSON.stringify({ vectors: { size: cfg.embeddingDimension, distance: "Cosine" } }),
+        body: JSON.stringify({
+            // Laptop-friendly storage: full-precision originals live on disk and are only
+            // read to rescore the top candidates; searches run on an in-RAM int8 copy
+            // (~4x smaller, SIMD-accelerated) with no practical recall loss for cosine
+            // text embeddings.
+            vectors: { size: cfg.embeddingDimension, distance: "Cosine", on_disk: true },
+            quantization_config: { scalar: { type: "int8", quantile: 0.99, always_ram: true } },
+        }),
     }, { label: "create collection", retries: 0 });
     // Payload index on file_path enables fast delete-by-file for incremental reindex.
     try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "embarsy-qdrant-mcp",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Codebase-indexing MCP server for Embarsy — semantic code search over a local Qdrant + OpenAI-compatible embeddings stack. Routes embeddings and Qdrant through Embarsy's proxy so auth and Monitoring counters just work.",
   "type": "module",
   "bin": {