maestro-flow 0.5.41 → 0.5.43

package/dashboard/dist-server/dashboard/src/server/wiki/embedding.js

@@ -11,6 +11,19 @@ import { join } from 'node:path';
 import { homedir } from 'node:os';
 import { createHash } from 'node:crypto';
 import { readFileSync, writeFileSync, existsSync, mkdirSync, unlinkSync, renameSync } from 'node:fs';
+let _zvecModule;
+async function getZvec() {
+    if (_zvecModule !== undefined)
+        return _zvecModule;
+    try {
+        _zvecModule = await import('@zvec/zvec');
+        return _zvecModule;
+    }
+    catch {
+        _zvecModule = null;
+        return null;
+    }
+}
 const API_CONFIG_PATH = join(homedir(), '.maestro', 'api-embedding.json');
 let _apiConfig;
 export function loadEmbeddingApiConfig() {
@@ -265,17 +278,18 @@ export async function detectDevice() {
         return _detectedConfig;
     const backends = await listBackends();
     const hasGpu = backends.some(b => b.name === 'dml' || b.name === 'cuda');
-    // For small models (all-MiniLM-L6-v2, 22M params), CPU is consistently faster
-    // due to CPU↔GPU data transfer overhead exceeding compute savings.
-    // GPU only wins for models >100M params or batch sizes >500.
+    const envDevice = process.env.MAESTRO_EMBEDDING_DEVICE;
+    const envBatch = process.env.MAESTRO_EMBEDDING_BATCH_SIZE;
+    const useGpu = envDevice === 'cpu' ? false : hasGpu;
     _detectedConfig = {
-        device: 'cpu',
-        dtype: 'fp32',
-        batchSize: 32,
+        device: useGpu ? 'gpu' : 'cpu',
+        dtype: useGpu ? 'fp16' : 'fp32',
+        batchSize: useGpu ? 128 : (hasGpu ? 64 : 32),
     };
-    if (hasGpu) {
-        // Store GPU availability for future large-model support
-        _detectedConfig.batchSize = 64;
+    if (envBatch) {
+        const parsed = parseInt(envBatch, 10);
+        if (parsed > 0)
+            _detectedConfig.batchSize = parsed;
     }
     return _detectedConfig;
 }
@@ -290,13 +304,21 @@ export async function getHardwareInfo() {
     const backends = await listBackends();
     const hasGpu = backends.some(b => b.name === 'dml' || b.name === 'cuda');
     const config = await detectDevice();
+    let reason;
+    if (!hasGpu) {
+        reason = 'CPU only — no GPU backend detected';
+    }
+    else if (config.device === 'gpu') {
+        reason = 'GPU auto-selected (DML/CUDA detected) — set MAESTRO_EMBEDDING_DEVICE=cpu to force CPU';
+    }
+    else {
+        reason = 'GPU available but CPU forced via MAESTRO_EMBEDDING_DEVICE=cpu';
+    }
     return {
         backends,
         gpuAvailable: hasGpu,
         selectedDevice: config,
-        reason: hasGpu
-            ? 'GPU available (DML/CUDA) but CPU selected — small model (22M params) runs faster on CPU due to transfer overhead'
-            : 'CPU only — no GPU backend detected',
+        reason,
     };
 }
 // ---------------------------------------------------------------------------
@@ -405,9 +427,55 @@ export async function embedQuery(query) {
     return new Float32Array(output.data);
 }
 // ---------------------------------------------------------------------------
-// Vector search (flat cosine — no index structure needed for <10K docs)
+// Vector search — zvec backend (default) with flat cosine fallback
 // ---------------------------------------------------------------------------
+const ZVEC_DIR = 'embedding.zvec';
 export function vectorSearch(queryVector, index, limit) {
+    // Feature flag: force flat cosine scan (bypass zvec)
+    if (process.env.MAESTRO_EMBEDDING_FLAT_SCAN) {
+        return flatCosineSearch(queryVector, index, limit);
+    }
+    // Sync path — zvec requires async setup, so use flat scan here.
+    // For zvec-accelerated search, callers should use vectorSearchZvec.
+    return flatCosineSearch(queryVector, index, limit);
+}
+/**
+ * Async vector search using zvec collection.
+ * Falls back to flat cosine scan when zvec is unavailable or feature flag is set.
+ */
+export async function vectorSearchZvec(queryVector, dir, limit) {
+    if (process.env.MAESTRO_EMBEDDING_FLAT_SCAN) {
+        return []; // caller handles fallback via sync vectorSearch
+    }
+    const zvec = await getZvec();
+    if (!zvec)
+        return []; // zvec not installed, caller handles fallback
+    const collectionPath = join(dir, ZVEC_DIR);
+    if (!existsSync(collectionPath))
+        return []; // no zvec collection yet
+    try {
+        const collection = zvec.ZVecOpen(collectionPath, { readOnly: true });
+        try {
+            const docs = collection.querySync({
+                fieldName: 'embedding',
+                vector: queryVector,
+                topk: limit,
+                outputFields: ['docId'],
+            });
+            return docs.map(d => ({
+                docId: d.fields.docId ?? d.id,
+                score: 1 - d.score,
+            }));
+        }
+        finally {
+            collection.closeSync();
+        }
+    }
+    catch {
+        return []; // zvec query failed, caller handles fallback
+    }
+}
+function flatCosineSearch(queryVector, index, limit) {
     const scored = [];
     for (let i = 0; i < index.docIds.length; i++) {
         const sim = cosineSimilarity(queryVector, index.vectors[i]);
@@ -418,7 +486,7 @@ export function vectorSearch(queryVector, index, limit) {
     return scored.slice(0, limit);
 }
 // ---------------------------------------------------------------------------
-// Persistence — SQLite binary BLOB (primary) with JSON fallback for migration
+// Persistence — zvec collection (primary) + binary fallback + legacy migration
 // ---------------------------------------------------------------------------
 const SQLITE_FILE = 'embedding-index.db';
 import { createRequire } from 'node:module';
@@ -426,6 +494,7 @@ const _require = createRequire(import.meta.url);
 const BINARY_FILE = 'embedding-index.bin';
 export function saveEmbeddingIndex(index, dir) {
     mkdirSync(dir, { recursive: true });
+    // --- Binary format (kept for backward compat and feature flag fallback) ---
     const dim = index.dimension;
     const n = index.docIds.length;
     const docIdsJson = JSON.stringify(index.docIds);
@@ -438,6 +507,7 @@ export function saveEmbeddingIndex(index, dir) {
         deviceUsed: index.deviceUsed,
         buildTimeMs: index.buildTimeMs,
         contentHashes: index.contentHashes,
+        chunkDocIds: index.chunkDocIds,
     });
     const metaBytes = Buffer.from(metaJson, 'utf-8');
     // Format: [metaLen:4][meta][docIdsLen:4][docIds][packedVectors:n*dim*4]
@@ -462,6 +532,10 @@ export function saveEmbeddingIndex(index, dir) {
     const tmpPath = join(dir, BINARY_FILE + '.tmp');
     writeFileSync(tmpPath, buf);
     renameSync(tmpPath, join(dir, BINARY_FILE));
+    // --- zvec collection save (async, best-effort) ---
+    saveZvecIndex(index, dir).catch(() => {
+        // zvec save failed — binary format still available as fallback
+    });
     // Remove legacy files
     for (const f of [CACHE_FILE, SQLITE_FILE, SQLITE_FILE + '-shm', SQLITE_FILE + '-wal', SQLITE_FILE + '-journal']) {
         try {
@@ -471,8 +545,95 @@ export function saveEmbeddingIndex(index, dir) {
         catch { /* ignore */ }
     }
 }
+/**
+ * Save embedding index to zvec collection format.
+ * Creates/replaces a zvec collection at `dir/embedding.zvec`.
+ */
+async function saveZvecIndex(index, dir) {
+    const zvec = await getZvec();
+    if (!zvec)
+        return;
+    const collectionPath = join(dir, ZVEC_DIR);
+    const dim = index.dimension;
+    // Remove existing collection directory if present
+    if (existsSync(collectionPath)) {
+        try {
+            const existing = zvec.ZVecOpen(collectionPath);
+            existing.destroySync();
+        }
+        catch {
+            // If we can't open it, try to remove the directory manually
+            try {
+                const { rmSync } = await import('node:fs');
+                rmSync(collectionPath, { recursive: true, force: true });
+            }
+            catch { /* ignore */ }
+        }
+    }
+    const schema = new zvec.ZVecCollectionSchema({
+        name: 'embedding',
+        vectors: {
+            name: 'embedding',
+            dataType: zvec.ZVecDataType.VECTOR_FP32,
+            dimension: dim,
+            indexParams: {
+                indexType: zvec.ZVecIndexType.FLAT,
+                metricType: zvec.ZVecMetricType.COSINE,
+            },
+        },
+        fields: [
+            { name: 'docId', dataType: zvec.ZVecDataType.STRING },
+        ],
+    });
+    const collection = zvec.ZVecCreateAndOpen(collectionPath, schema);
+    try {
+        // Batch upsert all vectors
+        const BATCH_SIZE = 500;
+        for (let i = 0; i < index.docIds.length; i += BATCH_SIZE) {
+            const batch = [];
+            const end = Math.min(i + BATCH_SIZE, index.docIds.length);
+            for (let j = i; j < end; j++) {
+                batch.push({
+                    id: index.docIds[j],
+                    vectors: { embedding: index.vectors[j] },
+                    fields: { docId: index.docIds[j] },
+                });
+            }
+            collection.upsertSync(batch);
+        }
+        // Save metadata as JSON sidecar (zvec doesn't store arbitrary metadata)
+        const metaSidecar = {
+            modelId: index.modelId,
+            dimension: dim,
+            builtAt: index.builtAt,
+            deviceUsed: index.deviceUsed,
+            buildTimeMs: index.buildTimeMs,
+            contentHashes: index.contentHashes,
+            chunkDocIds: index.chunkDocIds,
+            docIds: index.docIds,
+        };
+        writeFileSync(join(dir, ZVEC_DIR + '.meta.json'), JSON.stringify(metaSidecar));
+    }
+    finally {
+        collection.closeSync();
+    }
+}
 export function loadEmbeddingIndex(dir) {
-    // Primary: packed binary
+    // Primary: zvec collection + metadata sidecar
+    const zvecMetaPath = join(dir, ZVEC_DIR + '.meta.json');
+    const zvecCollPath = join(dir, ZVEC_DIR);
+    if (existsSync(zvecMetaPath) && existsSync(zvecCollPath)) {
+        try {
+            return loadFromZvecMeta(zvecMetaPath, zvecCollPath);
+        }
+        catch (e) {
+            if (process.env.DEBUG || process.env.MAESTRO_DEBUG) {
+                console.warn(`[embedding] zvec index load failed, falling back: ${e instanceof Error ? e.message : e}`);
+            }
+            // Fall through to binary
+        }
+    }
+    // Fallback: packed binary
     const binPath = join(dir, BINARY_FILE);
     if (existsSync(binPath)) {
         try {
@@ -509,6 +670,62 @@ export function loadEmbeddingIndex(dir) {
     }
     return null;
 }
+/**
+ * Load EmbeddingIndex from zvec metadata sidecar.
+ * The sidecar stores everything needed to reconstruct the in-memory index;
+ * the actual zvec collection is used for vectorSearchZvec queries.
+ */
+function loadFromZvecMeta(metaPath, _collectionPath) {
+    const meta = JSON.parse(readFileSync(metaPath, 'utf-8'));
+    // Use cached zvec module if available, otherwise try sync require
+    let zvec = _zvecModule ?? null;
+    if (!zvec) {
+        try {
+            zvec = _require('@zvec/zvec');
+            _zvecModule = zvec;
+        }
+        catch {
+            throw new Error('zvec not available for loading collection');
+        }
+    }
+    const collection = zvec.ZVecOpen(_collectionPath, { readOnly: true });
+    try {
+        const vectors = new Array(meta.docIds.length);
+        // Fetch vectors in batches by ID
+        const BATCH_SIZE = 500;
+        for (let i = 0; i < meta.docIds.length; i += BATCH_SIZE) {
+            const batchIds = meta.docIds.slice(i, Math.min(i + BATCH_SIZE, meta.docIds.length));
+            const fetched = collection.fetchSync({ ids: batchIds, includeVector: true, outputFields: [] });
+            const fetchedMap = Array.isArray(fetched)
+                ? Object.fromEntries(fetched.map((d) => [d.id, d]))
+                : fetched;
+            for (let j = 0; j < batchIds.length; j++) {
+                const doc = fetchedMap[batchIds[j]];
+                if (doc?.vectors?.embedding) {
+                    const v = doc.vectors.embedding;
+                    vectors[i + j] = v instanceof Float32Array ? v : new Float32Array(v);
+                }
+                else {
+                    vectors[i + j] = new Float32Array(meta.dimension);
+                }
+            }
+        }
+        return {
+            modelId: meta.modelId,
+            dimension: meta.dimension,
+            docIds: meta.docIds,
+            vectors,
+            contentHashes: meta.contentHashes,
+            chunkDocIds: meta.chunkDocIds,
+            builtAt: meta.builtAt,
+            deviceUsed: meta.deviceUsed,
+            buildTimeMs: meta.buildTimeMs,
+        };
+    }
+    finally {
+        collection.closeSync();
+    }
+}
 function loadFromBinary(filePath) {
     const raw = readFileSync(filePath);
     let offset = 0;
@@ -543,6 +760,7 @@ function loadFromBinary(filePath) {
         docIds,
         vectors,
         contentHashes: meta.contentHashes,
+        chunkDocIds: meta.chunkDocIds,
         builtAt: meta.builtAt,
         deviceUsed: meta.deviceUsed,
         buildTimeMs: meta.buildTimeMs,
@@ -590,68 +808,225 @@ function loadFromLegacyJson(filePath) {
     };
 }
 export function hashDocContent(d) {
-    const text = [d.title, d.summary, d.tags.join(','), d.body?.slice(0, 500) ?? ''].join('|');
+    const text = [d.title, d.summary, d.tags.join(','), d.body ?? ''].join('|');
     return createHash('md5').update(text).digest('hex');
 }
+/**
+ * Extract meaningful content from a markdown body: first paragraph + heading lines,
+ * up to `maxLen` characters.
+ */
+function extractMeaningfulContent(body, maxLen) {
+    const lines = body.split('\n');
+    const parts = [];
+    let len = 0;
+    let firstParaDone = false;
+    for (const line of lines) {
+        if (len >= maxLen)
+            break;
+        const trimmed = line.trim();
+        // Always include heading lines
+        if (/^#{1,3}\s/.test(trimmed)) {
+            parts.push(trimmed);
+            len += trimmed.length + 1;
+            continue;
+        }
+        // Include non-empty lines until we hit the first blank line (first paragraph)
+        if (!firstParaDone) {
+            if (trimmed === '') {
+                if (parts.length > 0)
+                    firstParaDone = true;
+                continue;
+            }
+            parts.push(trimmed);
+            len += trimmed.length + 1;
+        }
+    }
+    const result = parts.join('\n');
+    return result.length > maxLen ? result.slice(0, maxLen) : result;
+}
 function docToText(d) {
-    const parts = [d.title];
+    const parts = [`title: ${d.title}`];
     if (d.summary)
-        parts.push(d.summary);
+        parts.push(`summary: ${d.summary}`);
     if (d.tags.length > 0)
-        parts.push(d.tags.join(' '));
-    if (d.body)
-        parts.push(d.body.slice(0, 500));
-    const text = parts.join('. ');
+        parts.push(`tags: ${d.tags.join(', ')}`);
+    if (d.body) {
+        const body = d.body.length > 500 ? extractMeaningfulContent(d.body, 450) : d.body;
+        parts.push(`content: ${body}`);
+    }
+    parts.push(`title: ${d.title}`);
+    const text = parts.join('\n');
     return isApiMode() ? text : 'passage: ' + text;
 }
+/**
+ * Split a document into multiple chunks for embedding.
+ * Short docs (<500 chars body) produce a single chunk.
+ * Long docs are split by markdown heading regex /^#{1,3}\s/m, max 5 chunks.
+ * Each chunk inherits title+summary as context prefix.
+ */
+export function splitDocToChunks(d) {
+    const contextPrefix = [`title: ${d.title}`];
+    if (d.summary)
+        contextPrefix.push(`summary: ${d.summary}`);
+    if (d.tags.length > 0)
+        contextPrefix.push(`tags: ${d.tags.join(', ')}`);
+    const prefix = contextPrefix.join('\n');
+    // Short or empty body — single chunk using docToText
+    if (!d.body || d.body.length < 500) {
+        return [{ chunkId: `${d.id}#0`, text: docToText(d) }];
+    }
+    // Split body by markdown headings
+    const sections = d.body.split(/^(?=#{1,3}\s)/m).filter(s => s.trim().length > 0);
+    // If splitting produced only one section, single chunk
+    if (sections.length <= 1) {
+        return [{ chunkId: `${d.id}#0`, text: docToText(d) }];
+    }
+    // Cap at 5 chunks
+    const capped = sections.slice(0, 5);
+    const apiMode = isApiMode();
+    return capped.map((section, i) => {
+        const parts = [prefix, `content: ${section.trim()}`, `title: ${d.title}`];
+        const text = parts.join('\n');
+        return {
+            chunkId: `${d.id}#${i}`,
+            text: apiMode ? text : 'passage: ' + text,
+        };
+    });
+}
 export async function buildEmbeddingIndex(docs, existingIndex, precomputedHashes) {
     const apiMode = isApiMode();
     const config = apiMode ? null : await detectDevice();
     const t0 = Date.now();
     const currentHashes = precomputedHashes ?? docs.map(hashDocContent);
+    // Split all docs into chunks (1:N doc-to-chunk mapping)
+    const allChunkIds = [];
+    const allChunkDocIds = [];
+    const allChunkTexts = [];
+    // Track which doc index each chunk group belongs to (for incremental rebuild)
+    const docChunkRanges = [];
+    for (let i = 0; i < docs.length; i++) {
+        const chunks = splitDocToChunks(docs[i]);
+        const startSlot = allChunkIds.length;
+        for (const chunk of chunks) {
+            allChunkIds.push(chunk.chunkId);
+            allChunkDocIds.push(docs[i].id);
+            allChunkTexts.push(chunk.text);
+        }
+        docChunkRanges.push({ docIndex: i, startSlot, count: chunks.length });
+    }
     let vectors;
     const activeModel = getModelId();
     // Model changed → discard all cached vectors, force full rebuild
     const modelMatch = existingIndex && existingIndex.modelId === activeModel;
     if (modelMatch && existingIndex.docIds.length > 0) {
-        // Incremental: reuse vectors only for docs with matching content hash
-        const existingMap = new Map();
-        for (let i = 0; i < existingIndex.docIds.length; i++) {
-            existingMap.set(existingIndex.docIds[i], {
-                vector: existingIndex.vectors[i],
-                hash: existingIndex.contentHashes?.[i] ?? '',
-            });
+        // Build existing chunk cache: chunkId → { vector, parentDocId }
+        const existingChunkMap = new Map();
+        const existingDocHashes = new Map();
+        // Reconstruct per-doc hashes from existing index
+        if (existingIndex.chunkDocIds && existingIndex.contentHashes) {
+            for (let i = 0; i < existingIndex.docIds.length; i++) {
+                existingChunkMap.set(existingIndex.docIds[i], existingIndex.vectors[i]);
+            }
+            // contentHashes are per-doc; find unique doc→hash mapping
+            const seen = new Set();
+            for (let i = 0; i < existingIndex.chunkDocIds.length; i++) {
+                const parentId = existingIndex.chunkDocIds[i];
+                if (!seen.has(parentId) && existingIndex.contentHashes[i] !== undefined) {
+                    // contentHashes in chunk-based index: stored per-doc in the order of docChunkRanges
+                    seen.add(parentId);
+                }
+            }
+        }
+        else {
+            // Legacy index without chunkDocIds — docIds are direct doc IDs
+            for (let i = 0; i < existingIndex.docIds.length; i++) {
+                existingChunkMap.set(existingIndex.docIds[i], existingIndex.vectors[i]);
+                if (existingIndex.contentHashes?.[i]) {
+                    existingDocHashes.set(existingIndex.docIds[i], existingIndex.contentHashes[i]);
+                }
+            }
         }
-        const changedDocs = [];
-        vectors = new Array(docs.length);
-        for (let i = 0; i < docs.length; i++) {
-            const cached = existingMap.get(docs[i].id);
-            if (cached && cached.hash && cached.hash === currentHashes[i]) {
-                vectors[i] = cached.vector;
+        // Determine which docs changed (hash comparison at doc level)
+        // Rebuild per-doc hash from existing contentHashes
+        const existingPerDocHash = new Map();
+        if (existingIndex.contentHashes) {
+            if (existingIndex.chunkDocIds) {
+                // Chunk-based: contentHashes[i] corresponds to the doc that produced chunk i
+                // Each doc's hash is stored on its first chunk
+                const docSeen = new Set();
+                for (let i = 0; i < existingIndex.chunkDocIds.length; i++) {
+                    const pid = existingIndex.chunkDocIds[i];
+                    if (!docSeen.has(pid)) {
+                        docSeen.add(pid);
+                        existingPerDocHash.set(pid, existingIndex.contentHashes[i] ?? '');
+                    }
+                }
             }
             else {
-                changedDocs.push({ index: i, doc: docs[i] });
+                // Legacy: docIds are 1:1 with docs
+                for (let i = 0; i < existingIndex.docIds.length; i++) {
+                    existingPerDocHash.set(existingIndex.docIds[i], existingIndex.contentHashes[i] ?? '');
+                }
             }
         }
-        if (changedDocs.length > 0) {
-            const texts = changedDocs.map(nd => docToText(nd.doc));
+        vectors = new Array(allChunkIds.length);
+        const chunksToEmbed = [];
+        for (const range of docChunkRanges) {
+            const docId = docs[range.docIndex].id;
+            const cachedHash = existingPerDocHash.get(docId);
+            const currentHash = currentHashes[range.docIndex];
+            if (cachedHash && cachedHash === currentHash) {
+                // Doc unchanged — try to reuse cached chunk vectors
+                let allReused = true;
+                for (let s = range.startSlot; s < range.startSlot + range.count; s++) {
+                    const cachedVec = existingChunkMap.get(allChunkIds[s]);
+                    if (cachedVec) {
+                        vectors[s] = cachedVec;
+                    }
+                    else {
+                        allReused = false;
+                        break;
+                    }
+                }
+                if (!allReused) {
+                    // Chunk structure changed (e.g., headings added/removed) — re-embed all chunks
+                    for (let s = range.startSlot; s < range.startSlot + range.count; s++) {
+                        chunksToEmbed.push({ slot: s, text: allChunkTexts[s] });
+                    }
+                }
+            }
+            else {
+                // Doc changed — re-embed all its chunks
+                for (let s = range.startSlot; s < range.startSlot + range.count; s++) {
+                    chunksToEmbed.push({ slot: s, text: allChunkTexts[s] });
+                }
+            }
+        }
+        if (chunksToEmbed.length > 0) {
+            const texts = chunksToEmbed.map(c => c.text);
             const newVectors = await embedTexts(texts);
-            for (let j = 0; j < changedDocs.length; j++) {
-                vectors[changedDocs[j].index] = newVectors[j];
+            for (let j = 0; j < chunksToEmbed.length; j++) {
+                vectors[chunksToEmbed[j].slot] = newVectors[j];
             }
         }
     }
     else {
         // Full rebuild
-        const texts = docs.map(docToText);
-        vectors = await embedTexts(texts);
+        vectors = await embedTexts(allChunkTexts);
+    }
+    // Build per-chunk contentHashes (each chunk gets its parent doc's hash)
+    const parentToHash = new Map();
+    for (const range of docChunkRanges) {
+        parentToHash.set(docs[range.docIndex].id, currentHashes[range.docIndex]);
     }
+    const chunkContentHashes = allChunkDocIds.map(parentId => parentToHash.get(parentId) ?? '');
     return {
         modelId: activeModel,
         dimension: vectors[0]?.length ?? 384,
-        docIds: docs.map(d => d.id),
+        docIds: allChunkIds,
         vectors,
-        contentHashes: currentHashes,
+        contentHashes: chunkContentHashes,
+        chunkDocIds: allChunkDocIds,
         builtAt: Date.now(),
         deviceUsed: apiMode ? 'api' : `${config.device}/${config.dtype}`,
         buildTimeMs: Date.now() - t0,