@andespindola/brainlink 0.1.0-beta.9 → 0.1.0-beta.90

package/dist/infrastructure/index-state.js

@@ -0,0 +1,56 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+const indexStateFileName = 'index-state.json';
+const toIndexStatePath = (vaultPath) => join(vaultPath, '.brainlink', indexStateFileName);
+export const readIndexState = async (vaultPath) => {
+    try {
+        const parsed = JSON.parse(await readFile(toIndexStatePath(vaultPath), 'utf8'));
+        if (parsed.version !== 1 || !Array.isArray(parsed.files)) {
+            return null;
+        }
+        const files = parsed.files.flatMap((entry) => {
+            if (!entry || typeof entry !== 'object') {
+                return [];
+            }
+            const row = entry;
+            if (typeof row.path !== 'string' || typeof row.mtimeMs !== 'number' || typeof row.size !== 'number') {
+                return [];
+            }
+            return [
+                {
+                    path: row.path,
+                    mtimeMs: row.mtimeMs,
+                    size: row.size
+                }
+            ];
+        });
+        return {
+            version: 1,
+            updatedAt: typeof parsed.updatedAt === 'string' ? parsed.updatedAt : new Date().toISOString(),
+            chunkSize: typeof parsed.chunkSize === 'number' ? parsed.chunkSize : 1200,
+            embeddingProvider: typeof parsed.embeddingProvider === 'string' ? parsed.embeddingProvider : 'none',
+            searchPackRowChunkSize: typeof parsed.searchPackRowChunkSize === 'number' ? parsed.searchPackRowChunkSize : 5_000,
+            searchPackCompressionLevel: typeof parsed.searchPackCompressionLevel === 'number' ? parsed.searchPackCompressionLevel : 5,
+            searchPackUseDictionary: typeof parsed.searchPackUseDictionary === 'boolean' ? parsed.searchPackUseDictionary : true,
+            files,
+            pendingPackChanges: typeof parsed.pendingPackChanges === 'number' && parsed.pendingPackChanges >= 0 ? parsed.pendingPackChanges : 0
+        };
+    }
+    catch {
+        return null;
+    }
+};
+export const writeIndexState = async (vaultPath, state) => {
+    const payload = {
+        version: 1,
+        updatedAt: new Date().toISOString(),
+        chunkSize: state.chunkSize,
+        embeddingProvider: state.embeddingProvider,
+        searchPackRowChunkSize: state.searchPackRowChunkSize,
+        searchPackCompressionLevel: state.searchPackCompressionLevel,
+        searchPackUseDictionary: state.searchPackUseDictionary,
+        files: [...state.files].sort((left, right) => left.path.localeCompare(right.path)),
+        pendingPackChanges: Math.max(0, Math.floor(state.pendingPackChanges))
+    };
+    await writeFile(toIndexStatePath(vaultPath), `${JSON.stringify(payload)}\n`, 'utf8');
+};

package/dist/infrastructure/private-pack-codec.js

@@ -0,0 +1,134 @@
+import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'node:crypto';
+import { brotliCompressSync, brotliDecompressSync, constants as zlibConstants } from 'node:zlib';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { getBrainlinkHomePath } from './paths.js';
+const magic = Buffer.from('BLPK2', 'ascii');
+const legacyVersion = 1;
+const currentVersion = 2;
+const nonceLength = 12;
+const authTagLength = 16;
+const algorithm = 'aes-256-gcm';
+const compressionLevelMask = 0x0f;
+const compressionDictionaryMask = 0x10;
+const defaultCompressionLevel = 5;
+const builtinDictionary = Buffer.from([
+    '"documentId","agentId","title","path","chunkId","chunkOrdinal","content","tags"',
+    '"searchMode","textScore","semanticScore","weight","priority","shared"',
+    'agents/shared memory-hub architecture context index search graph markdown tags links',
+    '#memory #architecture #context #graph #search #index [[Memory Hub]] [[Architecture]]',
+    'The quick brown fox jumps over the lazy dog. Brainlink context package metadata.',
+    'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-:/.#[]{}(), '
+].join('\n'), 'utf8');
+const keyFilePath = (vaultPath) => {
+    const vaultHash = createHash('sha256').update(vaultPath).digest('hex').slice(0, 24);
+    return join(getBrainlinkHomePath(), 'keys', `search-pack-${vaultHash}.key`);
+};
+const deriveKeyFromSecret = (secret) => createHash('sha256').update(secret, 'utf8').digest();
+const readOrCreateKey = async (vaultPath) => {
+    const envSecret = process.env.BRAINLINK_SEARCH_PACK_KEY?.trim();
+    if (envSecret && envSecret.length > 0) {
+        return deriveKeyFromSecret(envSecret);
+    }
+    const path = keyFilePath(vaultPath);
+    try {
+        const existing = (await readFile(path, 'utf8')).trim();
+        if (existing.length > 0) {
+            return deriveKeyFromSecret(existing);
+        }
+    }
+    catch (error) {
+        if (!(error instanceof Error) || !('code' in error) || error.code !== 'ENOENT') {
+            throw error;
+        }
+    }
+    const secret = randomBytes(48).toString('base64url');
+    await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+    await writeFile(path, `${secret}\n`, { encoding: 'utf8', mode: 0o600 });
+    return deriveKeyFromSecret(secret);
+};
+const parseHeader = (payload) => {
+    if (payload.length < magic.length + 1 + nonceLength + authTagLength) {
+        throw new Error('Invalid private pack payload: too short.');
+    }
+    const payloadMagic = payload.subarray(0, magic.length);
+    const payloadVersion = payload[magic.length] ?? 0;
+    if (!payloadMagic.equals(magic) || (payloadVersion !== legacyVersion && payloadVersion !== currentVersion)) {
+        throw new Error('Invalid private pack payload: unsupported format.');
+    }
+    const hasCompressionSettings = payloadVersion >= 2;
+    const settingsByte = hasCompressionSettings ? payload[magic.length + 1] ?? 0 : null;
+    const nonceStart = magic.length + 1 + (hasCompressionSettings ? 1 : 0);
+    const authTagStart = nonceStart + nonceLength;
+    const dataStart = authTagStart + authTagLength;
+    return {
+        compression: settingsByte != null
+            ? {
+                compressionLevel: settingsByte & compressionLevelMask,
+                useDictionary: (settingsByte & compressionDictionaryMask) !== 0
+            }
+            : {
+                compressionLevel: defaultCompressionLevel,
+                useDictionary: false
+            },
+        nonce: payload.subarray(nonceStart, authTagStart),
+        authTag: payload.subarray(authTagStart, dataStart),
+        ciphertext: payload.subarray(dataStart)
+    };
+};
+const toCompressionLevel = (value) => {
+    if (typeof value !== 'number' || !Number.isFinite(value)) {
+        return defaultCompressionLevel;
+    }
+    const normalized = Math.round(value);
+    if (normalized < 0) {
+        return 0;
+    }
+    if (normalized > 11) {
+        return 11;
+    }
+    return normalized;
+};
+const encodeCompressionSettings = (settings) => (settings.compressionLevel & compressionLevelMask) | (settings.useDictionary ? compressionDictionaryMask : 0);
+const brotliEncode = (content, settings) => {
+    const options = {
+        params: {
+            [zlibConstants.BROTLI_PARAM_MODE]: zlibConstants.BROTLI_MODE_TEXT,
+            [zlibConstants.BROTLI_PARAM_QUALITY]: settings.compressionLevel
+        }
+    };
+    if (settings.useDictionary) {
+        options.dictionary = builtinDictionary;
+    }
+    return brotliCompressSync(content, options);
+};
+const brotliDecode = (content, settings) => {
+    const options = {};
+    if (settings.useDictionary) {
+        options.dictionary = builtinDictionary;
+    }
+    return brotliDecompressSync(content, options);
+};
+export const encodePrivatePack = async (vaultPath, content, settings) => {
+    const key = await readOrCreateKey(vaultPath);
+    const nonce = randomBytes(nonceLength);
+    const normalizedSettings = {
+        compressionLevel: toCompressionLevel(settings?.compressionLevel),
+        useDictionary: settings?.useDictionary ?? true
+    };
+    const compressed = brotliEncode(content, normalizedSettings);
+    const cipher = createCipheriv(algorithm, key, nonce);
+    const ciphertext = Buffer.concat([cipher.update(compressed), cipher.final()]);
+    const authTag = cipher.getAuthTag();
+    const settingsByte = Buffer.from([encodeCompressionSettings(normalizedSettings)]);
+    return Buffer.concat([magic, Buffer.from([currentVersion]), settingsByte, nonce, authTag, ciphertext]);
+};
+export const decodePrivatePack = async (vaultPath, payload) => {
+    const key = await readOrCreateKey(vaultPath);
+    const { nonce, authTag, ciphertext, compression } = parseHeader(payload);
+    const decipher = createDecipheriv(algorithm, key, nonce);
+    decipher.setAuthTag(authTag);
+    const compressed = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+    return brotliDecode(compressed, compression);
+};
+export const isPrivatePackPayload = (payload) => payload.length >= magic.length + 1 && payload.subarray(0, magic.length).equals(magic);

package/dist/infrastructure/search-packs.js

@@ -0,0 +1,452 @@
+import { gunzipSync } from 'node:zlib';
+import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { middleOutIndices } from '../domain/middle-out.js';
+import { decodePrivatePack, encodePrivatePack, isPrivatePackPayload } from './private-pack-codec.js';
+const packsDirectoryName = 'search-packs';
+const manifestFileName = 'manifest.json';
+const defaultBuildOptions = {
+    rowChunkSize: 5_000,
+    compressionLevel: 5,
+    useDictionary: true
+};
+const queryTokenPattern = /[\p{L}\p{N}_-]+/gu;
+const bloomBytes = 256;
+const bloomBitSize = bloomBytes * 8;
+const bloomSeeds = [0x9e3779b1, 0x85ebca6b, 0xc2b2ae35];
+const toPackDirectory = (vaultPath) => join(vaultPath, '.brainlink', packsDirectoryName);
+const toManifestPath = (vaultPath) => join(toPackDirectory(vaultPath), manifestFileName);
+const parseRowsFromPack = async (vaultPath, content) => {
+    const raw = isPrivatePackPayload(content) ? await decodePrivatePack(vaultPath, content) : gunzipSync(content);
+    return raw
+        .toString('utf8')
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line.length > 0)
+        .map((line) => JSON.parse(line))
+        .flatMap((row) => {
+        if (typeof row.documentId !== 'string' ||
+            typeof row.agentId !== 'string' ||
+            typeof row.title !== 'string' ||
+            typeof row.path !== 'string' ||
+            typeof row.chunkId !== 'string' ||
+            typeof row.content !== 'string') {
+            return [];
+        }
+        return [
+            {
+                documentId: row.documentId,
+                agentId: row.agentId,
+                title: row.title,
+                path: row.path,
+                chunkId: row.chunkId,
+                chunkOrdinal: typeof row.chunkOrdinal === 'number' ? row.chunkOrdinal : 0,
+                content: row.content,
+                tags: Array.isArray(row.tags) ? row.tags.filter((item) => typeof item === 'string') : []
+            }
+        ];
+    });
+};
+const toRows = (documents) => documents.flatMap((document) => document.chunks.map((chunk) => ({
+    documentId: document.document.id,
+    agentId: document.document.agentId,
+    title: document.document.title,
+    path: document.document.path,
+    chunkId: chunk.id,
+    chunkOrdinal: chunk.ordinal,
+    content: chunk.content,
+    tags: document.document.tags
+})));
+const writeManifest = async (vaultPath, manifest) => {
+    await writeFile(toManifestPath(vaultPath), `${JSON.stringify(manifest, null, 2)}\n`, 'utf8');
+};
+const readManifest = async (vaultPath) => {
+    try {
+        const parsed = JSON.parse(await readFile(toManifestPath(vaultPath), 'utf8'));
+        if (parsed.version === 2 && parsed.format === 'private-v2') {
+            return {
+                version: 2,
+                createdAt: typeof parsed.createdAt === 'string' ? parsed.createdAt : new Date().toISOString(),
+                packCount: typeof parsed.packCount === 'number' ? parsed.packCount : 0,
+                recordCount: typeof parsed.recordCount === 'number' ? parsed.recordCount : 0,
+                format: 'private-v2'
+            };
+        }
+        if (parsed.version === 3 && parsed.format === 'private-v2') {
+            const packIndex = Array.isArray(parsed.packIndex)
+                ? parsed.packIndex.flatMap((entry) => {
+                    if (!entry || typeof entry !== 'object') {
+                        return [];
+                    }
+                    const candidate = entry;
+                    if (typeof candidate.fileName !== 'string' || typeof candidate.tokenBloomB64 !== 'string') {
+                        return [];
+                    }
+                    return [
+                        {
+                            fileName: candidate.fileName,
+                            recordCount: typeof candidate.recordCount === 'number' ? candidate.recordCount : 0,
+                            agents: Array.isArray(candidate.agents) ? candidate.agents.filter((item) => typeof item === 'string') : [],
+                            tokenBloomB64: candidate.tokenBloomB64
+                        }
+                    ];
+                })
+                : [];
+            return {
+                version: 3,
+                createdAt: typeof parsed.createdAt === 'string' ? parsed.createdAt : new Date().toISOString(),
+                packCount: typeof parsed.packCount === 'number' ? parsed.packCount : packIndex.length,
+                recordCount: typeof parsed.recordCount === 'number' ? parsed.recordCount : 0,
+                format: 'private-v2',
+                packIndex,
+                ...(parsed.packConfig && typeof parsed.packConfig === 'object'
+                    ? {
+                        packConfig: {
+                            rowChunkSize: typeof parsed.packConfig.rowChunkSize === 'number'
+                                ? parsed.packConfig.rowChunkSize
+                                : defaultBuildOptions.rowChunkSize,
+                            compressionLevel: typeof parsed.packConfig.compressionLevel === 'number'
+                                ? parsed.packConfig.compressionLevel
+                                : defaultBuildOptions.compressionLevel,
+                            useDictionary: typeof parsed.packConfig.useDictionary === 'boolean'
+                                ? parsed.packConfig.useDictionary
+                                : defaultBuildOptions.useDictionary
+                        }
+                    }
+                    : {}),
+                ...(parsed.compression &&
+                    typeof parsed.compression === 'object' &&
+                    typeof parsed.compression.inputBytes === 'number' &&
+                    typeof parsed.compression.outputBytes === 'number' &&
+                    typeof parsed.compression.ratio === 'number' &&
+                    typeof parsed.compression.savedBytes === 'number'
+                    ? {
+                        compression: {
+                            inputBytes: parsed.compression.inputBytes,
+                            outputBytes: parsed.compression.outputBytes,
+                            ratio: parsed.compression.ratio,
+                            savedBytes: parsed.compression.savedBytes
+                        }
+                    }
+                    : {})
+            };
+        }
+        return null;
+    }
+    catch {
+        return null;
+    }
+};
+export const ensureSearchPackManifest = async (vaultPath) => {
+    const manifest = await readManifest(vaultPath);
+    if (manifest) {
+        return {
+            repaired: false,
+            source: 'not-needed',
+            packCount: manifest.packCount
+        };
+    }
+    const files = await sortedPackFiles(vaultPath);
+    const packFiles = files.filter((file) => file.endsWith('.blpk'));
+    if (packFiles.length === 0) {
+        return {
+            repaired: false,
+            source: 'no-packs',
+            packCount: 0
+        };
+    }
+    await writeManifest(vaultPath, {
+        version: 2,
+        createdAt: new Date().toISOString(),
+        packCount: packFiles.length,
+        recordCount: 0,
+        format: 'private-v2'
+    });
+    return {
+        repaired: true,
+        source: 'existing-packs',
+        packCount: packFiles.length
+    };
+};
+const chunkRows = (rows, size) => {
+    const chunks = [];
+    for (let index = 0; index < rows.length; index += size) {
+        chunks.push(rows.slice(index, index + size));
+    }
+    return chunks;
+};
+const normalizeToken = (value) => value
+    .normalize('NFKD')
+    .replace(/\p{Diacritic}/gu, '')
+    .toLowerCase();
+const tokenize = (query) => query
+    .match(queryTokenPattern)
+    ?.map(normalizeToken)
+    .filter((token) => token.length > 1) ?? [];
+const countOccurrences = (text, token) => {
+    let hits = 0;
+    let start = 0;
+    while (start < text.length) {
+        const index = text.indexOf(token, start);
+        if (index < 0) {
+            break;
+        }
+        hits += 1;
+        start = index + token.length;
+    }
+    return hits;
+};
+const hashToken = (token, seed) => {
+    let hash = seed >>> 0;
+    for (let index = 0; index < token.length; index += 1) {
+        hash ^= token.charCodeAt(index);
+        hash = Math.imul(hash, 16777619) >>> 0;
+    }
+    return hash >>> 0;
+};
+const createBloom = () => new Uint8Array(bloomBytes);
+const bloomAdd = (bloom, token) => {
+    bloomSeeds.forEach((seed) => {
+        const bit = hashToken(token, seed) % bloomBitSize;
+        bloom[Math.floor(bit / 8)] |= 1 << (bit % 8);
+    });
+};
+const bloomMayContain = (bloom, token) => bloomSeeds.every((seed) => {
+    const bit = hashToken(token, seed) % bloomBitSize;
+    return (bloom[Math.floor(bit / 8)] & (1 << (bit % 8))) !== 0;
+});
+const bloomFromRows = (rows) => {
+    const bloom = createBloom();
+    rows.forEach((row) => {
+        tokenize([row.title, row.path, row.tags.join(' '), row.content].join(' ')).forEach((token) => bloomAdd(bloom, token));
+    });
+    return bloom;
+};
+const bloomToBase64 = (bloom) => Buffer.from(bloom).toString('base64url');
+const bloomFromBase64 = (value) => {
+    try {
+        const decoded = Buffer.from(value, 'base64url');
+        if (decoded.byteLength === bloomBytes) {
+            return {
+                bloom: new Uint8Array(decoded),
+                valid: true
+            };
+        }
+    }
+    catch {
+        // fallback below
+    }
+    return {
+        bloom: createBloom(),
+        valid: false
+    };
+};
+const computeTextScore = (row, tokens) => {
+    if (tokens.length === 0) {
+        return 0;
+    }
+    const title = normalizeToken(row.title);
+    const path = normalizeToken(row.path);
+    const content = normalizeToken(row.content);
+    const tags = normalizeToken(row.tags.join(' '));
+    return tokens.reduce((score, token) => {
+        const titleHits = countOccurrences(title, token);
+        const tagHits = countOccurrences(tags, token);
+        const pathHits = countOccurrences(path, token);
+        const contentHits = countOccurrences(content, token);
+        return score + titleHits * 5 + tagHits * 4 + pathHits * 2 + Math.min(contentHits, 5);
+    }, 0);
+};
+const toSearchResult = (row, score) => ({
+    documentId: row.documentId,
+    agentId: row.agentId,
+    title: row.title,
+    path: row.path,
+    chunkId: row.chunkId,
+    chunkOrdinal: row.chunkOrdinal,
+    content: row.content,
+    score,
+    textScore: score,
+    semanticScore: 0,
+    searchMode: 'fts',
+    tags: row.tags
+});
+const sortedPackFiles = async (vaultPath) => {
+    try {
+        const files = await readdir(toPackDirectory(vaultPath));
+        return files
+            .filter((file) => file.endsWith('.blpk') || file.endsWith('.jsonl.gz'))
+            .sort((left, right) => left.localeCompare(right));
+    }
+    catch (error) {
+        if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
+            return [];
+        }
+        throw error;
+    }
+};
+const writeRowsAsPrivatePacks = async (vaultPath, rows, clearExisting, options) => {
+    const startedAt = process.hrtime.bigint();
+    const directory = toPackDirectory(vaultPath);
+    await mkdir(directory, { recursive: true });
+    if (clearExisting) {
+        const current = await readdir(directory);
+        await Promise.all(current
+            .filter((name) => name.endsWith('.blpk') || name.endsWith('.jsonl.gz') || name === manifestFileName)
+            .map((name) => rm(join(directory, name), { force: true })));
+    }
+    const chunks = chunkRows(rows, options.rowChunkSize);
+    const packIndex = [];
+    let inputBytes = 0;
+    let outputBytes = 0;
+    for (let index = 0; index < chunks.length; index += 1) {
+        const chunk = chunks[index];
+        const fileName = `pack-${String(index + 1).padStart(4, '0')}.blpk`;
+        const serialized = `${chunk.map((row) => JSON.stringify(row)).join('\n')}\n`;
+        const compressed = await encodePrivatePack(vaultPath, Buffer.from(serialized, 'utf8'), {
+            compressionLevel: options.compressionLevel,
+            useDictionary: options.useDictionary
+        });
+        const tokenBloomB64 = bloomToBase64(bloomFromRows(chunk));
+        await writeFile(join(directory, fileName), compressed);
+        inputBytes += Buffer.byteLength(serialized, 'utf8');
+        outputBytes += compressed.byteLength;
+        packIndex.push({
+            fileName,
+            recordCount: chunk.length,
+            agents: Array.from(new Set(chunk.map((row) => row.agentId))).sort((left, right) => left.localeCompare(right)),
+            tokenBloomB64
+        });
+    }
+    await writeManifest(vaultPath, {
+        version: 3,
+        createdAt: new Date().toISOString(),
+        packCount: chunks.length,
+        recordCount: rows.length,
+        format: 'private-v2',
+        packIndex,
+        packConfig: {
+            rowChunkSize: options.rowChunkSize,
+            compressionLevel: options.compressionLevel,
+            useDictionary: options.useDictionary
+        },
+        compression: {
+            inputBytes,
+            outputBytes,
+            ratio: outputBytes / Math.max(inputBytes, 1),
+            savedBytes: Math.max(inputBytes - outputBytes, 0)
+        }
+    });
+    const durationMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000;
+    const safeInput = Math.max(inputBytes, 1);
+    const savedBytes = Math.max(inputBytes - outputBytes, 0);
+    return {
+        packCount: chunks.length,
+        recordCount: rows.length,
+        compression: {
+            inputBytes,
+            outputBytes,
+            ratio: outputBytes / safeInput,
+            savedBytes
+        },
+        durationMs
+    };
+};
+const selectCandidatePackFiles = async (vaultPath, tokens, agentId) => {
+    const allFiles = await sortedPackFiles(vaultPath);
+    if (allFiles.length === 0) {
+        return [];
+    }
+    const manifest = await readManifest(vaultPath);
+    if (!manifest || manifest.version !== 3 || !Array.isArray(manifest.packIndex)) {
+        return allFiles;
+    }
+    const normalizedAgent = agentId?.trim();
+    const byAgent = manifest.packIndex.filter((entry) => normalizedAgent ? entry.agents.includes(normalizedAgent) : true);
+    if (tokens.length === 0) {
+        return byAgent.map((entry) => entry.fileName);
+    }
+    let hasInvalidBloomIndex = false;
+    const byToken = byAgent.filter((entry) => {
+        const decoded = bloomFromBase64(entry.tokenBloomB64);
+        if (!decoded.valid) {
+            hasInvalidBloomIndex = true;
+            return true;
+        }
+        return tokens.some((token) => bloomMayContain(decoded.bloom, token));
+    });
+    // Lossless guarantee: if compressed metadata is partially invalid, do not prune packs.
+    if (hasInvalidBloomIndex) {
+        return byAgent.map((entry) => entry.fileName);
+    }
+    if (byToken.length > 0) {
+        return byToken.map((entry) => entry.fileName);
+    }
+    return byAgent.length > 0 ? byAgent.map((entry) => entry.fileName) : allFiles;
+};
+export const buildSearchPacks = async (vaultPath, documents, options) => {
+    const resolvedOptions = {
+        rowChunkSize: options?.rowChunkSize ?? defaultBuildOptions.rowChunkSize,
+        compressionLevel: options?.compressionLevel ?? defaultBuildOptions.compressionLevel,
+        useDictionary: options?.useDictionary ?? defaultBuildOptions.useDictionary
+    };
+    return writeRowsAsPrivatePacks(vaultPath, toRows(documents), true, resolvedOptions);
+};
+export const ensurePrivatePacksFromLegacyIndex = async (vaultPath) => {
+    const files = await sortedPackFiles(vaultPath);
+    if (files.some((file) => file.endsWith('.blpk'))) {
+        return { imported: false };
+    }
+    const legacyPackFiles = files.filter((file) => file.endsWith('.jsonl.gz'));
+    if (legacyPackFiles.length > 0) {
+        const rows = [];
+        for (const file of legacyPackFiles) {
+            const parsed = await parseRowsFromPack(vaultPath, await readFile(join(toPackDirectory(vaultPath), file)));
+            rows.push(...parsed);
+        }
+        const report = await writeRowsAsPrivatePacks(vaultPath, rows, true, defaultBuildOptions);
+        return {
+            imported: true,
+            source: 'legacy-packs',
+            ...report
+        };
+    }
+    return { imported: false };
+};
+export const toSearchPackBuildOptions = (config) => ({
+    rowChunkSize: config.searchPack.rowChunkSize,
+    compressionLevel: config.searchPack.compressionLevel,
+    useDictionary: config.searchPack.useDictionary
+});
+export const searchInPacks = async (vaultPath, query, limit, agentId) => {
+    const normalizedAgent = agentId?.trim();
+    const tokens = tokenize(query);
+    if (limit <= 0 || tokens.length === 0) {
+        return [];
+    }
+    const files = await selectCandidatePackFiles(vaultPath, tokens, normalizedAgent);
+    if (files.length === 0) {
+        return [];
+    }
+    const scored = [];
+    for (const file of files) {
+        const rows = await parseRowsFromPack(vaultPath, await readFile(join(toPackDirectory(vaultPath), file)));
+        const traversal = middleOutIndices(rows.length, Math.floor(rows.length / 2));
+        traversal.forEach((rowIndex) => {
+            const row = rows[rowIndex];
+            if (!row) {
+                return;
+            }
+            if (normalizedAgent && row.agentId !== normalizedAgent) {
+                return;
+            }
+            const score = computeTextScore(row, tokens);
+            if (score > 0) {
+                scored.push(toSearchResult(row, score));
+            }
+        });
+    }
+    return scored
+        .sort((left, right) => right.score - left.score || left.title.localeCompare(right.title))
+        .slice(0, limit);
+};