@andespindola/brainlink 1.0.6 → 1.1.0

package/dist/application/vault-git.js

@@ -0,0 +1,180 @@
+// Git-backed versioning for the vault's canonical Markdown. The derived
+// `.brainlink/` directory is git-ignored; only `.md` is versioned. After any
+// operation that changes Markdown on disk (pull, restore), the index is rebuilt
+// so the vault stays immediately usable ("no losing the use of the data").
+//
+// Git is driven through the system binary via execFile (never a shell, args as
+// an array) to keep runtime dependencies minimal and history/merge behavior
+// native. A missing git binary degrades to a clear, actionable error.
+import { execFile } from 'node:child_process';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { promisify } from 'node:util';
+import { indexVault } from './index-vault.js';
+import { ensureVault } from '../infrastructure/file-system-vault.js';
+const execFileAsync = promisify(execFile);
+export class GitUnavailableError extends Error {
+    constructor() {
+        super('git executable not found. Install git to use vault versioning.');
+        this.name = 'GitUnavailableError';
+    }
+}
+export class GitCommandError extends Error {
+    constructor(args, stderr) {
+        super(`git ${args.join(' ')} failed: ${stderr.trim() || 'unknown error'}`);
+        this.name = 'GitCommandError';
+    }
+}
+const isMissingBinary = (error) => error instanceof Error && 'code' in error && error.code === 'ENOENT';
+const runGit = async (cwd, args) => {
+    try {
+        const { stdout } = await execFileAsync('git', [...args], { cwd, maxBuffer: 64 * 1024 * 1024 });
+        return stdout;
+    }
+    catch (error) {
+        if (isMissingBinary(error)) {
+            throw new GitUnavailableError();
+        }
+        const stderr = error.stderr ?? (error instanceof Error ? error.message : String(error));
+        throw new GitCommandError(args, stderr);
+    }
+};
+const isInsideGitRepo = async (cwd) => {
+    try {
+        const out = await runGit(cwd, ['rev-parse', '--is-inside-work-tree']);
+        return out.trim() === 'true';
+    }
+    catch (error) {
+        if (error instanceof GitUnavailableError) {
+            throw error;
+        }
+        return false;
+    }
+};
+const gitignoreLines = ['.brainlink/', '*.tmp', '.DS_Store'];
+const ensureGitignore = async (vaultRoot) => {
+    const path = join(vaultRoot, '.gitignore');
+    let current = '';
+    try {
+        current = await readFile(path, 'utf8');
+    }
+    catch {
+        current = '';
+    }
+    const existing = new Set(current.split('\n').map((line) => line.trim()));
+    const missing = gitignoreLines.filter((line) => !existing.has(line));
+    if (missing.length === 0 && current.length > 0) {
+        return;
+    }
+    const next = `${[current.trimEnd(), ...missing].filter(Boolean).join('\n')}\n`;
+    await writeFile(path, next, 'utf8');
+};
+// Reject credentials embedded in a remote URL: they would be written to
+// .git/config in clear text. Users should rely on SSH or a git credential
+// helper instead.
+const assertSafeRemote = (remote) => {
+    if (/\/\/[^/@]*:[^/@]*@/.test(remote) || /x-access-token|:[^/]*@github/.test(remote)) {
+        throw new Error('Refusing a remote URL with an inline credential. Use SSH or a git credential helper.');
+    }
+};
+const markdownOnly = (paths) => paths.filter((path) => path.toLowerCase().endsWith('.md'));
+export const initVaultGit = async (vaultPath, options = {}) => {
+    const root = await ensureVault(vaultPath);
+    await mkdir(root, { recursive: true });
+    const already = await isInsideGitRepo(root);
+    if (!already) {
+        await runGit(root, ['init']);
+    }
+    await ensureGitignore(root);
+    if (options.remote) {
+        assertSafeRemote(options.remote);
+        const remotes = await runGit(root, ['remote']);
+        const verb = remotes.split('\n').map((line) => line.trim()).includes('origin') ? 'set-url' : 'add';
+        await runGit(root, ['remote', verb, 'origin', options.remote]);
+    }
+    return { root, initialized: !already };
+};
+export const vaultGitStatus = async (vaultPath) => {
+    const root = await ensureVault(vaultPath);
+    const branch = (await runGit(root, ['rev-parse', '--abbrev-ref', 'HEAD']).catch(() => 'HEAD')).trim();
+    const porcelain = await runGit(root, ['status', '--porcelain']);
+    const changed = porcelain
+        .split('\n')
+        .map((line) => line.slice(3).trim())
+        .filter(Boolean);
+    return { branch, dirty: changed.length > 0, changedMarkdown: markdownOnly(changed) };
+};
+// Stage and commit only Markdown. Returns null when there is nothing to commit.
+export const commitVault = async (vaultPath, message) => {
+    const root = await ensureVault(vaultPath);
+    await runGit(root, ['add', '-A', '--', '*.md', ':(glob)**/*.md', '.gitignore']);
+    const staged = await runGit(root, ['diff', '--cached', '--name-only']);
+    if (staged.trim().length === 0) {
+        return null;
+    }
+    await runGit(root, ['commit', '-m', message]);
+    return (await runGit(root, ['rev-parse', 'HEAD'])).trim();
+};
+export const vaultHistory = async (vaultPath, limit = 20) => {
+    const root = await ensureVault(vaultPath);
+    const out = await runGit(root, [
+        'log',
+        `-n${Math.max(1, Math.floor(limit))}`,
+        '--format=%H%x00%an%x00%aI%x00%s',
+        '--',
+        '*.md'
+    ]).catch(() => '');
+    return out
+        .split('\n')
+        .map((line) => line.split('\u0000'))
+        .filter((parts) => parts.length === 4)
+        .map(([hash, author, date, subject]) => ({ hash: hash, author: author, date: date, subject: subject }));
+};
+const reindexAfterMarkdownChange = async (vaultPath, changedMarkdown, skipIndex) => !skipIndex && changedMarkdown.length > 0 ? indexVault(vaultPath) : undefined;
+export const pushVault = async (vaultPath, branch) => {
+    const root = await ensureVault(vaultPath);
+    const target = branch ?? (await runGit(root, ['rev-parse', '--abbrev-ref', 'HEAD'])).trim();
+    await runGit(root, ['push', '-u', 'origin', target]);
+};
+// Pull from origin and reindex when Markdown changed, so the vault is usable
+// immediately. On a merge conflict the merge is aborted and a clear error is
+// raised rather than leaving conflict markers in the Markdown.
+export const pullVault = async (vaultPath, options = {}) => {
+    const root = await ensureVault(vaultPath);
+    const before = (await runGit(root, ['rev-parse', 'HEAD']).catch(() => '')).trim();
+    try {
+        await runGit(root, ['pull', '--no-rebase', '--no-edit', 'origin']);
+    }
+    catch (error) {
+        await runGit(root, ['merge', '--abort']).catch(() => undefined);
+        throw error instanceof GitCommandError
+            ? new GitCommandError(['pull'], 'merge conflict; aborted. Resolve manually, then commit.')
+            : error;
+    }
+    const after = (await runGit(root, ['rev-parse', 'HEAD']).catch(() => '')).trim();
+    const diff = before && after && before !== after
+        ? await runGit(root, ['diff', '--name-only', before, after, '--', '*.md'])
+        : '';
+    const changedMarkdown = markdownOnly(diff.split('\n').map((line) => line.trim()).filter(Boolean));
+    const index = await reindexAfterMarkdownChange(vaultPath, changedMarkdown, options.skipIndex === true);
+    return { changedMarkdown, index };
+};
+export const restoreVault = async (vaultPath, ref, options = {}) => {
+    const root = await ensureVault(vaultPath);
+    const head = (await runGit(root, ['rev-parse', 'HEAD']).catch(() => '')).trim();
+    const diff = await runGit(root, ['diff', '--name-only', ref, '--', '*.md']).catch(() => '');
+    await runGit(root, ['checkout', ref, '--', '.']);
+    const changedMarkdown = markdownOnly(diff.split('\n').map((line) => line.trim()).filter(Boolean));
+    const index = await reindexAfterMarkdownChange(vaultPath, changedMarkdown, options.skipIndex === true);
+    void head;
+    return { changedMarkdown, index };
+};
+// Clone a remote vault and build its index from scratch so it is immediately
+// searchable. The clone target must not already exist.
+export const cloneVault = async (remote, targetPath, options = {}) => {
+    assertSafeRemote(remote);
+    await runGit(process.cwd(), ['clone', remote, targetPath]);
+    const root = await ensureVault(targetPath);
+    const index = options.skipIndex === true ? undefined : await indexVault(root, { full: true });
+    return { root, index };
+};

package/dist/application/vault-snapshot.js

@@ -0,0 +1,87 @@
+// Portable vault snapshot (`.blvault`): a single gzip-compressed envelope of the
+// canonical Markdown (with per-file SHA-256), self-contained and movable to any
+// machine. The derived index is never included — import rebuilds it so the vault
+// is immediately usable. Conflicts on import are preserved as `*.conflict-<ts>.md`
+// rather than overwriting local edits.
+import { createHash } from 'node:crypto';
+import { gzipSync, gunzipSync } from 'node:zlib';
+import { extname, relative } from 'node:path';
+import { readFile, writeFile } from 'node:fs/promises';
+import { indexVault } from './index-vault.js';
+import { ensureVault, listVaultFiles, writeMarkdownFile } from '../infrastructure/file-system-vault.js';
+const snapshotVersion = 1;
+const isMarkdown = (path) => extname(path).toLowerCase() === '.md';
+const toPosix = (path) => path.split('\\').join('/');
+const sha256 = (data) => createHash('sha256').update(data).digest('hex');
+const conflictName = (path) => {
+    const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\..+$/, 'Z');
+    const extension = extname(path);
+    const base = extension ? path.slice(0, -extension.length) : path;
+    return `${base}.conflict-${stamp}${extension}`;
+};
+export const exportVaultSnapshot = async (vaultPath, outputFile) => {
+    const root = await ensureVault(vaultPath);
+    const markdownFiles = (await listVaultFiles(root)).filter(isMarkdown);
+    const entries = await Promise.all(markdownFiles.map(async (absolutePath) => {
+        const content = await readFile(absolutePath);
+        const path = toPosix(relative(root, absolutePath));
+        return {
+            manifest: { path, sha256: sha256(content), bytes: content.byteLength },
+            file: { path, contentB64: content.toString('base64') }
+        };
+    }));
+    const envelope = {
+        version: snapshotVersion,
+        createdAt: new Date().toISOString(),
+        manifest: entries.map((entry) => entry.manifest),
+        files: entries.map((entry) => entry.file)
+    };
+    const packed = gzipSync(Buffer.from(JSON.stringify(envelope), 'utf8'), { level: 9 });
+    await writeFile(outputFile, packed, { mode: 0o600 });
+    return { file: outputFile, fileCount: entries.length, bytes: packed.byteLength };
+};
+const parseEnvelope = (raw) => {
+    const envelope = JSON.parse(gunzipSync(raw).toString('utf8'));
+    if (!envelope || envelope.version !== snapshotVersion || !Array.isArray(envelope.files)) {
+        throw new Error('Unrecognized .blvault snapshot format.');
+    }
+    return envelope;
+};
+export const importVaultSnapshot = async (snapshotFile, vaultPath, options = {}) => {
+    const root = await ensureVault(vaultPath);
+    const envelope = parseEnvelope(await readFile(snapshotFile));
+    const manifestByPath = new Map(envelope.manifest.map((entry) => [entry.path, entry]));
+    let imported = 0;
+    let conflicted = 0;
+    for (const file of envelope.files) {
+        if (!isMarkdown(file.path)) {
+            continue;
+        }
+        const content = Buffer.from(file.contentB64, 'base64');
+        const expected = manifestByPath.get(file.path)?.sha256;
+        if (expected && sha256(content) !== expected) {
+            throw new Error(`Checksum mismatch for ${file.path}; snapshot is corrupt.`);
+        }
+        // Preserve local edits: write divergent incoming files under a conflict name
+        // instead of overwriting. writeMarkdownFile guards against path traversal.
+        let existing = null;
+        try {
+            existing = await readFile(`${root}/${file.path}`);
+        }
+        catch {
+            existing = null;
+        }
+        if (existing && !existing.equals(content)) {
+            await writeMarkdownFile(root, conflictName(file.path), content.toString('utf8'));
+            conflicted += 1;
+            continue;
+        }
+        if (existing && existing.equals(content)) {
+            continue;
+        }
+        await writeMarkdownFile(root, file.path, content.toString('utf8'));
+        imported += 1;
+    }
+    const index = !options.skipIndex && imported + conflicted > 0 ? await indexVault(root) : undefined;
+    return { imported, conflicted, index };
+};

package/dist/cli/commands/vault-sync-commands.js

@@ -0,0 +1,115 @@
+import { cloneVault, commitVault, initVaultGit, pullVault, pushVault, restoreVault, vaultGitStatus, vaultHistory } from '../../application/vault-git.js';
+import { exportVaultSnapshot, importVaultSnapshot } from '../../application/vault-snapshot.js';
+import { print, resolveOptions } from '../runtime.js';
+export const registerVaultSyncCommands = (program) => {
+    program
+        .command('vault-init-git')
+        .option('--vault <vault>', 'vault directory')
+        .option('--remote <url>', 'git remote URL (SSH or credential-helper backed; inline tokens are rejected)')
+        .option('--json', 'print machine-readable JSON')
+        .description('initialize git versioning for the vault (ignores the derived .brainlink/)')
+        .action(async (options) => {
+        const { vault } = await resolveOptions(options);
+        const result = await initVaultGit(vault, { remote: options.remote });
+        print(options.json, result, () => `${result.initialized ? 'Initialized' : 'Reused'} git repo at ${result.root}${options.remote ? ` (origin: ${options.remote})` : ''}.`);
+    });
+    program
+        .command('vault-commit')
+        .requiredOption('-m, --message <message>', 'commit message')
+        .option('--vault <vault>', 'vault directory')
+        .option('--json', 'print machine-readable JSON')
+        .description('commit the vault Markdown')
+        .action(async (options) => {
+        const { vault } = await resolveOptions(options);
+        const hash = await commitVault(vault, options.message ?? 'Update vault');
+        print(options.json, { committed: hash != null, hash }, () => hash ? `Committed ${hash.slice(0, 10)}.` : 'Nothing to commit.');
+    });
+    program
+        .command('vault-status')
+        .option('--vault <vault>', 'vault directory')
+        .option('--json', 'print machine-readable JSON')
+        .description('show git status of the vault Markdown')
+        .action(async (options) => {
+        const { vault } = await resolveOptions(options);
+        const status = await vaultGitStatus(vault);
+        print(options.json, status, () => `Branch ${status.branch}; ${status.dirty ? `${status.changedMarkdown.length} changed markdown file(s)` : 'clean'}.`);
+    });
+    program
+        .command('vault-history')
+        .option('--vault <vault>', 'vault directory')
+        .option('--limit <n>', 'number of entries', '20')
+        .option('--json', 'print machine-readable JSON')
+        .description('show vault commit history')
+        .action(async (options) => {
+        const { vault } = await resolveOptions(options);
+        const entries = await vaultHistory(vault, Number.parseInt(options.limit ?? '20', 10));
+        print(options.json, entries, () => entries.map((entry) => `${entry.hash.slice(0, 10)}  ${entry.date}  ${entry.subject}`).join('\n') || 'No history.');
+    });
+    program
+        .command('vault-push')
+        .option('--vault <vault>', 'vault directory')
+        .option('--json', 'print machine-readable JSON')
+        .description('push the vault to its git remote')
+        .action(async (options) => {
+        const { vault } = await resolveOptions(options);
+        await pushVault(vault);
+        print(options.json, { pushed: true }, () => 'Pushed to origin.');
+    });
+    program
+        .command('vault-pull')
+        .option('--vault <vault>', 'vault directory')
+        .option('--no-index', 'skip reindexing after pull')
+        .option('--json', 'print machine-readable JSON')
+        .description('pull the vault from git and reindex changed notes')
+        .action(async (options) => {
+        const { vault } = await resolveOptions(options);
+        const result = await pullVault(vault, { skipIndex: options.index === false });
+        print(options.json, result, () => `Pulled. ${result.changedMarkdown.length} markdown file(s) changed${result.index ? `; reindexed ${result.index.documentCount} docs` : ''}.`);
+    });
+    program
+        .command('vault-restore')
+        .argument('<ref>', 'git ref (commit, tag, branch) to restore Markdown from')
+        .option('--vault <vault>', 'vault directory')
+        .option('--no-index', 'skip reindexing after restore')
+        .option('--json', 'print machine-readable JSON')
+        .description('restore vault Markdown from a git ref and reindex')
+        .action(async (ref, options) => {
+        const { vault } = await resolveOptions(options);
+        const result = await restoreVault(vault, ref, { skipIndex: options.index === false });
+        print(options.json, result, () => `Restored from ${ref}. ${result.changedMarkdown.length} markdown file(s) changed${result.index ? `; reindexed ${result.index.documentCount} docs` : ''}.`);
+    });
+    program
+        .command('vault-clone')
+        .argument('<remote>', 'git remote URL')
+        .argument('<dir>', 'target directory')
+        .option('--no-index', 'skip building the index after clone')
+        .option('--json', 'print machine-readable JSON')
+        .description('clone a remote vault and build its index')
+        .action(async (remote, dir, options) => {
+        const result = await cloneVault(remote, dir, { skipIndex: options.index === false });
+        print(options.json, result, () => `Cloned to ${result.root}${result.index ? `; indexed ${result.index.documentCount} docs` : ''}.`);
+    });
+    program
+        .command('vault-export')
+        .requiredOption('-o, --output <file>', 'output .blvault file')
+        .option('--vault <vault>', 'vault directory')
+        .option('--json', 'print machine-readable JSON')
+        .description('export the vault Markdown to a portable .blvault snapshot')
+        .action(async (options) => {
+        const { vault } = await resolveOptions(options);
+        const result = await exportVaultSnapshot(vault, options.output ?? 'vault.blvault');
+        print(options.json, result, () => `Exported ${result.fileCount} files to ${result.file} (${result.bytes} bytes).`);
+    });
+    program
+        .command('vault-import')
+        .argument('<file>', 'input .blvault file')
+        .option('--vault <vault>', 'vault directory')
+        .option('--no-index', 'skip reindexing after import')
+        .option('--json', 'print machine-readable JSON')
+        .description('import a .blvault snapshot into the vault and reindex')
+        .action(async (file, options) => {
+        const { vault } = await resolveOptions(options);
+        const result = await importVaultSnapshot(file, vault, { skipIndex: options.index === false });
+        print(options.json, result, () => `Imported ${result.imported} files (${result.conflicted} conflicts)${result.index ? `; reindexed ${result.index.documentCount} docs` : ''}.`);
+    });
+};

package/dist/cli/main.js

@@ -10,6 +10,7 @@ import { registerConfigCommands } from './commands/config-commands.js';
 import { registerPracticalCommands } from './commands/practical-commands.js';
 import { registerReadCommands } from './commands/read-commands.js';
 import { registerVaultCommands } from './commands/vault-commands.js';
+import { registerVaultSyncCommands } from './commands/vault-sync-commands.js';
 import { registerWriteCommands } from './commands/write-commands.js';
 const readPackageMetadata = () => {
     const packagePath = join(dirname(fileURLToPath(import.meta.url)), '../../package.json');
@@ -63,6 +64,7 @@ registerReadCommands(program);
 registerPracticalCommands(program);
 registerConfigCommands(program);
 registerVaultCommands(program);
+registerVaultSyncCommands(program);
 registerAgentCommands(program);
 await maybePrintUpdateNotice(packageMetadata);
 program.parseAsync().catch((error) => {

package/dist/domain/context.js

@@ -1,21 +1,32 @@
 import { middleOutIndices } from './middle-out.js';
+import { estimateTokenCount } from './tokens.js';
+import { orderByMaximalMarginalRelevance } from './diversity.js';
+const contentTokenPattern = /[\p{L}\p{N}_-]+/gu;
+const tokenizeForDiversity = (text) => new Set((text.toLowerCase().match(contentTokenPattern) ?? []).filter((token) => token.length > 1));
 const maxSectionsPerDocument = 3;
-const charsPerToken = 4;
+// Character budget used only for truncating an overflowing chunk back down to a
+// token target; the conservative per-token character ratio keeps the rendered
+// slice within budget even though estimateTokenCount is the source of truth for
+// counting.
+const truncationCharsPerToken = 4;
 const sectionSeparatorTokens = 1;
 const headerReserveTokens = 8;
-const estimateTokens = (text) => Math.ceil(text.length / charsPerToken);
 const truncateToTokens = (text, maxTokens) => {
-    const maxChars = Math.max(0, maxTokens) * charsPerToken;
+    const maxChars = Math.max(0, maxTokens) * truncationCharsPerToken;
     return text.length <= maxChars ? text : text.slice(0, maxChars);
 };
+// Prefer the token count cached on the chunk at index time; fall back to a live
+// estimate when the result did not carry one (legacy indexes, pack fallback).
+const contentTokens = (result) => result.tokenCount ?? estimateTokenCount(result.content);
 // Mirror the per-section framing that formatContextPackage renders (heading,
 // Source/Tags/Score/Mode lines and the trailing separator) so the budget
 // reflects the package the agent actually receives, not just chunk content.
-const estimateSectionTokens = (result) => {
+const estimateFramingTokens = (result) => {
     const tagsLine = result.tags.length > 0 ? `Tags: ${result.tags.map((tag) => `#${tag}`).join(' ')}\n` : '';
     const framing = `## . ${result.title}\nSource: ${result.path}\n${tagsLine}Score: 0.000\nMode: ${result.searchMode}\n\n`;
-    return estimateTokens(result.content) + estimateTokens(framing) + sectionSeparatorTokens;
+    return estimateTokenCount(framing) + sectionSeparatorTokens;
 };
+const estimateSectionTokens = (result) => contentTokens(result) + estimateFramingTokens(result);
 const byScore = (left, right) => right.score - left.score || left.title.localeCompare(right.title);
 const byOrdinal = (left, right) => (left.chunkOrdinal ?? Number.MAX_SAFE_INTEGER) - (right.chunkOrdinal ?? Number.MAX_SAFE_INTEGER);
 const middleOutDocumentResults = (results) => {
@@ -36,14 +47,16 @@ export const selectContextSections = (results, maxTokens) => {
         state.set(result.documentId, [...current, result]);
         return state;
     }, new Map());
-    const documentOrder = Array.from(results.reduce((state, result) => {
-        if (!state.has(result.documentId)) {
-            state.set(result.documentId, result.score);
-        }
-        return state;
-    }, new Map()).entries())
-        .sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0]))
-        .map(([documentId]) => documentId);
+    // Order documents by maximal marginal relevance so the budget is not spent on
+    // several near-duplicate notes: each pick trades its own relevance against how
+    // similar it is to documents already chosen. The strongest document still
+    // leads (no redundancy penalty on an empty selection).
+    const documentCandidates = Array.from(grouped.entries()).map(([documentId, group]) => ({
+        id: documentId,
+        relevance: group.reduce((max, result) => Math.max(max, result.score), Number.NEGATIVE_INFINITY),
+        tokens: tokenizeForDiversity(group.map((result) => result.content).join(' '))
+    }));
+    const documentOrder = orderByMaximalMarginalRelevance(documentCandidates);
     const selected = documentOrder.reduce((state, documentId) => {
         const ordered = middleOutDocumentResults(grouped.get(documentId) ?? []);
         let usedTokens = state.usedTokens;
@@ -91,7 +104,7 @@ export const selectContextSections = (results, maxTokens) => {
     // brainlink_context consistent with brainlink_search instead of reporting
     // "No relevant context found." whenever the strongest chunk is large.
     const topResult = [...results].sort(byScore)[0];
-    const framingTokens = estimateSectionTokens({ ...topResult, content: '' });
+    const framingTokens = estimateFramingTokens(topResult);
     const contentTokenBudget = maxTokens - headerReserveTokens - framingTokens;
     return [
         {

package/dist/domain/diversity.js

@@ -0,0 +1,64 @@
+// Maximal Marginal Relevance ordering. Greedily orders candidates so each pick
+// balances its own relevance against how redundant it is versus everything
+// already chosen, preventing a context window from filling with near-duplicate
+// material. Pure and deterministic: ties resolve by relevance then id.
+const jaccardSimilarity = (left, right) => {
+    if (left.size === 0 || right.size === 0) {
+        return 0;
+    }
+    let intersection = 0;
+    const [small, large] = left.size <= right.size ? [left, right] : [right, left];
+    for (const token of small) {
+        if (large.has(token)) {
+            intersection += 1;
+        }
+    }
+    const union = left.size + right.size - intersection;
+    return union === 0 ? 0 : intersection / union;
+};
+const normalizeRelevance = (candidates) => {
+    // Reduce instead of Math.min(...values): the candidate list can be as large as
+    // the caller's result limit, and spreading a very large array overflows the
+    // call stack.
+    const bounds = candidates.reduce((range, candidate) => ({
+        min: Math.min(range.min, candidate.relevance),
+        max: Math.max(range.max, candidate.relevance)
+    }), { min: Infinity, max: -Infinity });
+    const span = bounds.max - bounds.min;
+    return new Map(candidates.map((candidate) => [candidate.id, span === 0 ? 1 : (candidate.relevance - bounds.min) / span]));
+};
+// lambda in [0,1]: 1 ignores redundancy (pure relevance order), lower values
+// push diverse picks higher. The default balances relevance and diversity; the
+// strongest candidate still always leads (no redundancy penalty on an empty
+// selection).
+export const orderByMaximalMarginalRelevance = (candidates, lambda = 0.5) => {
+    if (candidates.length <= 1) {
+        return candidates.map((candidate) => candidate.id);
+    }
+    const relevanceById = normalizeRelevance(candidates);
+    const remaining = new Map(candidates.map((candidate) => [candidate.id, candidate]));
+    const selected = [];
+    const order = [];
+    while (remaining.size > 0) {
+        let best = null;
+        for (const candidate of remaining.values()) {
+            const relevance = relevanceById.get(candidate.id) ?? 0;
+            const redundancy = selected.reduce((max, picked) => Math.max(max, jaccardSimilarity(candidate.tokens, picked.tokens)), 0);
+            const score = lambda * relevance - (1 - lambda) * redundancy;
+            if (best === null ||
+                score > best.score ||
+                (score === best.score &&
+                    (relevance > best.relevance ||
+                        (relevance === best.relevance && candidate.id.localeCompare(best.candidate.id) < 0)))) {
+                best = { candidate, score, relevance };
+            }
+        }
+        if (best === null) {
+            break;
+        }
+        selected.push(best.candidate);
+        order.push(best.candidate.id);
+        remaining.delete(best.candidate.id);
+    }
+    return order;
+};

package/dist/domain/embeddings.js

@@ -141,4 +141,120 @@ export const createLocalEmbeddingProvider = () => ({
     name: 'local',
     embed: async (input) => input.map(createLocalEmbedding)
 });
-export const createEmbeddingProvider = (name) => name === 'local' ? createLocalEmbeddingProvider() : createDisabledEmbeddingProvider();
+const defaultExternalTimeoutMs = 10_000;
+const defaultExternalBatchSize = 64;
+const ollamaDefaultUrl = 'http://127.0.0.1:11434';
+const ollamaDefaultModel = 'nomic-embed-text';
+const openAiDefaultUrl = 'https://api.openai.com/v1';
+const openAiDefaultModel = 'text-embedding-3-small';
+const trimTrailingSlash = (value) => value.replace(/\/+$/, '');
+const chunkBatches = (items, size) => {
+    const batchSize = Number.isFinite(size) && size > 0 ? Math.floor(size) : defaultExternalBatchSize;
+    const batches = [];
+    for (let index = 0; index < items.length; index += batchSize) {
+        batches.push(items.slice(index, index + batchSize));
+    }
+    return batches;
+};
+const isNumberArray = (value) => Array.isArray(value) && value.every((entry) => typeof entry === 'number' && Number.isFinite(entry));
+const fetchJson = async (url, init, timeoutMs, providerName) => {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), Math.max(1, timeoutMs));
+    try {
+        const response = await fetch(url, { ...init, signal: controller.signal });
+        if (!response.ok) {
+            throw new Error(`${providerName} embedding request failed with HTTP ${response.status}`);
+        }
+        return (await response.json());
+    }
+    catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        throw new Error(`${providerName} embedding request to ${url} failed: ${reason}`);
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+};
+// External providers return vectors in their own space; L2-normalize so dot
+// product equals cosine similarity, matching the contract local embeddings rely
+// on across the retrieval and bucket-prefilter paths.
+const createOllamaEmbeddingProvider = (config) => {
+    const baseUrl = trimTrailingSlash(config.url?.trim() || ollamaDefaultUrl);
+    const model = config.model?.trim() || ollamaDefaultModel;
+    const timeoutMs = config.timeoutMs ?? defaultExternalTimeoutMs;
+    return {
+        name: 'ollama',
+        embed: async (input) => {
+            const vectors = [];
+            for (const batch of chunkBatches(input, config.batchSize ?? defaultExternalBatchSize)) {
+                const payload = await fetchJson(`${baseUrl}/api/embed`, {
+                    method: 'POST',
+                    headers: { 'content-type': 'application/json' },
+                    body: JSON.stringify({ model, input: batch })
+                }, timeoutMs, 'ollama');
+                const embeddings = payload.embeddings;
+                if (!Array.isArray(embeddings) || embeddings.length !== batch.length) {
+                    throw new Error('ollama embedding response shape was invalid');
+                }
+                for (const embedding of embeddings) {
+                    if (!isNumberArray(embedding) || embedding.length === 0) {
+                        throw new Error('ollama embedding response contained an invalid vector');
+                    }
+                    vectors.push(normalizeVector(embedding));
+                }
+            }
+            return vectors;
+        }
+    };
+};
+const createOpenAiEmbeddingProvider = (config) => {
+    const baseUrl = trimTrailingSlash(config.url?.trim() || openAiDefaultUrl);
+    const model = config.model?.trim() || openAiDefaultModel;
+    const timeoutMs = config.timeoutMs ?? defaultExternalTimeoutMs;
+    const apiKeyEnv = config.apiKeyEnv?.trim() || 'OPENAI_API_KEY';
+    return {
+        name: 'openai',
+        embed: async (input) => {
+            const apiKey = process.env[apiKeyEnv]?.trim();
+            if (!apiKey) {
+                throw new Error(`openai embedding requires the ${apiKeyEnv} environment variable to be set`);
+            }
+            const vectors = [];
+            for (const batch of chunkBatches(input, config.batchSize ?? defaultExternalBatchSize)) {
+                const payload = await fetchJson(`${baseUrl}/embeddings`, {
+                    method: 'POST',
+                    headers: {
+                        'content-type': 'application/json',
+                        authorization: `Bearer ${apiKey}`
+                    },
+                    body: JSON.stringify({ model, input: batch })
+                }, timeoutMs, 'openai');
+                const data = payload.data;
+                if (!Array.isArray(data) || data.length !== batch.length) {
+                    throw new Error('openai embedding response shape was invalid');
+                }
+                const ordered = [...data].sort((left, right) => (left.index ?? 0) - (right.index ?? 0));
+                for (const entry of ordered) {
+                    const embedding = entry.embedding;
+                    if (!isNumberArray(embedding) || embedding.length === 0) {
+                        throw new Error('openai embedding response contained an invalid vector');
+                    }
+                    vectors.push(normalizeVector(embedding));
+                }
+            }
+            return vectors;
+        }
+    };
+};
+export const createEmbeddingProvider = (name, config = {}) => {
+    switch (name) {
+        case 'local':
+            return createLocalEmbeddingProvider();
+        case 'ollama':
+            return createOllamaEmbeddingProvider(config);
+        case 'openai':
+            return createOpenAiEmbeddingProvider(config);
+        default:
+            return createDisabledEmbeddingProvider();
+    }
+};