@andespindola/brainlink 1.0.7 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -1
- package/README.md +63 -2
- package/dist/application/analyze-vault.js +1 -1
- package/dist/application/build-context.js +14 -7
- package/dist/application/get-graph-node.js +2 -2
- package/dist/application/get-graph-summary.js +2 -2
- package/dist/application/get-graph.js +2 -2
- package/dist/application/index-vault-phases.js +1 -0
- package/dist/application/index-vault.js +16 -2
- package/dist/application/list-agents.js +2 -2
- package/dist/application/list-links.js +3 -3
- package/dist/application/ports/knowledge-store.js +1 -0
- package/dist/application/provision-vault.js +74 -0
- package/dist/application/ranking/run-search.js +212 -0
- package/dist/application/search-graph-node-ids.js +3 -2
- package/dist/application/search-knowledge.js +9 -17
- package/dist/application/server/routes.js +6 -1
- package/dist/application/vault-encryption.js +64 -0
- package/dist/application/vault-git-core.js +168 -0
- package/dist/application/vault-git.js +75 -0
- package/dist/application/vault-snapshot.js +87 -0
- package/dist/cli/commands/read-commands.js +9 -5
- package/dist/cli/commands/vault-sync-commands.js +137 -0
- package/dist/cli/commands/write/vault-lifecycle-commands.js +1 -1
- package/dist/cli/main.js +2 -0
- package/dist/infrastructure/config.js +16 -1
- package/dist/infrastructure/context-packs.js +57 -18
- package/dist/infrastructure/file-index.js +22 -409
- package/dist/infrastructure/index-signature.js +27 -0
- package/dist/infrastructure/index-state.js +6 -0
- package/dist/infrastructure/knowledge-store/binary-store.js +333 -0
- package/dist/infrastructure/knowledge-store/index.js +37 -0
- package/dist/infrastructure/knowledge-store/stored-index.js +216 -0
- package/dist/infrastructure/vault-crypto.js +78 -0
- package/dist/mcp/server.js +46 -1
- package/dist/mcp/tools/maintenance-tools.js +2 -2
- package/dist/mcp/tools/read-tools.js +8 -4
- package/dist/mcp/tools/vault-tools.js +163 -0
- package/dist/mcp/tools.js +1 -0
- package/package.json +3 -2
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { stat } from 'node:fs/promises';
|
|
2
2
|
import { ensureVault } from '../infrastructure/file-system-vault.js';
|
|
3
|
-
import { indexStoragePath
|
|
3
|
+
import { indexStoragePath } from '../infrastructure/file-index.js';
|
|
4
|
+
import { openKnowledgeStore } from '../infrastructure/knowledge-store/index.js';
|
|
4
5
|
import { getGraphLayout } from './get-graph-layout.js';
|
|
5
6
|
const graphSearchCacheTtlMs = 20_000;
|
|
6
7
|
const graphSearchCacheMaxEntries = 120;
|
|
@@ -52,7 +53,7 @@ export const searchGraphNodeIds = async (vaultPath, query, limit, agentId, conte
|
|
|
52
53
|
const contextNodeIds = context
|
|
53
54
|
? new Set((await getGraphLayout(absoluteVaultPath, { agentId, context })).layout.nodes.map((node) => node.id))
|
|
54
55
|
: new Set();
|
|
55
|
-
const index =
|
|
56
|
+
const index = openKnowledgeStore(absoluteVaultPath);
|
|
56
57
|
try {
|
|
57
58
|
const searchLimit = context ? Math.max(limit, 5000) : limit;
|
|
58
59
|
const foundNodeIds = await index.searchGraphNodeIds(query, searchLimit, agentId);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { stat } from 'node:fs/promises';
|
|
2
1
|
import { ensureVault } from '../infrastructure/file-system-vault.js';
|
|
3
2
|
import { ensurePrivatePacksFromLegacyIndex, searchInPacks } from '../infrastructure/search-packs.js';
|
|
4
|
-
import {
|
|
3
|
+
import { indexCommitSignature } from '../infrastructure/index-signature.js';
|
|
4
|
+
import { openKnowledgeStore } from '../infrastructure/knowledge-store/index.js';
|
|
5
5
|
import { createEmbeddingProvider } from '../domain/embeddings.js';
|
|
6
6
|
import { loadBrainlinkConfig, sanitizeSearchMode } from '../infrastructure/config.js';
|
|
7
7
|
const embedQuerySafely = async (provider, query) => {
|
|
@@ -17,26 +17,18 @@ const embedQuerySafely = async (provider, query) => {
|
|
|
17
17
|
const hybridCacheTtlMs = 30_000;
|
|
18
18
|
const hybridCacheMaxEntries = 200;
|
|
19
19
|
const hybridSearchCache = new Map();
|
|
20
|
-
const readIndexMtimeMs = async (vaultPath) => {
|
|
21
|
-
try {
|
|
22
|
-
return (await stat(indexStoragePath(vaultPath))).mtimeMs;
|
|
23
|
-
}
|
|
24
|
-
catch {
|
|
25
|
-
return 0;
|
|
26
|
-
}
|
|
27
|
-
};
|
|
28
20
|
const toCacheKey = (vaultPath, query, limit, agentId) => JSON.stringify({
|
|
29
21
|
vaultPath,
|
|
30
22
|
query: query.trim().toLowerCase(),
|
|
31
23
|
limit,
|
|
32
24
|
agentId: agentId?.trim().toLowerCase() ?? '*'
|
|
33
25
|
});
|
|
34
|
-
const cacheGet = (key,
|
|
26
|
+
const cacheGet = (key, indexSignature) => {
|
|
35
27
|
const entry = hybridSearchCache.get(key);
|
|
36
28
|
if (!entry) {
|
|
37
29
|
return undefined;
|
|
38
30
|
}
|
|
39
|
-
const fresh = Date.now() - entry.createdAt <= hybridCacheTtlMs && entry.
|
|
31
|
+
const fresh = Date.now() - entry.createdAt <= hybridCacheTtlMs && entry.indexSignature === indexSignature;
|
|
40
32
|
if (!fresh) {
|
|
41
33
|
hybridSearchCache.delete(key);
|
|
42
34
|
return undefined;
|
|
@@ -58,8 +50,8 @@ export const searchKnowledge = async (vaultPath, query, limit, agentId, mode) =>
|
|
|
58
50
|
const searchMode = sanitizeSearchMode(mode, config.defaultSearchMode);
|
|
59
51
|
await ensurePrivatePacksFromLegacyIndex(absoluteVaultPath);
|
|
60
52
|
const cacheKey = searchMode === 'hybrid' ? toCacheKey(absoluteVaultPath, query, limit, agentId) : undefined;
|
|
61
|
-
const
|
|
62
|
-
const cached = cacheKey ? cacheGet(cacheKey,
|
|
53
|
+
const indexSignature = cacheKey ? await indexCommitSignature(absoluteVaultPath) : '';
|
|
54
|
+
const cached = cacheKey ? cacheGet(cacheKey, indexSignature) : undefined;
|
|
63
55
|
if (cached) {
|
|
64
56
|
return cached;
|
|
65
57
|
}
|
|
@@ -73,14 +65,14 @@ export const searchKnowledge = async (vaultPath, query, limit, agentId, mode) =>
|
|
|
73
65
|
// to lexical (fts) so a transient provider outage still yields results.
|
|
74
66
|
const effectiveMode = searchMode === 'semantic' && shouldEmbedQuery && queryEmbedding.length === 0 ? 'fts' : searchMode;
|
|
75
67
|
try {
|
|
76
|
-
const index =
|
|
68
|
+
const index = openKnowledgeStore(absoluteVaultPath);
|
|
77
69
|
try {
|
|
78
70
|
const results = await index.search(query, limit, agentId, effectiveMode, queryEmbedding);
|
|
79
71
|
if (cacheKey) {
|
|
80
72
|
cacheSet({
|
|
81
73
|
key: cacheKey,
|
|
82
74
|
createdAt: Date.now(),
|
|
83
|
-
|
|
75
|
+
indexSignature,
|
|
84
76
|
results
|
|
85
77
|
});
|
|
86
78
|
}
|
|
@@ -96,7 +88,7 @@ export const searchKnowledge = async (vaultPath, query, limit, agentId, mode) =>
|
|
|
96
88
|
cacheSet({
|
|
97
89
|
key: cacheKey,
|
|
98
90
|
createdAt: Date.now(),
|
|
99
|
-
|
|
91
|
+
indexSignature,
|
|
100
92
|
results: fallbackResults
|
|
101
93
|
});
|
|
102
94
|
}
|
|
@@ -40,6 +40,10 @@ const readContextCacheTtlMs = async (url) => {
|
|
|
40
40
|
const defaults = await readRuntimeDefaults(url);
|
|
41
41
|
return defaults.defaultContextCacheTtlMs;
|
|
42
42
|
};
|
|
43
|
+
const readCagPackTtlMs = async (url) => {
|
|
44
|
+
const defaults = await readRuntimeDefaults(url);
|
|
45
|
+
return defaults.cagPackTtlMs;
|
|
46
|
+
};
|
|
43
47
|
const hasInvalidSearchMode = (url) => {
|
|
44
48
|
const mode = url.searchParams.get('mode');
|
|
45
49
|
return mode !== null && !['fts', 'semantic', 'hybrid'].includes(mode);
|
|
@@ -445,13 +449,14 @@ export const route = async (request, url, vaultPath) => {
|
|
|
445
449
|
const mode = await readSearchMode(url);
|
|
446
450
|
const strategy = await readContextStrategy(url);
|
|
447
451
|
const contextCacheTtlMs = await readContextCacheTtlMs(url);
|
|
452
|
+
const cagPackTtlMs = await readCagPackTtlMs(url);
|
|
448
453
|
if (hasInvalidSearchMode(url)) {
|
|
449
454
|
return createResponse(createJsonResponse({ error: 'Invalid mode. Use fts, semantic or hybrid.' }), 400, contentTypes['.json']);
|
|
450
455
|
}
|
|
451
456
|
if (hasInvalidContextStrategy(url)) {
|
|
452
457
|
return createResponse(createJsonResponse({ error: 'Invalid strategy. Use rag, cag or auto.' }), 400, contentTypes['.json']);
|
|
453
458
|
}
|
|
454
|
-
return createResponse(createJsonResponse(await buildContextPackage(vaultPath, query, limit, tokens, readAgentQuery(url), mode, strategy, contextCacheTtlMs)), 200, contentTypes['.json']);
|
|
459
|
+
return createResponse(createJsonResponse(await buildContextPackage(vaultPath, query, limit, tokens, readAgentQuery(url), mode, strategy, contextCacheTtlMs, cagPackTtlMs)), 200, contentTypes['.json']);
|
|
455
460
|
}
|
|
456
461
|
if (isReadMethod(request) && url.pathname === '/api/links') {
|
|
457
462
|
return createResponse(createJsonResponse({ links: await listLinks(vaultPath, readAgentQuery(url)) }), 200, contentTypes['.json']);
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// Encrypted vault versioning: the git repo tracks `<note>.md.enc` ciphertext
|
|
2
|
+
// (opaque without the local key), while the plaintext `.md` stays the canonical
|
|
3
|
+
// working copy and is git-ignored. Committing refreshes the ciphertext mirror;
|
|
4
|
+
// pulling/cloning/restoring decrypts it back to Markdown before reindexing. Only
|
|
5
|
+
// content that changed is rewritten, so encryption adds no git churn.
|
|
6
|
+
import { readFile, rm, writeFile } from 'node:fs/promises';
|
|
7
|
+
import { extname } from 'node:path';
|
|
8
|
+
import { loadBrainlinkConfig } from '../infrastructure/config.js';
|
|
9
|
+
import { ensureVault, listVaultFiles } from '../infrastructure/file-system-vault.js';
|
|
10
|
+
import { decryptVaultContent, encryptVaultContent, readOrCreateVaultKey } from '../infrastructure/vault-crypto.js';
|
|
11
|
+
const encryptedSuffix = '.md.enc';
|
|
12
|
+
const isMarkdown = (path) => extname(path).toLowerCase() === '.md';
|
|
13
|
+
const isEncryptedMarkdown = (path) => path.toLowerCase().endsWith(encryptedSuffix);
|
|
14
|
+
export const vaultEncryptionEnabled = async () => (await loadBrainlinkConfig()).vaultEncryption === true;
|
|
15
|
+
// Refresh the ciphertext mirror: (re)write `<note>.md.enc` for every Markdown
|
|
16
|
+
// file whose ciphertext is missing or stale, and drop orphan `.md.enc` whose
|
|
17
|
+
// source Markdown was deleted. Deterministic encryption means an unchanged note
|
|
18
|
+
// produces identical bytes, so it is skipped (no write, no git churn).
|
|
19
|
+
export const encryptVaultTree = async (vaultPath) => {
|
|
20
|
+
const root = await ensureVault(vaultPath);
|
|
21
|
+
const key = await readOrCreateVaultKey(root);
|
|
22
|
+
const files = await listVaultFiles(root);
|
|
23
|
+
const markdown = files.filter(isMarkdown);
|
|
24
|
+
const expected = new Set(markdown.map((path) => `${path}.enc`));
|
|
25
|
+
let written = 0;
|
|
26
|
+
for (const source of markdown) {
|
|
27
|
+
const payload = encryptVaultContent(key, await readFile(source));
|
|
28
|
+
const target = `${source}.enc`;
|
|
29
|
+
let current = null;
|
|
30
|
+
try {
|
|
31
|
+
current = await readFile(target);
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
current = null;
|
|
35
|
+
}
|
|
36
|
+
if (!current || !current.equals(payload)) {
|
|
37
|
+
await writeFile(target, payload, { mode: 0o600 });
|
|
38
|
+
written += 1;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
let removed = 0;
|
|
42
|
+
for (const encrypted of files.filter(isEncryptedMarkdown)) {
|
|
43
|
+
if (!expected.has(encrypted)) {
|
|
44
|
+
await rm(encrypted, { force: true });
|
|
45
|
+
removed += 1;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return { written, removed };
|
|
49
|
+
};
|
|
50
|
+
// Rebuild plaintext Markdown from the ciphertext mirror after a checkout. Fails
|
|
51
|
+
// loudly if the key cannot decrypt a payload (wrong/missing key) rather than
|
|
52
|
+
// writing garbage.
|
|
53
|
+
export const decryptVaultTree = async (vaultPath) => {
|
|
54
|
+
const root = await ensureVault(vaultPath);
|
|
55
|
+
const key = await readOrCreateVaultKey(root);
|
|
56
|
+
const encrypted = (await listVaultFiles(root)).filter(isEncryptedMarkdown);
|
|
57
|
+
let written = 0;
|
|
58
|
+
for (const source of encrypted) {
|
|
59
|
+
const plaintext = decryptVaultContent(key, await readFile(source));
|
|
60
|
+
await writeFile(source.slice(0, -'.enc'.length), plaintext, { mode: 0o600 });
|
|
61
|
+
written += 1;
|
|
62
|
+
}
|
|
63
|
+
return { written };
|
|
64
|
+
};
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
// Git primitives for vault versioning, with no dependency on the indexing layer
|
|
2
|
+
// so they can be reused by the auto-version hook that runs *after* indexing
|
|
3
|
+
// (avoiding a cycle with index-vault). Only the canonical Markdown is versioned;
|
|
4
|
+
// the derived `.brainlink/` directory is git-ignored. Git is driven through the
|
|
5
|
+
// system binary via execFile (never a shell), and a missing binary degrades to a
|
|
6
|
+
// clear, actionable error.
|
|
7
|
+
import { execFile } from 'node:child_process';
|
|
8
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
9
|
+
import { join } from 'node:path';
|
|
10
|
+
import { promisify } from 'node:util';
|
|
11
|
+
import { ensureVault } from '../infrastructure/file-system-vault.js';
|
|
12
|
+
import { encryptVaultTree, vaultEncryptionEnabled } from './vault-encryption.js';
|
|
13
|
+
const execFileAsync = promisify(execFile);
|
|
14
|
+
export class GitUnavailableError extends Error {
|
|
15
|
+
constructor() {
|
|
16
|
+
super('git executable not found. Install git to use vault versioning.');
|
|
17
|
+
this.name = 'GitUnavailableError';
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
export class GitCommandError extends Error {
|
|
21
|
+
constructor(args, stderr) {
|
|
22
|
+
super(`git ${args.join(' ')} failed: ${stderr.trim() || 'unknown error'}`);
|
|
23
|
+
this.name = 'GitCommandError';
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
const isMissingBinary = (error) => error instanceof Error && 'code' in error && error.code === 'ENOENT';
|
|
27
|
+
export const runGit = async (cwd, args) => {
|
|
28
|
+
try {
|
|
29
|
+
const { stdout } = await execFileAsync('git', [...args], { cwd, maxBuffer: 64 * 1024 * 1024 });
|
|
30
|
+
return stdout;
|
|
31
|
+
}
|
|
32
|
+
catch (error) {
|
|
33
|
+
if (isMissingBinary(error)) {
|
|
34
|
+
throw new GitUnavailableError();
|
|
35
|
+
}
|
|
36
|
+
const stderr = error.stderr ?? (error instanceof Error ? error.message : String(error));
|
|
37
|
+
throw new GitCommandError(args, stderr);
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
const isInsideGitRepo = async (cwd) => {
|
|
41
|
+
try {
|
|
42
|
+
const out = await runGit(cwd, ['rev-parse', '--is-inside-work-tree']);
|
|
43
|
+
return out.trim() === 'true';
|
|
44
|
+
}
|
|
45
|
+
catch (error) {
|
|
46
|
+
if (error instanceof GitUnavailableError) {
|
|
47
|
+
throw error;
|
|
48
|
+
}
|
|
49
|
+
return false;
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
const baseGitignoreLines = ['.brainlink/', '*.tmp', '.DS_Store'];
|
|
53
|
+
// In encrypted mode the plaintext Markdown is never committed — only the
|
|
54
|
+
// `<note>.md.enc` ciphertext is — so the working `.md` files are git-ignored.
|
|
55
|
+
const encryptedGitignoreLines = [...baseGitignoreLines, '*.md'];
|
|
56
|
+
const ensureGitignore = async (vaultRoot, lines) => {
|
|
57
|
+
const path = join(vaultRoot, '.gitignore');
|
|
58
|
+
let current = '';
|
|
59
|
+
try {
|
|
60
|
+
current = await readFile(path, 'utf8');
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
current = '';
|
|
64
|
+
}
|
|
65
|
+
const existing = new Set(current.split('\n').map((line) => line.trim()));
|
|
66
|
+
const missing = lines.filter((line) => !existing.has(line));
|
|
67
|
+
if (missing.length === 0 && current.length > 0) {
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
const next = `${[current.trimEnd(), ...missing].filter(Boolean).join('\n')}\n`;
|
|
71
|
+
await writeFile(path, next, 'utf8');
|
|
72
|
+
};
|
|
73
|
+
// Reject credentials embedded in a remote URL: they would be written to
|
|
74
|
+
// .git/config in clear text. Users should rely on SSH or a git credential
|
|
75
|
+
// helper instead.
|
|
76
|
+
export const assertSafeRemote = (remote) => {
|
|
77
|
+
if (/\/\/[^/@]*:[^/@]*@/.test(remote) || /x-access-token|:[^/]*@github/.test(remote)) {
|
|
78
|
+
throw new Error('Refusing a remote URL with an inline credential. Use SSH or a git credential helper.');
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
export const markdownOnly = (paths) => paths.filter((path) => path.toLowerCase().endsWith('.md'));
|
|
82
|
+
// True when the repo has an 'origin' remote, so auto-versioning knows whether a
|
|
83
|
+
// push is possible.
|
|
84
|
+
export const hasRemote = async (root, remote = 'origin') => {
|
|
85
|
+
const remotes = await runGit(root, ['remote']).catch(() => '');
|
|
86
|
+
return remotes.split('\n').map((line) => line.trim()).includes(remote);
|
|
87
|
+
};
|
|
88
|
+
export const initVaultGit = async (vaultPath, options = {}) => {
|
|
89
|
+
const root = await ensureVault(vaultPath);
|
|
90
|
+
await mkdir(root, { recursive: true });
|
|
91
|
+
const already = await isInsideGitRepo(root);
|
|
92
|
+
if (!already) {
|
|
93
|
+
await runGit(root, ['init']);
|
|
94
|
+
}
|
|
95
|
+
await ensureGitignore(root, (await vaultEncryptionEnabled()) ? encryptedGitignoreLines : baseGitignoreLines);
|
|
96
|
+
if (options.remote) {
|
|
97
|
+
assertSafeRemote(options.remote);
|
|
98
|
+
const verb = (await hasRemote(root)) ? 'set-url' : 'add';
|
|
99
|
+
await runGit(root, ['remote', verb, 'origin', options.remote]);
|
|
100
|
+
}
|
|
101
|
+
return { root, initialized: !already };
|
|
102
|
+
};
|
|
103
|
+
export const vaultGitStatus = async (vaultPath) => {
|
|
104
|
+
const root = await ensureVault(vaultPath);
|
|
105
|
+
const branch = (await runGit(root, ['rev-parse', '--abbrev-ref', 'HEAD']).catch(() => 'HEAD')).trim();
|
|
106
|
+
const porcelain = await runGit(root, ['status', '--porcelain']);
|
|
107
|
+
const changed = porcelain
|
|
108
|
+
.split('\n')
|
|
109
|
+
.map((line) => line.slice(3).trim())
|
|
110
|
+
.filter(Boolean);
|
|
111
|
+
return { branch, dirty: changed.length > 0, changedMarkdown: markdownOnly(changed) };
|
|
112
|
+
};
|
|
113
|
+
// Stage and commit the versioned content. Returns null when there is nothing to
|
|
114
|
+
// commit. In encrypted mode the ciphertext mirror is refreshed first and only
|
|
115
|
+
// `<note>.md.enc` is staged; otherwise the plaintext Markdown is staged.
|
|
116
|
+
export const commitVault = async (vaultPath, message) => {
|
|
117
|
+
const root = await ensureVault(vaultPath);
|
|
118
|
+
const encrypted = await vaultEncryptionEnabled();
|
|
119
|
+
if (encrypted) {
|
|
120
|
+
await encryptVaultTree(root);
|
|
121
|
+
await ensureGitignore(root, encryptedGitignoreLines);
|
|
122
|
+
}
|
|
123
|
+
const patterns = encrypted
|
|
124
|
+
? ['*.md.enc', ':(glob)**/*.md.enc', '.gitignore']
|
|
125
|
+
: ['*.md', ':(glob)**/*.md', '.gitignore'];
|
|
126
|
+
await runGit(root, ['add', '-A', '--', ...patterns]);
|
|
127
|
+
const staged = await runGit(root, ['diff', '--cached', '--name-only']);
|
|
128
|
+
if (staged.trim().length === 0) {
|
|
129
|
+
return null;
|
|
130
|
+
}
|
|
131
|
+
await runGit(root, ['commit', '-m', message]);
|
|
132
|
+
return (await runGit(root, ['rev-parse', 'HEAD'])).trim();
|
|
133
|
+
};
|
|
134
|
+
export const pushVault = async (vaultPath, branch) => {
|
|
135
|
+
const root = await ensureVault(vaultPath);
|
|
136
|
+
const target = branch ?? (await runGit(root, ['rev-parse', '--abbrev-ref', 'HEAD'])).trim();
|
|
137
|
+
await runGit(root, ['push', '-u', 'origin', target]);
|
|
138
|
+
};
|
|
139
|
+
export const vaultHistory = async (vaultPath, limit = 20) => {
|
|
140
|
+
const root = await ensureVault(vaultPath);
|
|
141
|
+
const out = await runGit(root, [
|
|
142
|
+
'log',
|
|
143
|
+
`-n${Math.max(1, Math.floor(limit))}`,
|
|
144
|
+
'--format=%H%x00%an%x00%aI%x00%s',
|
|
145
|
+
'--',
|
|
146
|
+
'*.md'
|
|
147
|
+
]).catch(() => '');
|
|
148
|
+
return out
|
|
149
|
+
.split('\n')
|
|
150
|
+
.map((line) => line.split('\u0000'))
|
|
151
|
+
.filter((parts) => parts.length === 4)
|
|
152
|
+
.map(([hash, author, date, subject]) => ({ hash: hash, author: author, date: date, subject: subject }));
|
|
153
|
+
};
|
|
154
|
+
// Best-effort commit + push of the canonical Markdown, used by the auto-version
|
|
155
|
+
// hook after a successful index. A no-op unless the vault is a git repo with an
|
|
156
|
+
// origin remote. Commits only when Markdown actually changed.
|
|
157
|
+
export const autoVersionVault = async (vaultPath, message) => {
|
|
158
|
+
const root = await ensureVault(vaultPath);
|
|
159
|
+
if (!(await isInsideGitRepo(root)) || !(await hasRemote(root))) {
|
|
160
|
+
return { committed: null, pushed: false };
|
|
161
|
+
}
|
|
162
|
+
const committed = await commitVault(vaultPath, message);
|
|
163
|
+
if (!committed) {
|
|
164
|
+
return { committed: null, pushed: false };
|
|
165
|
+
}
|
|
166
|
+
await pushVault(vaultPath);
|
|
167
|
+
return { committed, pushed: true };
|
|
168
|
+
};
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// Vault versioning operations that reindex after changing Markdown on disk
|
|
2
|
+
// (pull, restore, clone) so the vault stays immediately usable ("no losing the
|
|
3
|
+
// use of the data"). The git primitives live in vault-git-core (which has no
|
|
4
|
+
// indexing dependency) and are re-exported here so callers keep a single import.
|
|
5
|
+
import { indexVault } from './index-vault.js';
|
|
6
|
+
import { ensureVault } from '../infrastructure/file-system-vault.js';
|
|
7
|
+
import { decryptVaultTree, vaultEncryptionEnabled } from './vault-encryption.js';
|
|
8
|
+
import { assertSafeRemote, GitCommandError, markdownOnly, runGit } from './vault-git-core.js';
|
|
9
|
+
export * from './vault-git-core.js';
|
|
10
|
+
const reindexAfterMarkdownChange = async (vaultPath, changedMarkdown, skipIndex) => !skipIndex && changedMarkdown.length > 0 ? indexVault(vaultPath) : undefined;
|
|
11
|
+
// Pull from origin and reindex when Markdown changed, so the vault is usable
|
|
12
|
+
// immediately. On a merge conflict the merge is aborted and a clear error is
|
|
13
|
+
// raised rather than leaving conflict markers in the Markdown.
|
|
14
|
+
export const pullVault = async (vaultPath, options = {}) => {
|
|
15
|
+
const root = await ensureVault(vaultPath);
|
|
16
|
+
const before = (await runGit(root, ['rev-parse', 'HEAD']).catch(() => '')).trim();
|
|
17
|
+
try {
|
|
18
|
+
await runGit(root, ['pull', '--no-rebase', '--no-edit', 'origin']);
|
|
19
|
+
}
|
|
20
|
+
catch (error) {
|
|
21
|
+
await runGit(root, ['merge', '--abort']).catch(() => undefined);
|
|
22
|
+
throw error instanceof GitCommandError
|
|
23
|
+
? new GitCommandError(['pull'], 'merge conflict; aborted. Resolve manually, then commit.')
|
|
24
|
+
: error;
|
|
25
|
+
}
|
|
26
|
+
const after = (await runGit(root, ['rev-parse', 'HEAD']).catch(() => '')).trim();
|
|
27
|
+
const changed = Boolean(before && after && before !== after);
|
|
28
|
+
// Encrypted vaults track ciphertext; rebuild the plaintext Markdown from it
|
|
29
|
+
// before reindexing. Individual filenames are not reported (the diff is over
|
|
30
|
+
// opaque `.md.enc`).
|
|
31
|
+
if (await vaultEncryptionEnabled()) {
|
|
32
|
+
if (changed) {
|
|
33
|
+
await decryptVaultTree(root);
|
|
34
|
+
}
|
|
35
|
+
const index = changed ? await reindexAfterMarkdownChange(vaultPath, ['(encrypted)'], options.skipIndex === true) : undefined;
|
|
36
|
+
return { changedMarkdown: changed ? ['(encrypted)'] : [], index };
|
|
37
|
+
}
|
|
38
|
+
const diff = changed ? await runGit(root, ['diff', '--name-only', before, after, '--', '*.md']) : '';
|
|
39
|
+
const changedMarkdown = markdownOnly(diff.split('\n').map((line) => line.trim()).filter(Boolean));
|
|
40
|
+
const index = await reindexAfterMarkdownChange(vaultPath, changedMarkdown, options.skipIndex === true);
|
|
41
|
+
return { changedMarkdown, index };
|
|
42
|
+
};
|
|
43
|
+
export const restoreVault = async (vaultPath, ref, options = {}) => {
|
|
44
|
+
const root = await ensureVault(vaultPath);
|
|
45
|
+
const encrypted = await vaultEncryptionEnabled();
|
|
46
|
+
const diff = await runGit(root, ['diff', '--name-only', ref, '--', encrypted ? '*.md.enc' : '*.md']).catch(() => '');
|
|
47
|
+
await runGit(root, ['checkout', ref, '--', '.']);
|
|
48
|
+
// Restore is an explicit request to materialize this ref's Markdown, so in
|
|
49
|
+
// encrypted mode always rebuild the plaintext from the (now checked-out)
|
|
50
|
+
// ciphertext — the plaintext is derived and may be missing or stale even when
|
|
51
|
+
// the ciphertext already matched the ref.
|
|
52
|
+
if (encrypted) {
|
|
53
|
+
const rebuilt = await decryptVaultTree(root);
|
|
54
|
+
const index = await reindexAfterMarkdownChange(vaultPath, rebuilt.written > 0 ? ['(encrypted)'] : [], options.skipIndex === true);
|
|
55
|
+
return { changedMarkdown: rebuilt.written > 0 ? ['(encrypted)'] : [], index };
|
|
56
|
+
}
|
|
57
|
+
const changedMarkdown = markdownOnly(diff.split('\n').map((line) => line.trim()).filter(Boolean));
|
|
58
|
+
const index = await reindexAfterMarkdownChange(vaultPath, changedMarkdown, options.skipIndex === true);
|
|
59
|
+
return { changedMarkdown, index };
|
|
60
|
+
};
|
|
61
|
+
// Clone a remote vault and build its index from scratch so it is immediately
|
|
62
|
+
// searchable. The clone target must not already exist.
|
|
63
|
+
export const cloneVault = async (remote, targetPath, options = {}) => {
|
|
64
|
+
assertSafeRemote(remote);
|
|
65
|
+
await runGit(process.cwd(), ['clone', remote, targetPath]);
|
|
66
|
+
const root = await ensureVault(targetPath);
|
|
67
|
+
// Encrypted clone: decrypt the ciphertext mirror before indexing. Requires the
|
|
68
|
+
// vault key on this machine (BRAINLINK_VAULT_KEY or a copied key file);
|
|
69
|
+
// decryption throws a clear error if the key is missing or wrong.
|
|
70
|
+
if (await vaultEncryptionEnabled()) {
|
|
71
|
+
await decryptVaultTree(root);
|
|
72
|
+
}
|
|
73
|
+
const index = options.skipIndex === true ? undefined : await indexVault(root, { full: true });
|
|
74
|
+
return { root, index };
|
|
75
|
+
};
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
// Portable vault snapshot (`.blvault`): a single gzip-compressed envelope of the
|
|
2
|
+
// canonical Markdown (with per-file SHA-256), self-contained and movable to any
|
|
3
|
+
// machine. The derived index is never included — import rebuilds it so the vault
|
|
4
|
+
// is immediately usable. Conflicts on import are preserved as `*.conflict-<ts>.md`
|
|
5
|
+
// rather than overwriting local edits.
|
|
6
|
+
import { createHash } from 'node:crypto';
|
|
7
|
+
import { gzipSync, gunzipSync } from 'node:zlib';
|
|
8
|
+
import { extname, relative } from 'node:path';
|
|
9
|
+
import { readFile, writeFile } from 'node:fs/promises';
|
|
10
|
+
import { indexVault } from './index-vault.js';
|
|
11
|
+
import { ensureVault, listVaultFiles, writeMarkdownFile } from '../infrastructure/file-system-vault.js';
|
|
12
|
+
const snapshotVersion = 1;
|
|
13
|
+
const isMarkdown = (path) => extname(path).toLowerCase() === '.md';
|
|
14
|
+
const toPosix = (path) => path.split('\\').join('/');
|
|
15
|
+
const sha256 = (data) => createHash('sha256').update(data).digest('hex');
|
|
16
|
+
const conflictName = (path) => {
|
|
17
|
+
const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\..+$/, 'Z');
|
|
18
|
+
const extension = extname(path);
|
|
19
|
+
const base = extension ? path.slice(0, -extension.length) : path;
|
|
20
|
+
return `${base}.conflict-${stamp}${extension}`;
|
|
21
|
+
};
|
|
22
|
+
export const exportVaultSnapshot = async (vaultPath, outputFile) => {
|
|
23
|
+
const root = await ensureVault(vaultPath);
|
|
24
|
+
const markdownFiles = (await listVaultFiles(root)).filter(isMarkdown);
|
|
25
|
+
const entries = await Promise.all(markdownFiles.map(async (absolutePath) => {
|
|
26
|
+
const content = await readFile(absolutePath);
|
|
27
|
+
const path = toPosix(relative(root, absolutePath));
|
|
28
|
+
return {
|
|
29
|
+
manifest: { path, sha256: sha256(content), bytes: content.byteLength },
|
|
30
|
+
file: { path, contentB64: content.toString('base64') }
|
|
31
|
+
};
|
|
32
|
+
}));
|
|
33
|
+
const envelope = {
|
|
34
|
+
version: snapshotVersion,
|
|
35
|
+
createdAt: new Date().toISOString(),
|
|
36
|
+
manifest: entries.map((entry) => entry.manifest),
|
|
37
|
+
files: entries.map((entry) => entry.file)
|
|
38
|
+
};
|
|
39
|
+
const packed = gzipSync(Buffer.from(JSON.stringify(envelope), 'utf8'), { level: 9 });
|
|
40
|
+
await writeFile(outputFile, packed, { mode: 0o600 });
|
|
41
|
+
return { file: outputFile, fileCount: entries.length, bytes: packed.byteLength };
|
|
42
|
+
};
|
|
43
|
+
const parseEnvelope = (raw) => {
|
|
44
|
+
const envelope = JSON.parse(gunzipSync(raw).toString('utf8'));
|
|
45
|
+
if (!envelope || envelope.version !== snapshotVersion || !Array.isArray(envelope.files)) {
|
|
46
|
+
throw new Error('Unrecognized .blvault snapshot format.');
|
|
47
|
+
}
|
|
48
|
+
return envelope;
|
|
49
|
+
};
|
|
50
|
+
export const importVaultSnapshot = async (snapshotFile, vaultPath, options = {}) => {
|
|
51
|
+
const root = await ensureVault(vaultPath);
|
|
52
|
+
const envelope = parseEnvelope(await readFile(snapshotFile));
|
|
53
|
+
const manifestByPath = new Map(envelope.manifest.map((entry) => [entry.path, entry]));
|
|
54
|
+
let imported = 0;
|
|
55
|
+
let conflicted = 0;
|
|
56
|
+
for (const file of envelope.files) {
|
|
57
|
+
if (!isMarkdown(file.path)) {
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
const content = Buffer.from(file.contentB64, 'base64');
|
|
61
|
+
const expected = manifestByPath.get(file.path)?.sha256;
|
|
62
|
+
if (expected && sha256(content) !== expected) {
|
|
63
|
+
throw new Error(`Checksum mismatch for ${file.path}; snapshot is corrupt.`);
|
|
64
|
+
}
|
|
65
|
+
// Preserve local edits: write divergent incoming files under a conflict name
|
|
66
|
+
// instead of overwriting. writeMarkdownFile guards against path traversal.
|
|
67
|
+
let existing = null;
|
|
68
|
+
try {
|
|
69
|
+
existing = await readFile(`${root}/${file.path}`);
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
existing = null;
|
|
73
|
+
}
|
|
74
|
+
if (existing && !existing.equals(content)) {
|
|
75
|
+
await writeMarkdownFile(root, conflictName(file.path), content.toString('utf8'));
|
|
76
|
+
conflicted += 1;
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
if (existing && existing.equals(content)) {
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
await writeMarkdownFile(root, file.path, content.toString('utf8'));
|
|
83
|
+
imported += 1;
|
|
84
|
+
}
|
|
85
|
+
const index = !options.skipIndex && imported + conflicted > 0 ? await indexVault(root) : undefined;
|
|
86
|
+
return { imported, conflicted, index };
|
|
87
|
+
};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { getBrokenLinksReport, getExtendedStats, getOrphansReport, getStats, validateVault } from '../../application/analyze-vault.js';
|
|
2
|
-
import { buildContextPackage, readContextDataSignature } from '../../application/build-context.js';
|
|
2
|
+
import { buildContextPackage, readContextDataSignature, readVolatileSignature } from '../../application/build-context.js';
|
|
3
3
|
import { getGraph } from '../../application/get-graph.js';
|
|
4
4
|
import { listAgents } from '../../application/list-agents.js';
|
|
5
5
|
import { listBacklinks, listLinks } from '../../application/list-links.js';
|
|
@@ -82,24 +82,28 @@ export const registerReadCommands = (program) => {
|
|
|
82
82
|
const resolved = await resolveOptions(options);
|
|
83
83
|
const mode = sanitizeSearchMode(options.mode, resolved.defaults.defaultSearchMode);
|
|
84
84
|
const strategy = sanitizeContextStrategy(options.strategy, resolved.defaults.defaultContextStrategy);
|
|
85
|
-
const contextPackage = await buildContextPackage(resolved.vault, query, parsePositiveInteger(options.limit ?? String(resolved.defaults.defaultSearchLimit), resolved.defaults.defaultSearchLimit), parsePositiveInteger(options.tokens ?? String(resolved.defaults.defaultContextTokens), resolved.defaults.defaultContextTokens), resolved.agent, mode, strategy, resolved.defaults.defaultContextCacheTtlMs);
|
|
85
|
+
const contextPackage = await buildContextPackage(resolved.vault, query, parsePositiveInteger(options.limit ?? String(resolved.defaults.defaultSearchLimit), resolved.defaults.defaultSearchLimit), parsePositiveInteger(options.tokens ?? String(resolved.defaults.defaultContextTokens), resolved.defaults.defaultContextTokens), resolved.agent, mode, strategy, resolved.defaults.defaultContextCacheTtlMs, resolved.defaults.cagPackTtlMs);
|
|
86
86
|
print(options.json, contextPackage, () => contextPackage.content);
|
|
87
87
|
});
|
|
88
88
|
program
|
|
89
89
|
.command('context-packs')
|
|
90
90
|
.option('-v, --vault <vault>', 'vault directory')
|
|
91
91
|
.option('-a, --agent <agent>', 'accepted for consistency; context packs are already keyed by agent')
|
|
92
|
-
.option('--stale', 'operate only on packs
|
|
92
|
+
.option('--stale', 'operate only on packs whose cited documents (or volatile memory) have changed')
|
|
93
93
|
.option('--clear', 'remove context packs instead of listing them')
|
|
94
94
|
.option('--json', 'print machine-readable JSON')
|
|
95
95
|
.description('list or clear persisted CAG context packs')
|
|
96
96
|
.action(async (options) => {
|
|
97
97
|
const resolved = await resolveOptions(options);
|
|
98
98
|
const dataSignature = await readContextDataSignature(resolved.vault);
|
|
99
|
+
const freshness = {
|
|
100
|
+
volatileSignature: await readVolatileSignature(resolved.vault),
|
|
101
|
+
ttlMs: resolved.defaults.cagPackTtlMs
|
|
102
|
+
};
|
|
99
103
|
if (options.clear) {
|
|
100
104
|
const result = await clearContextPacks(resolved.vault, {
|
|
101
105
|
staleOnly: options.stale === true,
|
|
102
|
-
|
|
106
|
+
freshness
|
|
103
107
|
});
|
|
104
108
|
print(options.json, { vault: resolved.vault, dataSignature, ...result }, () => [
|
|
105
109
|
`Removed context packs: ${result.removed.length}`,
|
|
@@ -107,7 +111,7 @@ export const registerReadCommands = (program) => {
|
|
|
107
111
|
].join('\n'));
|
|
108
112
|
return;
|
|
109
113
|
}
|
|
110
|
-
const packs = await listContextPacks(resolved.vault,
|
|
114
|
+
const packs = await listContextPacks(resolved.vault, freshness);
|
|
111
115
|
const visiblePacks = options.stale ? packs.filter((pack) => pack.stale) : packs;
|
|
112
116
|
print(options.json, { vault: resolved.vault, dataSignature, packs: visiblePacks }, () => visiblePacks.length === 0
|
|
113
117
|
? 'No context packs found.'
|