mcp-memory-bucket 0.7.4 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -0
- package/dist/client/assets/{index-Cr5jNKWL.js → index-BfU_uQyK.js} +397 -262
- package/dist/client/index.html +7 -1
- package/dist/src/attachments/repository.js +22 -22
- package/dist/src/attachments/tools.js +12 -12
- package/dist/src/config.js +70 -3
- package/dist/src/memory/repository.js +142 -39
- package/dist/src/memory/tools.js +32 -13
- package/dist/src/remote/credentials.js +31 -0
- package/dist/src/remote/folderfoo-client.js +149 -0
- package/dist/src/remote/remote-sync.js +176 -0
- package/dist/src/server.js +18 -3
- package/dist/src/shared/body-edits.js +87 -0
- package/dist/src/shared/relocate-tool.js +2 -2
- package/dist/src/shared/relocate.js +9 -5
- package/dist/src/skills/repository.js +141 -47
- package/dist/src/skills/tools.js +34 -15
- package/dist/src/store/sync.js +19 -2
- package/dist/src/web/routes.js +220 -25
- package/package.json +1 -1
package/dist/client/index.html
CHANGED
|
@@ -26,8 +26,14 @@
|
|
|
26
26
|
:root[data-theme='light'] { color-scheme: light; }
|
|
27
27
|
:root[data-theme='dark'] { color-scheme: dark; }
|
|
28
28
|
body { font-family: system-ui, sans-serif; margin: 0; background: var(--bg); color: var(--fg); }
|
|
29
|
+
folderfoo-profile-circle {
|
|
30
|
+
position: fixed;
|
|
31
|
+
top: 16px;
|
|
32
|
+
right: 16px;
|
|
33
|
+
z-index: 2147483000;
|
|
34
|
+
}
|
|
29
35
|
</style>
|
|
30
|
-
<script type="module" crossorigin src="/assets/index-
|
|
36
|
+
<script type="module" crossorigin src="/assets/index-BfU_uQyK.js"></script>
|
|
31
37
|
</head>
|
|
32
38
|
<body>
|
|
33
39
|
<mem-bucket-app></mem-bucket-app>
|
|
@@ -9,34 +9,34 @@ export class AttachmentRepository {
|
|
|
9
9
|
this.memoryRepo = memoryRepo;
|
|
10
10
|
this.skillRepo = skillRepo;
|
|
11
11
|
}
|
|
12
|
-
getDoc(kind, docIdOrName) {
|
|
13
|
-
const doc = kind === 'memory' ? this.memoryRepo.get(docIdOrName) : this.skillRepo.get(docIdOrName);
|
|
12
|
+
async getDoc(kind, docIdOrName) {
|
|
13
|
+
const doc = kind === 'memory' ? await this.memoryRepo.get(docIdOrName) : await this.skillRepo.get(docIdOrName);
|
|
14
14
|
if (!doc)
|
|
15
15
|
throw new Error(`${kind} doc "${docIdOrName}" not found`);
|
|
16
16
|
return doc;
|
|
17
17
|
}
|
|
18
|
-
saveAttachmentsList(kind, docIdOrName, attachments) {
|
|
18
|
+
async saveAttachmentsList(kind, docIdOrName, attachments) {
|
|
19
19
|
if (kind === 'memory') {
|
|
20
|
-
this.memoryRepo.update(docIdOrName, { attachments });
|
|
20
|
+
await this.memoryRepo.update(docIdOrName, { attachments });
|
|
21
21
|
}
|
|
22
22
|
else {
|
|
23
|
-
this.skillRepo.update(docIdOrName, { attachments });
|
|
23
|
+
await this.skillRepo.update(docIdOrName, { attachments });
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
|
-
add(kind, docIdOrName, filename, data) {
|
|
27
|
-
const doc = this.getDoc(kind, docIdOrName);
|
|
26
|
+
async add(kind, docIdOrName, filename, data) {
|
|
27
|
+
const doc = await this.getDoc(kind, docIdOrName);
|
|
28
28
|
const dir = attachmentsDirFor(doc.source_path, kind);
|
|
29
29
|
const entry = writeAttachmentFile(dir, filename, data);
|
|
30
30
|
const existing = doc.attachments ?? [];
|
|
31
|
-
this.saveAttachmentsList(kind, docIdOrName, [...existing, entry]);
|
|
31
|
+
await this.saveAttachmentsList(kind, docIdOrName, [...existing, entry]);
|
|
32
32
|
return entry;
|
|
33
33
|
}
|
|
34
|
-
get(kind, docIdOrName, filename) {
|
|
35
|
-
const doc = this.getDoc(kind, docIdOrName);
|
|
34
|
+
async get(kind, docIdOrName, filename) {
|
|
35
|
+
const doc = await this.getDoc(kind, docIdOrName);
|
|
36
36
|
return (doc.attachments ?? []).find((a) => a.filename === filename);
|
|
37
37
|
}
|
|
38
|
-
update(kind, docIdOrName, filename, data) {
|
|
39
|
-
const doc = this.getDoc(kind, docIdOrName);
|
|
38
|
+
async update(kind, docIdOrName, filename, data) {
|
|
39
|
+
const doc = await this.getDoc(kind, docIdOrName);
|
|
40
40
|
const dir = attachmentsDirFor(doc.source_path, kind);
|
|
41
41
|
const existingEntry = (doc.attachments ?? []).find((a) => a.filename === filename);
|
|
42
42
|
if (!existingEntry)
|
|
@@ -46,16 +46,16 @@ export class AttachmentRepository {
|
|
|
46
46
|
const written = writeAttachmentFile(dir, filename, data);
|
|
47
47
|
// Full replace: preserve position, overwrite metadata for this filename entry.
|
|
48
48
|
const next = (doc.attachments ?? []).map((a) => (a.filename === filename ? written : a));
|
|
49
|
-
this.saveAttachmentsList(kind, docIdOrName, next);
|
|
49
|
+
await this.saveAttachmentsList(kind, docIdOrName, next);
|
|
50
50
|
return written;
|
|
51
51
|
}
|
|
52
|
-
remove(kind, docIdOrName, filename) {
|
|
53
|
-
const doc = this.getDoc(kind, docIdOrName);
|
|
52
|
+
async remove(kind, docIdOrName, filename) {
|
|
53
|
+
const doc = await this.getDoc(kind, docIdOrName);
|
|
54
54
|
const dir = attachmentsDirFor(doc.source_path, kind);
|
|
55
55
|
const safePath = resolveWithinBase(dir, undefined, filename);
|
|
56
56
|
fs.rmSync(safePath, { force: true });
|
|
57
57
|
const next = (doc.attachments ?? []).filter((a) => a.filename !== filename);
|
|
58
|
-
this.saveAttachmentsList(kind, docIdOrName, next);
|
|
58
|
+
await this.saveAttachmentsList(kind, docIdOrName, next);
|
|
59
59
|
if (listAttachmentFiles(dir).length === 0) {
|
|
60
60
|
fs.rmSync(dir, { recursive: true, force: true });
|
|
61
61
|
if (kind === 'memory') {
|
|
@@ -68,16 +68,16 @@ export class AttachmentRepository {
|
|
|
68
68
|
}
|
|
69
69
|
}
|
|
70
70
|
}
|
|
71
|
-
list(kind, docIdOrName) {
|
|
72
|
-
return this.getDoc(kind, docIdOrName).attachments ?? [];
|
|
71
|
+
async list(kind, docIdOrName) {
|
|
72
|
+
return (await this.getDoc(kind, docIdOrName)).attachments ?? [];
|
|
73
73
|
}
|
|
74
74
|
/** Absolute filesystem path for an attachment's file, for callers (e.g. MCP tools) that need to Read it. */
|
|
75
|
-
absolutePathFor(kind, docIdOrName, filename) {
|
|
76
|
-
const doc = this.getDoc(kind, docIdOrName);
|
|
75
|
+
async absolutePathFor(kind, docIdOrName, filename) {
|
|
76
|
+
const doc = await this.getDoc(kind, docIdOrName);
|
|
77
77
|
return path.join(attachmentsDirFor(doc.source_path, kind), filename);
|
|
78
78
|
}
|
|
79
|
-
reconcile(kind, docIdOrName) {
|
|
80
|
-
const doc = this.getDoc(kind, docIdOrName);
|
|
79
|
+
async reconcile(kind, docIdOrName) {
|
|
80
|
+
const doc = await this.getDoc(kind, docIdOrName);
|
|
81
81
|
const dir = attachmentsDirFor(doc.source_path, kind);
|
|
82
82
|
const declared = new Set((doc.attachments ?? []).map((a) => a.filename));
|
|
83
83
|
const onDisk = new Set(listAttachmentFiles(dir));
|
|
@@ -19,8 +19,8 @@ export function registerAttachmentTools(mcp, attachRepo) {
|
|
|
19
19
|
try {
|
|
20
20
|
assertFileSizeOk(file_path);
|
|
21
21
|
const data = fs.readFileSync(file_path);
|
|
22
|
-
const entry = attachRepo.add(kind, doc, filename, data);
|
|
23
|
-
const absolute_path = attachRepo.absolutePathFor(kind, doc, entry.filename);
|
|
22
|
+
const entry = await attachRepo.add(kind, doc, filename, data);
|
|
23
|
+
const absolute_path = await attachRepo.absolutePathFor(kind, doc, entry.filename);
|
|
24
24
|
return { content: [{ type: 'text', text: JSON.stringify({ ...entry, absolute_path }, null, 2) }] };
|
|
25
25
|
}
|
|
26
26
|
catch (err) {
|
|
@@ -29,10 +29,10 @@ export function registerAttachmentTools(mcp, attachRepo) {
|
|
|
29
29
|
});
|
|
30
30
|
mcp.tool('attachment_get', 'Returns the on-disk path (not content) of an attachment, plus its metadata. Use Read on the returned absolute_path for the content.', { kind: kindSchema, doc: z.string(), filename: z.string() }, async ({ kind, doc, filename }) => {
|
|
31
31
|
try {
|
|
32
|
-
const entry = attachRepo.get(kind, doc, filename);
|
|
32
|
+
const entry = await attachRepo.get(kind, doc, filename);
|
|
33
33
|
if (!entry)
|
|
34
34
|
throw new Error(`attachment "${filename}" not found`);
|
|
35
|
-
const absolute_path = attachRepo.absolutePathFor(kind, doc, entry.filename);
|
|
35
|
+
const absolute_path = await attachRepo.absolutePathFor(kind, doc, entry.filename);
|
|
36
36
|
return { content: [{ type: 'text', text: JSON.stringify({ ...entry, absolute_path }, null, 2) }] };
|
|
37
37
|
}
|
|
38
38
|
catch (err) {
|
|
@@ -43,8 +43,8 @@ export function registerAttachmentTools(mcp, attachRepo) {
|
|
|
43
43
|
try {
|
|
44
44
|
assertFileSizeOk(file_path);
|
|
45
45
|
const data = fs.readFileSync(file_path);
|
|
46
|
-
const entry = attachRepo.update(kind, doc, filename, data);
|
|
47
|
-
const absolute_path = attachRepo.absolutePathFor(kind, doc, entry.filename);
|
|
46
|
+
const entry = await attachRepo.update(kind, doc, filename, data);
|
|
47
|
+
const absolute_path = await attachRepo.absolutePathFor(kind, doc, entry.filename);
|
|
48
48
|
return { content: [{ type: 'text', text: JSON.stringify({ ...entry, absolute_path }, null, 2) }] };
|
|
49
49
|
}
|
|
50
50
|
catch (err) {
|
|
@@ -53,7 +53,7 @@ export function registerAttachmentTools(mcp, attachRepo) {
|
|
|
53
53
|
});
|
|
54
54
|
mcp.tool('attachment_remove', 'Deletes an attachment from a memory doc or skill.', { kind: kindSchema, doc: z.string(), filename: z.string() }, async ({ kind, doc, filename }) => {
|
|
55
55
|
try {
|
|
56
|
-
attachRepo.remove(kind, doc, filename);
|
|
56
|
+
await attachRepo.remove(kind, doc, filename);
|
|
57
57
|
return { content: [{ type: 'text', text: `removed "${filename}"` }] };
|
|
58
58
|
}
|
|
59
59
|
catch (err) {
|
|
@@ -62,11 +62,11 @@ export function registerAttachmentTools(mcp, attachRepo) {
|
|
|
62
62
|
});
|
|
63
63
|
mcp.tool('attachment_list', 'Lists all attachments on a memory doc or skill, each with its absolute on-disk path.', { kind: kindSchema, doc: z.string() }, async ({ kind, doc }) => {
|
|
64
64
|
try {
|
|
65
|
-
const entries = attachRepo.list(kind, doc);
|
|
66
|
-
const withPaths = entries.map((entry) => ({
|
|
65
|
+
const entries = await attachRepo.list(kind, doc);
|
|
66
|
+
const withPaths = await Promise.all(entries.map(async (entry) => ({
|
|
67
67
|
...entry,
|
|
68
|
-
absolute_path: attachRepo.absolutePathFor(kind, doc, entry.filename),
|
|
69
|
-
}));
|
|
68
|
+
absolute_path: await attachRepo.absolutePathFor(kind, doc, entry.filename),
|
|
69
|
+
})));
|
|
70
70
|
return { content: [{ type: 'text', text: JSON.stringify(withPaths, null, 2) }] };
|
|
71
71
|
}
|
|
72
72
|
catch (err) {
|
|
@@ -75,7 +75,7 @@ export function registerAttachmentTools(mcp, attachRepo) {
|
|
|
75
75
|
});
|
|
76
76
|
mcp.tool('attachment_reconcile', "Compares a doc's declared attachments against what is actually on disk; reports orphans (declared but missing) and unlisted files (present but not declared). Report-only, does not repair.", { kind: kindSchema, doc: z.string() }, async ({ kind, doc }) => {
|
|
77
77
|
try {
|
|
78
|
-
const result = attachRepo.reconcile(kind, doc);
|
|
78
|
+
const result = await attachRepo.reconcile(kind, doc);
|
|
79
79
|
return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] };
|
|
80
80
|
}
|
|
81
81
|
catch (err) {
|
package/dist/src/config.js
CHANGED
|
@@ -1,9 +1,36 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
|
+
const FOLDERFOO_DEV_HOST = 'http://localhost:3000';
|
|
4
|
+
const FOLDERFOO_CLOUD_HOST = 'https://files.cuul.cc';
|
|
5
|
+
function isRemoteEntry(entry) {
|
|
6
|
+
return typeof entry === 'object' && 'remote' in entry;
|
|
7
|
+
}
|
|
8
|
+
/** Local mirror directory a remote source's content is synced into - see the RemoteFolder doc comment above. */
|
|
9
|
+
export function mirrorDirFor(baseDir, name) {
|
|
10
|
+
return path.join(baseDir, '.memory-bucket-remote-cache', sanitizeFolderName(name));
|
|
11
|
+
}
|
|
3
12
|
function memoryDirFlag(argv) {
|
|
4
13
|
const idx = argv.indexOf('--memory-dir');
|
|
5
14
|
return idx !== -1 ? argv[idx + 1] : undefined;
|
|
6
15
|
}
|
|
16
|
+
function folderfooModeFlag(argv) {
|
|
17
|
+
const idx = argv.indexOf('--folderfoo-mode');
|
|
18
|
+
return idx !== -1 ? argv[idx + 1] : undefined;
|
|
19
|
+
}
|
|
20
|
+
function resolveFolderfooMode(argv) {
|
|
21
|
+
const raw = (folderfooModeFlag(argv) ?? process.env.FOLDERFOO_MODE ?? 'off').toLowerCase();
|
|
22
|
+
if (raw === 'dev' || raw === 'cloud' || raw === 'off')
|
|
23
|
+
return raw;
|
|
24
|
+
console.error(`[memory-bucket] unrecognized --folderfoo-mode/FOLDERFOO_MODE "${raw}" — expected off|dev|cloud, defaulting to "off"`);
|
|
25
|
+
return 'off';
|
|
26
|
+
}
|
|
27
|
+
function folderfooHostFor(mode) {
|
|
28
|
+
if (mode === 'dev')
|
|
29
|
+
return FOLDERFOO_DEV_HOST;
|
|
30
|
+
if (mode === 'cloud')
|
|
31
|
+
return FOLDERFOO_CLOUD_HOST;
|
|
32
|
+
return null;
|
|
33
|
+
}
|
|
7
34
|
function readConfigFile(configPath) {
|
|
8
35
|
if (!fs.existsSync(configPath))
|
|
9
36
|
return {};
|
|
@@ -12,16 +39,32 @@ function readConfigFile(configPath) {
|
|
|
12
39
|
function nameFromPath(p) {
|
|
13
40
|
return path.basename(p);
|
|
14
41
|
}
|
|
42
|
+
/**
|
|
43
|
+
* Resolves every configured source entry to a NamedFolder - for a remote
|
|
44
|
+
* entry, NamedFolder.path is the local MIRROR directory (see mirrorDirFor),
|
|
45
|
+
* not folderfoo itself, so every existing downstream consumer of
|
|
46
|
+
* NamedFolder (chokidar, sync.ts's upsertFile/folderForFile) keeps working
|
|
47
|
+
* completely unchanged. remoteFolders carries the folderfoo-specific
|
|
48
|
+
* coordinates a remote entry also needs, threaded separately into the
|
|
49
|
+
* poller rather than into NamedFolder/TableSyncSpec.
|
|
50
|
+
*/
|
|
15
51
|
function resolveFolders(entries, baseDir) {
|
|
52
|
+
const remoteFolders = [];
|
|
16
53
|
const folders = entries.map((entry) => {
|
|
17
54
|
if (typeof entry === 'string') {
|
|
18
55
|
return { name: nameFromPath(entry), path: path.resolve(baseDir, entry) };
|
|
19
56
|
}
|
|
57
|
+
if (isRemoteEntry(entry)) {
|
|
58
|
+
const mirrorDir = mirrorDirFor(baseDir, entry.name);
|
|
59
|
+
remoteFolders.push({ name: entry.name, ...entry.remote, mirrorDir });
|
|
60
|
+
return { name: entry.name, path: mirrorDir };
|
|
61
|
+
}
|
|
20
62
|
return { name: entry.name, path: path.resolve(baseDir, entry.path) };
|
|
21
63
|
});
|
|
22
64
|
// de-dupe by resolved path (e.g. default + explicit config both pointing at ./skills)
|
|
23
65
|
const seen = new Set();
|
|
24
|
-
|
|
66
|
+
const deduped = folders.filter((f) => (seen.has(f.path) ? false : (seen.add(f.path), true)));
|
|
67
|
+
return { folders: deduped, remoteFolders };
|
|
25
68
|
}
|
|
26
69
|
export function loadConfig(cwd = process.cwd(), argv = process.argv) {
|
|
27
70
|
const explicitDir = memoryDirFlag(argv) ?? process.env.MEMORY_BUCKET_DIR;
|
|
@@ -37,14 +80,19 @@ export function loadConfig(cwd = process.cwd(), argv = process.argv) {
|
|
|
37
80
|
// rather than silently registering a directory that was never created.
|
|
38
81
|
const defaultSkillSources = overrides.skill_sources ?? (fs.existsSync(path.resolve(baseDir, './skills')) ? ['./skills'] : []);
|
|
39
82
|
const defaultMemorySources = overrides.memory_sources ?? (fs.existsSync(path.resolve(baseDir, './docs')) ? ['./docs'] : []);
|
|
40
|
-
const skillFolders = resolveFolders(defaultSkillSources, baseDir);
|
|
41
|
-
const memoryFolders = resolveFolders(defaultMemorySources, baseDir);
|
|
83
|
+
const { folders: skillFolders, remoteFolders: remoteSkillFolders } = resolveFolders(defaultSkillSources, baseDir);
|
|
84
|
+
const { folders: memoryFolders, remoteFolders: remoteMemoryFolders } = resolveFolders(defaultMemorySources, baseDir);
|
|
85
|
+
const folderfooMode = resolveFolderfooMode(argv);
|
|
42
86
|
return {
|
|
43
87
|
skillFolders,
|
|
44
88
|
memoryFolders,
|
|
89
|
+
remoteSkillFolders,
|
|
90
|
+
remoteMemoryFolders,
|
|
45
91
|
cacheDbPath: path.join(baseDir, '.memory-bucket-cache.sqlite'),
|
|
46
92
|
configPath,
|
|
47
93
|
baseDir,
|
|
94
|
+
folderfooMode,
|
|
95
|
+
folderfooHost: folderfooHostFor(folderfooMode),
|
|
48
96
|
};
|
|
49
97
|
}
|
|
50
98
|
/**
|
|
@@ -68,6 +116,25 @@ export function saveFolder(config, kind, folder) {
|
|
|
68
116
|
};
|
|
69
117
|
fs.writeFileSync(config.configPath, JSON.stringify(next, null, 2) + '\n');
|
|
70
118
|
}
|
|
119
|
+
/**
|
|
120
|
+
* Appends a new REMOTE (folderfoo) source entry — same append-only behavior as saveFolder above,
|
|
121
|
+
* but writes the `{name, remote: {...}}` shape instead of `{name, path}`. The JWT itself is never
|
|
122
|
+
* written here — that's credentials.ts's job, kept in a separate untracked file (see its own doc
|
|
123
|
+
* comment for why: this config file is a plain JSON file that might reasonably get committed/shared).
|
|
124
|
+
*/
|
|
125
|
+
export function saveRemoteFolder(config, kind, entry) {
|
|
126
|
+
const current = readConfigFile(config.configPath);
|
|
127
|
+
const key = kind === 'skill' ? 'skill_sources' : 'memory_sources';
|
|
128
|
+
const existing = current[key] ?? [];
|
|
129
|
+
const next = {
|
|
130
|
+
...current,
|
|
131
|
+
[key]: [
|
|
132
|
+
...existing,
|
|
133
|
+
{ name: entry.name, remote: { server: entry.server, tenantId: entry.tenantId, folderPath: entry.folderPath } },
|
|
134
|
+
],
|
|
135
|
+
};
|
|
136
|
+
fs.writeFileSync(config.configPath, JSON.stringify(next, null, 2) + '\n');
|
|
137
|
+
}
|
|
71
138
|
/** Lowercase-hyphenate a folder-derived name, same shape as skill names. */
|
|
72
139
|
export function sanitizeFolderName(raw) {
|
|
73
140
|
return raw
|
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { randomUUID } from 'node:crypto';
|
|
4
|
+
import matter from 'gray-matter';
|
|
4
5
|
import { writeMarkdownFile } from '../store/markdown-file.js';
|
|
5
6
|
import { slugify } from '../store/slug.js';
|
|
6
7
|
import { resolveWithinBase } from '../store/safe-path.js';
|
|
7
8
|
import { upsertFile, removeFile, scanSingleFolder, unregisterFolder, memorySyncSpec } from '../store/sync.js';
|
|
8
9
|
import { SearchQueryError, sanitizeFtsQuery } from '../store/search.js';
|
|
10
|
+
import { applyBodyEdits } from '../shared/body-edits.js';
|
|
9
11
|
import { attachmentsDirFor } from '../attachments/storage.js';
|
|
10
12
|
import { normalizeKey } from '../types.js';
|
|
13
|
+
import { readFile as readRemoteFile, writeFile as writeRemoteFile, joinRemoteFolderPath } from '../remote/folderfoo-client.js';
|
|
11
14
|
/** Uppercases and strips everything but letters/digits — used to compare keys that differ only in
|
|
12
15
|
* punctuation/whitespace formatting (e.g. `RMXS-15` and `RMXS15` strip to the same `RMXS15`). */
|
|
13
16
|
export function stripKey(s) {
|
|
@@ -35,13 +38,21 @@ function rowToDoc(row) {
|
|
|
35
38
|
export class MemoryRepository {
|
|
36
39
|
db;
|
|
37
40
|
folders;
|
|
41
|
+
remoteFolders;
|
|
42
|
+
credentialsBaseDir;
|
|
38
43
|
syncSpec;
|
|
39
44
|
watcher;
|
|
40
|
-
constructor(db, folders) {
|
|
45
|
+
constructor(db, folders, remoteFolders = [], credentialsBaseDir) {
|
|
41
46
|
this.db = db;
|
|
42
47
|
this.folders = folders;
|
|
48
|
+
this.remoteFolders = remoteFolders;
|
|
49
|
+
this.credentialsBaseDir = credentialsBaseDir;
|
|
43
50
|
this.syncSpec = memorySyncSpec(folders);
|
|
44
51
|
}
|
|
52
|
+
/** The RemoteFolder a NamedFolder name resolves to, or undefined for a local (non-remote) folder. */
|
|
53
|
+
remoteFor(folderName) {
|
|
54
|
+
return this.remoteFolders.find((f) => f.name === folderName);
|
|
55
|
+
}
|
|
45
56
|
/** Attaches the live chokidar watcher so addFolder/removeFolder can mutate it without a restart. */
|
|
46
57
|
setWatcher(watcher) {
|
|
47
58
|
this.watcher = watcher;
|
|
@@ -49,6 +60,14 @@ export class MemoryRepository {
|
|
|
49
60
|
listFolders() {
|
|
50
61
|
return [...this.folders];
|
|
51
62
|
}
|
|
63
|
+
/** Full RemoteFolder records (server/tenantId/folderPath/mirrorDir) for every connected remote source — for matching an incoming folderfoo-file-open address back to a configured source. */
|
|
64
|
+
listRemoteFolders() {
|
|
65
|
+
return [...this.remoteFolders];
|
|
66
|
+
}
|
|
67
|
+
/** Same as listFolders(), but tags each entry with whether it's a remote (folderfoo) source — for the web UI's folder list, e.g. to render remote folders in a distinct color. */
|
|
68
|
+
listFoldersWithRemoteInfo() {
|
|
69
|
+
return this.folders.map((f) => ({ ...f, remote: !!this.remoteFor(f.name) }));
|
|
70
|
+
}
|
|
52
71
|
resolveFolder(folderName) {
|
|
53
72
|
if (folderName) {
|
|
54
73
|
const found = this.folders.find((f) => f.name === folderName);
|
|
@@ -64,6 +83,24 @@ export class MemoryRepository {
|
|
|
64
83
|
}
|
|
65
84
|
throw new Error(`multiple memory folders configured — specify folder: one of ${this.folders.map((f) => f.name).join(', ')}`);
|
|
66
85
|
}
|
|
86
|
+
/**
|
|
87
|
+
* Registers a new REMOTE (folderfoo) folder: creates its local mirror
|
|
88
|
+
* directory, registers it exactly like a local addFolder (so it starts
|
|
89
|
+
* watching/scanning immediately — empty at first, since nothing's been
|
|
90
|
+
* pulled from folderfoo yet), and records the folderfoo coordinates so
|
|
91
|
+
* get()/create()/update() know to treat this folder as remote. Does NOT
|
|
92
|
+
* perform the initial pull itself — the caller (the web route) does one
|
|
93
|
+
* immediate poll right after this returns, so content shows up without
|
|
94
|
+
* waiting for the first interval tick.
|
|
95
|
+
*/
|
|
96
|
+
registerRemoteFolder(remote) {
|
|
97
|
+
if (this.folders.some((f) => f.name === remote.name)) {
|
|
98
|
+
throw new Error(`a memory folder named "${remote.name}" already exists`);
|
|
99
|
+
}
|
|
100
|
+
fs.mkdirSync(remote.mirrorDir, { recursive: true });
|
|
101
|
+
this.remoteFolders.push(remote);
|
|
102
|
+
this.addFolder({ name: remote.name, path: remote.mirrorDir });
|
|
103
|
+
}
|
|
67
104
|
/** Registers a new folder: appends it, scans it once, and starts watching it live. */
|
|
68
105
|
addFolder(folder) {
|
|
69
106
|
if (this.folders.some((f) => f.name === folder.name)) {
|
|
@@ -149,13 +186,40 @@ export class MemoryRepository {
|
|
|
149
186
|
throw new SearchQueryError(query, err);
|
|
150
187
|
}
|
|
151
188
|
}
|
|
152
|
-
|
|
189
|
+
/**
|
|
190
|
+
* For a doc in a LOCAL folder, returns the cached row unchanged (identical behavior to before
|
|
191
|
+
* remote sources existed). For a doc in a REMOTE folder, the cached `body` is only ever a
|
|
192
|
+
* poll-interval-stale mirror snapshot — per the settled design, `get` always fetches the current
|
|
193
|
+
* body live from folderfoo instead of trusting it, so a caller never sees stale content just
|
|
194
|
+
* because the last poll tick hasn't run yet.
|
|
195
|
+
*/
|
|
196
|
+
async get(id) {
|
|
153
197
|
const row = this.db.prepare(`SELECT * FROM memory_docs WHERE id = ?`).get(id);
|
|
154
|
-
|
|
198
|
+
if (!row)
|
|
199
|
+
return null;
|
|
200
|
+
const doc = rowToDoc(row);
|
|
201
|
+
const remote = this.remoteFor(doc.folder);
|
|
202
|
+
if (!remote || !this.credentialsBaseDir)
|
|
203
|
+
return doc;
|
|
204
|
+
const relPath = path.relative(remote.mirrorDir, doc.source_path).replace(/\.md$/, '');
|
|
205
|
+
const dir = joinRemoteFolderPath(remote.folderPath, path.dirname(relPath));
|
|
206
|
+
const name = path.basename(relPath);
|
|
207
|
+
// readRemoteFile returns the file's RAW bytes as folderfoo stored them -
|
|
208
|
+
// the whole markdown file, frontmatter block included, since folderfoo
|
|
209
|
+
// has no concept of frontmatter/body separation (that's purely a
|
|
210
|
+
// mem-bucket/gray-matter concept). Using this raw content directly as
|
|
211
|
+
// `body` was a real bug: the next update() would writeMarkdownFile a
|
|
212
|
+
// FRESH frontmatter block wrapped around this already-frontmattered
|
|
213
|
+
// blob, nesting one level deeper on every single edit (confirmed via a
|
|
214
|
+
// real corrupted doc: 3 levels of self-nested frontmatter+body after 3
|
|
215
|
+
// edits). Must parse it exactly like a local file read would.
|
|
216
|
+
const liveBody = matter(await readRemoteFile(remote.server, this.credentialsBaseDir, remote.tenantId, dir, name)).content.trim();
|
|
217
|
+
return { ...doc, body: liveBody };
|
|
155
218
|
}
|
|
156
219
|
/** Fetches many memory docs by id in one call — e.g. hydrating full bodies for a batch of search() hits. Missing ids are simply absent from the result, not errors. */
|
|
157
|
-
bulkGet(ids) {
|
|
158
|
-
|
|
220
|
+
async bulkGet(ids) {
|
|
221
|
+
const docs = await Promise.all(ids.map((id) => this.get(id)));
|
|
222
|
+
return docs.filter((doc) => doc !== null);
|
|
159
223
|
}
|
|
160
224
|
listKeys(keyPrefix) {
|
|
161
225
|
const rows = this.db
|
|
@@ -195,10 +259,21 @@ export class MemoryRepository {
|
|
|
195
259
|
.slice(0, limit)
|
|
196
260
|
.map(({ key, docCount }) => ({ key, docCount }));
|
|
197
261
|
}
|
|
198
|
-
create(input) {
|
|
262
|
+
async create(input) {
|
|
199
263
|
const normalizedKey = normalizeKey(input.key);
|
|
200
|
-
const id = `${slugify(normalizedKey)}-${slugify(input.description)}-${randomUUID().slice(0, 8)}`;
|
|
201
264
|
const targetFolder = this.resolveFolder(input.folder);
|
|
265
|
+
let id = `${slugify(normalizedKey)}-${slugify(input.description)}-${randomUUID().slice(0, 8)}`;
|
|
266
|
+
// folderfoo's POST /save/:filename silently strips every character
|
|
267
|
+
// outside [0-9a-zA-Z_] from the final filename segment (hyphens
|
|
268
|
+
// included) - a remote-bound id must not contain characters folderfoo
|
|
269
|
+
// will drop, or the name mem-bucket thinks the file is called
|
|
270
|
+
// permanently diverges from what folderfoo actually stored it as the
|
|
271
|
+
// moment it's written, breaking every future get/update for that doc
|
|
272
|
+
// with a 404 (confirmed: this is exactly what happened before this
|
|
273
|
+
// fix). Stripped ONLY for remote-bound docs - local-only folders keep
|
|
274
|
+
// the more readable hyphenated id unchanged.
|
|
275
|
+
if (this.remoteFor(targetFolder.name))
|
|
276
|
+
id = id.replace(/-/g, '');
|
|
202
277
|
const filePath = resolveWithinBase(targetFolder.path, input.subfolder, `${id}.md`);
|
|
203
278
|
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
204
279
|
const fm = {
|
|
@@ -216,6 +291,18 @@ export class MemoryRepository {
|
|
|
216
291
|
folder: targetFolder.name,
|
|
217
292
|
};
|
|
218
293
|
writeMarkdownFile(filePath, stripSourcePath(fm), input.body);
|
|
294
|
+
// Remote write happens BEFORE upsertFile, live/synchronous per the settled design (no local
|
|
295
|
+
// staging) — if folderfoo rejects the write, this throws and the mirror is never indexed as
|
|
296
|
+
// if the doc were successfully saved. Reads the just-written mirror file back so folderfoo
|
|
297
|
+
// gets writeMarkdownFile's own formatting verbatim, not a hand-reconstructed string.
|
|
298
|
+
const remote = this.remoteFor(targetFolder.name);
|
|
299
|
+
if (remote && this.credentialsBaseDir) {
|
|
300
|
+
const relPath = path.relative(remote.mirrorDir, filePath).replace(/\.md$/, '');
|
|
301
|
+
const dir = joinRemoteFolderPath(remote.folderPath, path.dirname(relPath));
|
|
302
|
+
const name = path.basename(relPath);
|
|
303
|
+
const fileContents = fs.readFileSync(filePath, 'utf-8');
|
|
304
|
+
await writeRemoteFile(remote.server, this.credentialsBaseDir, remote.tenantId, dir, name, fileContents);
|
|
305
|
+
}
|
|
219
306
|
upsertFile(this.db, this.syncSpec, filePath);
|
|
220
307
|
return { ...fm, body: input.body, paused: false };
|
|
221
308
|
}
|
|
@@ -224,19 +311,21 @@ export class MemoryRepository {
|
|
|
224
311
|
* create()'s input. Returns per-key results (with the id filled in on
|
|
225
312
|
* success) so one bad entry doesn't abort the rest of the batch.
|
|
226
313
|
*/
|
|
227
|
-
bulkCreate(entries) {
|
|
228
|
-
|
|
314
|
+
async bulkCreate(entries) {
|
|
315
|
+
const results = [];
|
|
316
|
+
for (const entry of entries) {
|
|
229
317
|
try {
|
|
230
|
-
const doc = this.create(entry);
|
|
231
|
-
|
|
318
|
+
const doc = await this.create(entry);
|
|
319
|
+
results.push({ key: entry.key, ok: true, id: doc.id });
|
|
232
320
|
}
|
|
233
321
|
catch (err) {
|
|
234
|
-
|
|
322
|
+
results.push({ key: entry.key, ok: false, error: err.message });
|
|
235
323
|
}
|
|
236
|
-
}
|
|
324
|
+
}
|
|
325
|
+
return results;
|
|
237
326
|
}
|
|
238
|
-
update(id, frontmatter, body) {
|
|
239
|
-
const existing = this.get(id);
|
|
327
|
+
async update(id, frontmatter, body, bodyEdits) {
|
|
328
|
+
const existing = await this.get(id);
|
|
240
329
|
if (!existing)
|
|
241
330
|
throw new Error(`memory doc with id "${id}" not found`);
|
|
242
331
|
// `paused` is local-cache-only and must never reach writeMarkdownFile — split it off of
|
|
@@ -248,8 +337,16 @@ export class MemoryRepository {
|
|
|
248
337
|
id: existing.id,
|
|
249
338
|
key: frontmatter?.key ? normalizeKey(frontmatter.key) : existing.key,
|
|
250
339
|
};
|
|
251
|
-
const newBody = body ?? existing.body;
|
|
340
|
+
const newBody = bodyEdits ? applyBodyEdits(existing.body, bodyEdits).body : (body ?? existing.body);
|
|
252
341
|
writeMarkdownFile(existing.source_path, stripSourcePath(merged), newBody);
|
|
342
|
+
const remote = this.remoteFor(existing.folder);
|
|
343
|
+
if (remote && this.credentialsBaseDir) {
|
|
344
|
+
const relPath = path.relative(remote.mirrorDir, existing.source_path).replace(/\.md$/, '');
|
|
345
|
+
const dir = joinRemoteFolderPath(remote.folderPath, path.dirname(relPath));
|
|
346
|
+
const name = path.basename(relPath);
|
|
347
|
+
const fileContents = fs.readFileSync(existing.source_path, 'utf-8');
|
|
348
|
+
await writeRemoteFile(remote.server, this.credentialsBaseDir, remote.tenantId, dir, name, fileContents);
|
|
349
|
+
}
|
|
253
350
|
upsertFile(this.db, this.syncSpec, existing.source_path);
|
|
254
351
|
return { ...merged, body: newBody, paused: existingPaused };
|
|
255
352
|
}
|
|
@@ -261,10 +358,11 @@ export class MemoryRepository {
|
|
|
261
358
|
* when provided. Never touches body. Returns per-id results so partial
|
|
262
359
|
* failures (e.g. an unknown id) don't abort the rest of the batch.
|
|
263
360
|
*/
|
|
264
|
-
bulkUpdate(ids, changes) {
|
|
265
|
-
|
|
361
|
+
async bulkUpdate(ids, changes) {
|
|
362
|
+
const results = [];
|
|
363
|
+
for (const id of ids) {
|
|
266
364
|
try {
|
|
267
|
-
const existing = this.get(id);
|
|
365
|
+
const existing = await this.get(id);
|
|
268
366
|
if (!existing)
|
|
269
367
|
throw new Error(`memory doc with id "${id}" not found`);
|
|
270
368
|
let tags = existing.tags;
|
|
@@ -272,18 +370,19 @@ export class MemoryRepository {
|
|
|
272
370
|
tags = Array.from(new Set([...tags, ...changes.add_tags]));
|
|
273
371
|
if (changes.remove_tags?.length)
|
|
274
372
|
tags = tags.filter((t) => !changes.remove_tags.includes(t));
|
|
275
|
-
this.update(id, {
|
|
373
|
+
await this.update(id, {
|
|
276
374
|
tags,
|
|
277
375
|
...(changes.status !== undefined ? { status: changes.status } : {}),
|
|
278
376
|
...(changes.related_to !== undefined ? { related_to: changes.related_to } : {}),
|
|
279
377
|
...(changes.deprecated !== undefined ? { deprecated: changes.deprecated } : {}),
|
|
280
378
|
});
|
|
281
|
-
|
|
379
|
+
results.push({ id, ok: true });
|
|
282
380
|
}
|
|
283
381
|
catch (err) {
|
|
284
|
-
|
|
382
|
+
results.push({ id, ok: false, error: err.message });
|
|
285
383
|
}
|
|
286
|
-
}
|
|
384
|
+
}
|
|
385
|
+
return results;
|
|
287
386
|
}
|
|
288
387
|
/**
|
|
289
388
|
* Pauses/resumes memory docs by id — a local-only toggle stored directly in this cache file's
|
|
@@ -293,22 +392,24 @@ export class MemoryRepository {
|
|
|
293
392
|
* local-only, the flag does not follow the doc to another machine's cache or survive the cache
|
|
294
393
|
* file being deleted. Returns per-id results so one bad id doesn't abort the rest of the batch.
|
|
295
394
|
*/
|
|
296
|
-
setPaused(ids, paused) {
|
|
297
|
-
|
|
395
|
+
async setPaused(ids, paused) {
|
|
396
|
+
const results = [];
|
|
397
|
+
for (const id of ids) {
|
|
298
398
|
try {
|
|
299
|
-
const existing = this.get(id);
|
|
399
|
+
const existing = await this.get(id);
|
|
300
400
|
if (!existing)
|
|
301
401
|
throw new Error(`memory doc with id "${id}" not found`);
|
|
302
402
|
this.db.prepare(`UPDATE memory_docs SET paused = ? WHERE id = ?`).run(paused ? 1 : 0, id);
|
|
303
|
-
|
|
403
|
+
results.push({ id, ok: true });
|
|
304
404
|
}
|
|
305
405
|
catch (err) {
|
|
306
|
-
|
|
406
|
+
results.push({ id, ok: false, error: err.message });
|
|
307
407
|
}
|
|
308
|
-
}
|
|
408
|
+
}
|
|
409
|
+
return results;
|
|
309
410
|
}
|
|
310
|
-
delete(id) {
|
|
311
|
-
const existing = this.get(id);
|
|
411
|
+
async delete(id) {
|
|
412
|
+
const existing = await this.get(id);
|
|
312
413
|
if (!existing)
|
|
313
414
|
throw new Error(`memory doc with id "${id}" not found`);
|
|
314
415
|
// The <id>/ wrapper directory exists solely to hold attachments/ for this doc (memory docs
|
|
@@ -326,8 +427,8 @@ export class MemoryRepository {
|
|
|
326
427
|
* deriveFrontmatter's fallback, so the caller should treat this doc as gone under its old id
|
|
327
428
|
* once this returns (look it up by the new filename-derived id/key instead).
|
|
328
429
|
*/
|
|
329
|
-
stripFrontmatter(id) {
|
|
330
|
-
const existing = this.get(id);
|
|
430
|
+
async stripFrontmatter(id) {
|
|
431
|
+
const existing = await this.get(id);
|
|
331
432
|
if (!existing)
|
|
332
433
|
throw new Error(`memory doc with id "${id}" not found`);
|
|
333
434
|
writeMarkdownFile(existing.source_path, {}, existing.body);
|
|
@@ -338,16 +439,18 @@ export class MemoryRepository {
|
|
|
338
439
|
* abandoned docs found via search(). Returns per-id results so one bad id
|
|
339
440
|
* doesn't abort the rest of the batch.
|
|
340
441
|
*/
|
|
341
|
-
bulkDelete(ids) {
|
|
342
|
-
|
|
442
|
+
async bulkDelete(ids) {
|
|
443
|
+
const results = [];
|
|
444
|
+
for (const id of ids) {
|
|
343
445
|
try {
|
|
344
|
-
this.delete(id);
|
|
345
|
-
|
|
446
|
+
await this.delete(id);
|
|
447
|
+
results.push({ id, ok: true });
|
|
346
448
|
}
|
|
347
449
|
catch (err) {
|
|
348
|
-
|
|
450
|
+
results.push({ id, ok: false, error: err.message });
|
|
349
451
|
}
|
|
350
|
-
}
|
|
452
|
+
}
|
|
453
|
+
return results;
|
|
351
454
|
}
|
|
352
455
|
}
|
|
353
456
|
function stripSourcePath(fm) {
|