@moxxy/plugin-memory 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/consolidate.d.ts +62 -0
- package/dist/consolidate.d.ts.map +1 -0
- package/dist/consolidate.js +417 -0
- package/dist/consolidate.js.map +1 -0
- package/dist/embedding-cache.d.ts +32 -0
- package/dist/embedding-cache.d.ts.map +1 -0
- package/dist/embedding-cache.js +146 -0
- package/dist/embedding-cache.js.map +1 -0
- package/dist/index.d.ts +31 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +217 -0
- package/dist/index.js.map +1 -0
- package/dist/parse.d.ts +5 -0
- package/dist/parse.d.ts.map +1 -0
- package/dist/parse.js +9 -0
- package/dist/parse.js.map +1 -0
- package/dist/stm.d.ts +20 -0
- package/dist/stm.d.ts.map +1 -0
- package/dist/stm.js +38 -0
- package/dist/stm.js.map +1 -0
- package/dist/store/io.d.ts +13 -0
- package/dist/store/io.d.ts.map +1 -0
- package/dist/store/io.js +124 -0
- package/dist/store/io.js.map +1 -0
- package/dist/store/search.d.ts +10 -0
- package/dist/store/search.d.ts.map +1 -0
- package/dist/store/search.js +160 -0
- package/dist/store/search.js.map +1 -0
- package/dist/store/types.d.ts +33 -0
- package/dist/store/types.d.ts.map +1 -0
- package/dist/store/types.js +11 -0
- package/dist/store/types.js.map +1 -0
- package/dist/store.d.ts +95 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +220 -0
- package/dist/store.js.map +1 -0
- package/dist/tfidf.d.ts +29 -0
- package/dist/tfidf.d.ts.map +1 -0
- package/dist/tfidf.js +103 -0
- package/dist/tfidf.js.map +1 -0
- package/package.json +62 -0
- package/src/consolidate-nudge.test.ts +98 -0
- package/src/consolidate.test.ts +328 -0
- package/src/consolidate.ts +535 -0
- package/src/discovery.test.ts +33 -0
- package/src/embedding-cache.test.ts +268 -0
- package/src/embedding-cache.ts +156 -0
- package/src/index.test.ts +67 -0
- package/src/index.ts +247 -0
- package/src/parse.ts +12 -0
- package/src/stm.test.ts +69 -0
- package/src/stm.ts +48 -0
- package/src/store/io.test.ts +100 -0
- package/src/store/io.ts +138 -0
- package/src/store/search.test.ts +185 -0
- package/src/store/search.ts +197 -0
- package/src/store/types.ts +23 -0
- package/src/store.test.ts +186 -0
- package/src/store.ts +292 -0
- package/src/tfidf.test.ts +59 -0
- package/src/tfidf.ts +105 -0
- package/src/vector-recall.test.ts +88 -0
package/dist/store/io.js
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { promises as fs } from 'node:fs';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import { writeFileAtomic } from '@moxxy/sdk/server';
|
|
4
|
+
import { parseMdFile } from '../parse.js';
|
|
5
|
+
import { memoryFrontmatterSchema, } from './types.js';
|
|
6
|
+
export async function safeRead(filePath) {
|
|
7
|
+
try {
|
|
8
|
+
const raw = await fs.readFile(filePath, 'utf8');
|
|
9
|
+
const parsed = parseMdFile(raw);
|
|
10
|
+
const result = memoryFrontmatterSchema.safeParse(parsed.frontmatter);
|
|
11
|
+
if (!result.success)
|
|
12
|
+
return null;
|
|
13
|
+
// Trim the body so safeRead agrees with readEntry/listEntries on the same
|
|
14
|
+
// file — they both `.trim()`. safeRead's only caller (writeEntry) reads just
|
|
15
|
+
// createdAt and discards the body, so this is behavior-preserving today; it
|
|
16
|
+
// removes the latent trap where a future body consumer would see different
|
|
17
|
+
// whitespace depending on which read path it used.
|
|
18
|
+
return { frontmatter: result.data, body: parsed.body.trim() };
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
export function isEnoent(err) {
|
|
25
|
+
return err instanceof Error && 'code' in err && err.code === 'ENOENT';
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Cap on simultaneously-open file descriptors when reading a memory dir. The
|
|
29
|
+
* soft cap on entry count is warn-only, so a long-lived dir can hold thousands
|
|
30
|
+
* of files; an unbounded `Promise.all(readFile)` over all of them would open
|
|
31
|
+
* thousands of fds at once and can hit the OS limit (EMFILE) — failing every
|
|
32
|
+
* list/recall. Batching keeps the common case fast but bounds fd pressure.
|
|
33
|
+
*/
|
|
34
|
+
const READ_CONCURRENCY = 32;
|
|
35
|
+
export async function listEntries(dir, filterType) {
|
|
36
|
+
let names;
|
|
37
|
+
try {
|
|
38
|
+
names = await fs.readdir(dir, { withFileTypes: true });
|
|
39
|
+
}
|
|
40
|
+
catch (err) {
|
|
41
|
+
if (isEnoent(err))
|
|
42
|
+
return [];
|
|
43
|
+
throw err;
|
|
44
|
+
}
|
|
45
|
+
// recall() and rows() both call this on the hot path; with up to
|
|
46
|
+
// DEFAULT_MAX_MEMORIES (500) entries, reading+parsing each file serially is
|
|
47
|
+
// 500 strictly-ordered disk round-trips. The reads are independent, so fan
|
|
48
|
+
// them out concurrently (bounded by READ_CONCURRENCY) and preserve dirent
|
|
49
|
+
// order in the result.
|
|
50
|
+
const candidates = names.filter((d) => d.isFile() && d.name.endsWith('.md') && d.name !== 'MEMORY.md');
|
|
51
|
+
const readOne = async (dirent) => {
|
|
52
|
+
const filePath = path.join(dir, dirent.name);
|
|
53
|
+
let raw;
|
|
54
|
+
try {
|
|
55
|
+
raw = await fs.readFile(filePath, 'utf8');
|
|
56
|
+
}
|
|
57
|
+
catch (err) {
|
|
58
|
+
// One unreadable file (deleted between readdir and readFile, transient
|
|
59
|
+
// permission error) must not fail the whole list. Drop it, surface
|
|
60
|
+
// non-ENOENT causes for diagnosis.
|
|
61
|
+
if (!isEnoent(err)) {
|
|
62
|
+
console.warn(`[plugin-memory] skipping unreadable memory file ${filePath}: ${String(err)}`);
|
|
63
|
+
}
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
const md = parseMdFile(raw);
|
|
67
|
+
const result = memoryFrontmatterSchema.safeParse(md.frontmatter);
|
|
68
|
+
if (!result.success) {
|
|
69
|
+
// A malformed entry silently vanishing from recall/index is a data-loss
|
|
70
|
+
// surprise; leave a breadcrumb so it's diagnosable.
|
|
71
|
+
console.warn(`[plugin-memory] ignoring memory file with invalid frontmatter: ${filePath}`);
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
if (filterType && result.data.type !== filterType)
|
|
75
|
+
return null;
|
|
76
|
+
return {
|
|
77
|
+
frontmatter: result.data,
|
|
78
|
+
body: md.body.trim(),
|
|
79
|
+
path: filePath,
|
|
80
|
+
};
|
|
81
|
+
};
|
|
82
|
+
const parsed = [];
|
|
83
|
+
for (let i = 0; i < candidates.length; i += READ_CONCURRENCY) {
|
|
84
|
+
const batch = candidates.slice(i, i + READ_CONCURRENCY);
|
|
85
|
+
parsed.push(...(await Promise.all(batch.map(readOne))));
|
|
86
|
+
}
|
|
87
|
+
return parsed.filter((e) => e !== null);
|
|
88
|
+
}
|
|
89
|
+
export async function readEntry(filePath) {
|
|
90
|
+
try {
|
|
91
|
+
const raw = await fs.readFile(filePath, 'utf8');
|
|
92
|
+
const parsed = parseMdFile(raw);
|
|
93
|
+
const result = memoryFrontmatterSchema.safeParse(parsed.frontmatter);
|
|
94
|
+
if (!result.success)
|
|
95
|
+
return null;
|
|
96
|
+
return { frontmatter: result.data, body: parsed.body.trim(), path: filePath };
|
|
97
|
+
}
|
|
98
|
+
catch (err) {
|
|
99
|
+
if (isEnoent(err))
|
|
100
|
+
return null;
|
|
101
|
+
throw err;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
export async function writeIndex(dir, entries) {
|
|
105
|
+
const lines = ['# Memory index', ''];
|
|
106
|
+
const byType = new Map();
|
|
107
|
+
for (const e of entries) {
|
|
108
|
+
const list = byType.get(e.frontmatter.type) ?? [];
|
|
109
|
+
list.push(e);
|
|
110
|
+
byType.set(e.frontmatter.type, list);
|
|
111
|
+
}
|
|
112
|
+
for (const t of ['fact', 'preference', 'project', 'reference']) {
|
|
113
|
+
const items = byType.get(t);
|
|
114
|
+
if (!items || items.length === 0)
|
|
115
|
+
continue;
|
|
116
|
+
lines.push(`## ${t}`);
|
|
117
|
+
for (const item of items) {
|
|
118
|
+
lines.push(`- [${item.frontmatter.name}](${path.basename(item.path)}) — ${item.frontmatter.description}`);
|
|
119
|
+
}
|
|
120
|
+
lines.push('');
|
|
121
|
+
}
|
|
122
|
+
await writeFileAtomic(path.join(dir, 'MEMORY.md'), lines.join('\n'));
|
|
123
|
+
}
|
|
124
|
+
//# sourceMappingURL=io.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"io.js","sourceRoot":"","sources":["../../src/store/io.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EACL,uBAAuB,GAIxB,MAAM,YAAY,CAAC;AAEpB,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,QAAgB;IAEhB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,uBAAuB,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACrE,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QACjC,0EAA0E;QAC1E,6EAA6E;QAC7E,4EAA4E;QAC5E,2EAA2E;QAC3E,mDAAmD;QACnD,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;IAChE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,GAAY;IACnC,OAAO,GAAG,YAAY,KAAK,IAAI,MAAM,IAAI,GAAG,IAAK,GAA6B,CAAC,IAAI,KAAK,QAAQ,CAAC;AACnG,CAAC;AAED;;;;;;GAMG;AACH,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,GAAW,EACX,UAAuB;IAEvB,IAAI,KAAiC,CAAC;IACtC,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,QAAQ,CAAC,GAAG,CAAC;YAAE,OAAO,EAAE,CAAC;QAC7B,MAAM,GAAG,CAAC;IACZ,CAAC;IACD,iEAAiE;IACjE,4EAA4E;IAC5E,2EAA2E;IAC3E,0EAA0E;IAC1E,uBAAuB;IACvB,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAC7B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,CACtE,CAAC;IACF,MAAM,OAAO,GAAG,KAAK,EAAE,MAAgC,EAA+B,EAAE;QACtF,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACH,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,uEAAuE;YACvE,mEAAmE;YACnE,mCAAmC;YACnC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnB,OAAO,CAAC,IAAI,CAAC,mDAAmD,QAAQ,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9F,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;QACjE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,wEAAwE;YACxE,oDAAoD;YACpD,OAAO,CAAC,IAAI,CAAC,kEAAkE,QAAQ,EAAE,CAAC,CAAC;YAC3F,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,UAAU,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,UAAU;YAAE,OAAO,IAAI,CAAC;QAC/D,OAAO;YACL,WAAW,EAAE,MAAM,CAAC,IAAI;YACxB,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE;YACpB,IAAI,EAAE,QAAQ;SACO,CAAC;IAC1B,CAAC,CAAC;IACF,MAAM,MAAM,GAA8B,EAAE,CAAC;IAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;QAC7D,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC;QACxD,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAoB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAgB;IAC9C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,uBAAuB,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACrE,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QACjC,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAChF,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,QAAQ,CAAC,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAC/B,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAMD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,GAAW,EACX,OAAgC;IAEhC,MAAM,KAAK,GAAG,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0B,CAAC;IACjD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAClD,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACvC,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,WAAW,CAAU,EAAE,CAAC;QACxE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC3C,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACtB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,KAAK,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC,CAAC;QAC5G,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IACD,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AACvE,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { EmbeddingProvider, Mutex } from '@moxxy/sdk';
|
|
2
|
+
import type { EmbeddingIndex } from '../embedding-cache.js';
|
|
3
|
+
import type { MemoryEntry } from './types.js';
|
|
4
|
+
export interface RankedMemory {
|
|
5
|
+
readonly entry: MemoryEntry;
|
|
6
|
+
readonly score: number;
|
|
7
|
+
}
|
|
8
|
+
export declare function recallVector(all: ReadonlyArray<MemoryEntry>, query: string, limit: number, embedder: EmbeddingProvider, index: EmbeddingIndex | null, mutex: Mutex): Promise<ReadonlyArray<RankedMemory>>;
|
|
9
|
+
export declare function rankByKeywords(all: ReadonlyArray<MemoryEntry>, query: string, limit: number): ReadonlyArray<RankedMemory>;
|
|
10
|
+
//# sourceMappingURL=search.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/store/search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAE3D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,wBAAsB,YAAY,CAChC,GAAG,EAAE,aAAa,CAAC,WAAW,CAAC,EAC/B,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,iBAAiB,EAC3B,KAAK,EAAE,cAAc,GAAG,IAAI,EAC5B,KAAK,EAAE,KAAK,GACX,OAAO,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,CA4DtC;AAqBD,wBAAgB,cAAc,CAC5B,GAAG,EAAE,aAAa,CAAC,WAAW,CAAC,EAC/B,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,aAAa,CAAC,YAAY,CAAC,CAO7B"}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { TfIdfEmbedder, cosineSimilarity, tokenize } from '../tfidf.js';
|
|
2
|
+
export async function recallVector(all, query, limit, embedder, index, mutex) {
|
|
3
|
+
const corpus = all.map((e) => entryForEmbedding(e));
|
|
4
|
+
// TF-IDF special-cases the persistent cache (vocab is corpus-wide).
|
|
5
|
+
if (embedder instanceof TfIdfEmbedder) {
|
|
6
|
+
embedder.fit([...corpus, query]);
|
|
7
|
+
return rankAllFresh(all, corpus, query, limit, embedder);
|
|
8
|
+
}
|
|
9
|
+
// Neural embedders: consult the persistent cache, only embed misses + query.
|
|
10
|
+
if (index) {
|
|
11
|
+
// The index load->lookup->set->prune->flush cycle mutates the shared
|
|
12
|
+
// on-disk cache, so it must run under the store's write mutex — otherwise
|
|
13
|
+
// two concurrent recalls (or a recall racing forget()'s rebuildIndex)
|
|
14
|
+
// read the same snapshot and clobber each other's writes. Only the cache
|
|
15
|
+
// bookkeeping is serialized; the pure cosine ranking stays outside.
|
|
16
|
+
const { vectors, queryVec } = await mutex.run(async () => {
|
|
17
|
+
await index.load();
|
|
18
|
+
const cached = [];
|
|
19
|
+
const misses = [];
|
|
20
|
+
for (let i = 0; i < all.length; i++) {
|
|
21
|
+
const hit = index.lookup(all[i].frontmatter.name, corpus[i]);
|
|
22
|
+
cached.push(hit);
|
|
23
|
+
if (!hit)
|
|
24
|
+
misses.push({ index: i, text: corpus[i] });
|
|
25
|
+
}
|
|
26
|
+
const queryIdx = misses.length;
|
|
27
|
+
const toEmbed = [...misses.map((m) => m.text), query];
|
|
28
|
+
const fresh = await embedder.embed(toEmbed);
|
|
29
|
+
const qVec = fresh[queryIdx];
|
|
30
|
+
// Map each missed corpus index to its freshly-embedded vector so the
|
|
31
|
+
// stitch loop below stays O(1) per entry instead of scanning `misses`.
|
|
32
|
+
// A misbehaving embedder may under-return; only valid vectors get mapped,
|
|
33
|
+
// and missing ones stay absent so rankCosine drops them (never crashes).
|
|
34
|
+
const freshByEntryIndex = new Map();
|
|
35
|
+
for (const [j, m] of misses.entries()) {
|
|
36
|
+
const v = fresh[j];
|
|
37
|
+
if (Array.isArray(v))
|
|
38
|
+
freshByEntryIndex.set(m.index, v);
|
|
39
|
+
}
|
|
40
|
+
// Stitch results: cached + freshly-embedded. Holes (undefined) survive to
|
|
41
|
+
// rankCosine, which skips them rather than throwing.
|
|
42
|
+
const vecs = [];
|
|
43
|
+
for (let i = 0; i < all.length; i++) {
|
|
44
|
+
vecs.push(cached[i] ?? freshByEntryIndex.get(i));
|
|
45
|
+
}
|
|
46
|
+
// Persist ONLY valid fresh vectors — never write an `undefined`/non-array
|
|
47
|
+
// vector to the on-disk cache, which would poison it permanently (every
|
|
48
|
+
// future recall would read back a corrupt entry).
|
|
49
|
+
for (const [j, m] of misses.entries()) {
|
|
50
|
+
const v = fresh[j];
|
|
51
|
+
if (Array.isArray(v))
|
|
52
|
+
index.set(all[m.index].frontmatter.name, m.text, v);
|
|
53
|
+
}
|
|
54
|
+
index.prune(all.map((e) => e.frontmatter.name));
|
|
55
|
+
await index.flush();
|
|
56
|
+
return { vectors: vecs, queryVec: qVec };
|
|
57
|
+
});
|
|
58
|
+
return rankCosine(all, vectors, queryVec, limit);
|
|
59
|
+
}
|
|
60
|
+
// No cache configured — embed everything every time.
|
|
61
|
+
return rankAllFresh(all, corpus, query, limit, embedder);
|
|
62
|
+
}
|
|
63
|
+
// Embed `[...corpus, query]` in one batch, then cosine-rank the corpus against
|
|
64
|
+
// the (last) query vector. Shared by the TF-IDF and no-cache branches.
|
|
65
|
+
async function rankAllFresh(all, corpus, query, limit, embedder) {
|
|
66
|
+
const vectors = await embedder.embed([...corpus, query]);
|
|
67
|
+
// The query is embedded at index `corpus.length`. Address it by that fixed
|
|
68
|
+
// position (not `vectors.length - 1`): if the embedder under-returns and drops
|
|
69
|
+
// the query, `length - 1` would silently grab the last CORPUS vector and use
|
|
70
|
+
// it as the query. `vectors[corpus.length]` is `undefined` in that case, which
|
|
71
|
+
// rankCosine handles by returning an empty result — degrade, never mislead.
|
|
72
|
+
const queryVec = vectors[corpus.length];
|
|
73
|
+
return rankCosine(all, vectors.slice(0, all.length), queryVec, limit);
|
|
74
|
+
}
|
|
75
|
+
export function rankByKeywords(all, query, limit) {
|
|
76
|
+
const tokens = tokenize(query);
|
|
77
|
+
return all
|
|
78
|
+
.map((entry) => ({ entry, score: scoreEntry(entry, tokens) }))
|
|
79
|
+
.filter((r) => r.score > 0)
|
|
80
|
+
.sort((a, b) => b.score - a.score)
|
|
81
|
+
.slice(0, limit);
|
|
82
|
+
}
|
|
83
|
+
function rankCosine(entries, vectors, query, limit) {
|
|
84
|
+
// A misbehaving/hostile embedder can return fewer vectors than requested (or
|
|
85
|
+
// a non-array element) — the EmbeddingProvider.embed contract promises order
|
|
86
|
+
// but not count. Without this guard `query.length`/`vec.length` throws an
|
|
87
|
+
// opaque TypeError and recall crashes instead of degrading. A bad embedder
|
|
88
|
+
// must yield an empty/partial result set, never crash memory_recall.
|
|
89
|
+
if (!Array.isArray(query) || query.length === 0) {
|
|
90
|
+
console.warn('[plugin-memory] embedder returned no usable query vector; skipping vector recall');
|
|
91
|
+
return [];
|
|
92
|
+
}
|
|
93
|
+
const ranked = [];
|
|
94
|
+
for (let i = 0; i < entries.length; i++) {
|
|
95
|
+
const vec = vectors[i];
|
|
96
|
+
// cosineSimilarity silently truncates to the shorter vector, so a stale
|
|
97
|
+
// cached vector or a provider quirk of the wrong dimensionality would
|
|
98
|
+
// produce a plausible-but-wrong score (invisible corruption). A missing
|
|
99
|
+
// vector (embedder under-returned) is the same hazard. Skip the entry
|
|
100
|
+
// loudly instead of ranking it on a mismatched or absent basis.
|
|
101
|
+
if (!Array.isArray(vec) || vec.length !== query.length) {
|
|
102
|
+
console.warn(`[plugin-memory] skipping '${entries[i].frontmatter.name}' in recall: ` +
|
|
103
|
+
`vector dim ${Array.isArray(vec) ? vec.length : 'missing'} != query dim ${query.length} (cache/embedder drift)`);
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
const score = cosineSimilarity(vec, query);
|
|
107
|
+
if (score > 0)
|
|
108
|
+
ranked.push({ entry: entries[i], score });
|
|
109
|
+
}
|
|
110
|
+
ranked.sort((a, b) => b.score - a.score);
|
|
111
|
+
return ranked.slice(0, limit);
|
|
112
|
+
}
|
|
113
|
+
// Count non-overlapping occurrences of `needle` in `haystack` without
|
|
114
|
+
// allocating the intermediate array `haystack.split(needle)` would build.
|
|
115
|
+
// Identical result to `split(needle).length - 1` for the non-empty tokens
|
|
116
|
+
// `tokenize` yields. Tokens are `[a-z0-9_-]+`, so there are no overlap or
|
|
117
|
+
// empty-needle edge cases to worry about.
|
|
118
|
+
function countOccurrences(haystack, needle) {
|
|
119
|
+
let n = 0;
|
|
120
|
+
let i = haystack.indexOf(needle);
|
|
121
|
+
while (i !== -1) {
|
|
122
|
+
n += 1;
|
|
123
|
+
i = haystack.indexOf(needle, i + needle.length);
|
|
124
|
+
}
|
|
125
|
+
return n;
|
|
126
|
+
}
|
|
127
|
+
function entryForEmbedding(entry) {
|
|
128
|
+
return [
|
|
129
|
+
entry.frontmatter.name,
|
|
130
|
+
entry.frontmatter.description,
|
|
131
|
+
(entry.frontmatter.tags ?? []).join(' '),
|
|
132
|
+
entry.body,
|
|
133
|
+
].join('\n');
|
|
134
|
+
}
|
|
135
|
+
function scoreEntry(entry, tokens) {
|
|
136
|
+
if (tokens.length === 0)
|
|
137
|
+
return 1;
|
|
138
|
+
const haystack = (entry.frontmatter.name +
|
|
139
|
+
' ' +
|
|
140
|
+
entry.frontmatter.description +
|
|
141
|
+
' ' +
|
|
142
|
+
(entry.frontmatter.tags ?? []).join(' ') +
|
|
143
|
+
' ' +
|
|
144
|
+
entry.body).toLowerCase();
|
|
145
|
+
let score = 0;
|
|
146
|
+
for (const t of tokens) {
|
|
147
|
+
if (!t)
|
|
148
|
+
continue;
|
|
149
|
+
const matches = countOccurrences(haystack, t);
|
|
150
|
+
if (matches > 0) {
|
|
151
|
+
score += matches;
|
|
152
|
+
if (entry.frontmatter.name.toLowerCase().includes(t))
|
|
153
|
+
score += 3;
|
|
154
|
+
if (entry.frontmatter.description.toLowerCase().includes(t))
|
|
155
|
+
score += 2;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
return score;
|
|
159
|
+
}
|
|
160
|
+
//# sourceMappingURL=search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/store/search.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AASxE,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAA+B,EAC/B,KAAa,EACb,KAAa,EACb,QAA2B,EAC3B,KAA4B,EAC5B,KAAY;IAEZ,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;IAEpD,oEAAoE;IACpE,IAAI,QAAQ,YAAY,aAAa,EAAE,CAAC;QACtC,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QACjC,OAAO,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;IAC3D,CAAC;IAED,6EAA6E;IAC7E,IAAI,KAAK,EAAE,CAAC;QACV,qEAAqE;QACrE,0EAA0E;QAC1E,sEAAsE;QACtE,yEAAyE;QACzE,oEAAoE;QACpE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;YACvD,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,MAAM,MAAM,GAAwC,EAAE,CAAC;YACvD,MAAM,MAAM,GAAsC,EAAE,CAAC;YACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpC,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC;gBAC/D,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACjB,IAAI,CAAC,GAAG;oBAAE,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC;YACxD,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;YAC/B,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;YACtD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5C,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC7B,qEAAqE;YACrE,uEAAuE;YACvE,0EAA0E;YAC1E,yEAAyE;YACzE,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAiC,CAAC;YACnE,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;gBACtC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACnB,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;oBAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC1D,CAAC;YACD,0EAA0E;YAC1E,qDAAqD;YACrD,MAAM,IAAI,GAA6C,EAAE,CAAC;YAC1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACnD,CAAC;YACD,0EAA0E;YAC1E,wEAAwE;YACxE,kDAAkD;YAClD,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;gBACtC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACnB,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;oBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAE,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC7E,CAAC;YACD,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;YAChD,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACpB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC3C,CAAC,CAAC,CAAC;QACH,OAAO,UAAU,CAAC,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IACnD,CAAC;IAED,qDAAqD;IACrD,OAAO,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;AAC3D,CAAC;AAED,+EAA+E;AAC/E,uEAAuE;AACvE,KAAK,UAAU,YAAY,CACzB,GAA+B,EAC/B,MAA6B,EAC7B,KAAa,EACb,KAAa,EACb,QAA2B;IAE3B,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;IACzD,2EAA2E;IAC3E,+EAA+E;IAC/E,6EAA6E;IAC7E,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACxC,OAAO,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,GAA+B,EAC/B,KAAa,EACb,KAAa;IAEb,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC/B,OAAO,GAAG;SACP,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;SAC7D,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;SACjC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AACrB,CAAC;AAED,SAAS,UAAU,CACjB,OAAmC,EACnC,OAAyD,EACzD,KAAwC,EACxC,KAAa;IAEb,6EAA6E;IAC7E,6EAA6E;IAC7E,0EAA0E;IAC1E,2EAA2E;IAC3E,qEAAqE;IACrE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,OAAO,CAAC,IAAI,CACV,kFAAkF,CACnF,CAAC;QACF,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACvB,wEAAwE;QACxE,sEAAsE;QACtE,wEAAwE;QACxE,sEAAsE;QACtE,gEAAgE;QAChE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;YACvD,OAAO,CAAC,IAAI,CACV,6BAA6B,OAAO,CAAC,CAAC,CAAE,CAAC,WAAW,CAAC,IAAI,eAAe;gBACtE,cAAc,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,iBAAiB,KAAK,CAAC,MAAM,yBAAyB,CAClH,CAAC;YACF,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAG,gBAAgB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC3C,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACzC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AAChC,CAAC;AAED,sEAAsE;AACtE,0EAA0E;AAC1E,0EAA0E;AAC1E,0EAA0E;AAC1E,0CAA0C;AAC1C,SAAS,gBAAgB,CAAC,QAAgB,EAAE,MAAc;IACxD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,IAAI,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAChB,CAAC,IAAI,CAAC,CAAC;QACP,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAkB;IAC3C,OAAO;QACL,KAAK,CAAC,WAAW,CAAC,IAAI;QACtB,KAAK,CAAC,WAAW,CAAC,WAAW;QAC7B,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QACxC,KAAK,CAAC,IAAI;KACX,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,UAAU,CAAC,KAAkB,EAAE,MAA6B;IACnE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,CACf,KAAK,CAAC,WAAW,CAAC,IAAI;QACtB,GAAG;QACH,KAAK,CAAC,WAAW,CAAC,WAAW;QAC7B,GAAG;QACH,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QACxC,GAAG;QACH,KAAK,CAAC,IAAI,CACX,CAAC,WAAW,EAAE,CAAC;IAChB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,MAAM,OAAO,GAAG,gBAAgB,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAC9C,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,KAAK,IAAI,OAAO,CAAC;YACjB,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAAE,KAAK,IAAI,CAAC,CAAC;YACjE,IAAI,KAAK,CAAC,WAAW,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAAE,KAAK,IAAI,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export declare const memoryTypeSchema: z.ZodEnum<["fact", "preference", "project", "reference"]>;
|
|
3
|
+
export type MemoryType = z.infer<typeof memoryTypeSchema>;
|
|
4
|
+
export declare const memoryFrontmatterSchema: z.ZodObject<{
|
|
5
|
+
name: z.ZodString;
|
|
6
|
+
type: z.ZodEnum<["fact", "preference", "project", "reference"]>;
|
|
7
|
+
description: z.ZodString;
|
|
8
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
9
|
+
createdAt: z.ZodOptional<z.ZodString>;
|
|
10
|
+
updatedAt: z.ZodOptional<z.ZodString>;
|
|
11
|
+
}, "strip", z.ZodTypeAny, {
|
|
12
|
+
name: string;
|
|
13
|
+
type: "fact" | "preference" | "project" | "reference";
|
|
14
|
+
description: string;
|
|
15
|
+
tags?: string[] | undefined;
|
|
16
|
+
createdAt?: string | undefined;
|
|
17
|
+
updatedAt?: string | undefined;
|
|
18
|
+
}, {
|
|
19
|
+
name: string;
|
|
20
|
+
type: "fact" | "preference" | "project" | "reference";
|
|
21
|
+
description: string;
|
|
22
|
+
tags?: string[] | undefined;
|
|
23
|
+
createdAt?: string | undefined;
|
|
24
|
+
updatedAt?: string | undefined;
|
|
25
|
+
}>;
|
|
26
|
+
export type MemoryFrontmatter = z.infer<typeof memoryFrontmatterSchema>;
|
|
27
|
+
export interface MemoryEntry {
|
|
28
|
+
readonly frontmatter: MemoryFrontmatter;
|
|
29
|
+
readonly body: string;
|
|
30
|
+
readonly path: string;
|
|
31
|
+
}
|
|
32
|
+
export type RecallMode = 'auto' | 'vector' | 'keyword';
|
|
33
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/store/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,gBAAgB,2DAAyD,CAAC;AACvF,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;EAOlC,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,WAAW,EAAE,iBAAiB,CAAC;IACxC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export const memoryTypeSchema = z.enum(['fact', 'preference', 'project', 'reference']);
|
|
3
|
+
export const memoryFrontmatterSchema = z.object({
|
|
4
|
+
name: z.string().min(1).max(120).regex(/^[a-z0-9][a-z0-9-]*$/, 'name must be slug-like'),
|
|
5
|
+
type: memoryTypeSchema,
|
|
6
|
+
description: z.string().min(1).max(280),
|
|
7
|
+
tags: z.array(z.string().min(1)).optional(),
|
|
8
|
+
createdAt: z.string().optional(),
|
|
9
|
+
updatedAt: z.string().optional(),
|
|
10
|
+
});
|
|
11
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/store/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC;AAGvF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,sBAAsB,EAAE,wBAAwB,CAAC;IACxF,IAAI,EAAE,gBAAgB;IACtB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IACvC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC3C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACjC,CAAC,CAAC"}
|
package/dist/store.d.ts
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { type EmbeddingProvider } from '@moxxy/sdk';
|
|
2
|
+
import { type MemoryEntry, type MemoryFrontmatter, type MemoryType, type RecallMode } from './store/types.js';
|
|
3
|
+
import { type RankedMemory } from './store/search.js';
|
|
4
|
+
export { memoryTypeSchema, memoryFrontmatterSchema, type MemoryEntry, type MemoryFrontmatter, type MemoryType, type RecallMode, } from './store/types.js';
|
|
5
|
+
export type { RankedMemory } from './store/search.js';
|
|
6
|
+
export interface MemoryStoreOptions {
|
|
7
|
+
readonly dir?: string;
|
|
8
|
+
/**
|
|
9
|
+
* Optional embedding provider. When supplied, `recall()` uses cosine
|
|
10
|
+
* similarity over dense vectors. When omitted, the built-in TF-IDF
|
|
11
|
+
* embedder is used. Pass `embedder: null` to force keyword-only recall.
|
|
12
|
+
*
|
|
13
|
+
* May also be a lazy resolver `() => EmbeddingProvider | null`, resolved once
|
|
14
|
+
* on first recall — so the host can wire the registry-selected embedder
|
|
15
|
+
* (`() => session.embedders.tryGetActive()`) that isn't known until plugins
|
|
16
|
+
* have loaded, without forcing the store to be built after the session.
|
|
17
|
+
*/
|
|
18
|
+
readonly embedder?: EmbeddingProvider | null | (() => EmbeddingProvider | null);
|
|
19
|
+
/**
|
|
20
|
+
* Cache computed embeddings on disk (`<dir>/.embeddings.json`) so unchanged
|
|
21
|
+
* memories aren't re-embedded on every recall. Defaults to true for all
|
|
22
|
+
* embedders EXCEPT TF-IDF (which derives vocab from the whole corpus, so
|
|
23
|
+
* per-entry caching doesn't help).
|
|
24
|
+
*/
|
|
25
|
+
readonly persistEmbeddings?: boolean;
|
|
26
|
+
/**
|
|
27
|
+
* Soft cap on the number of stored memories. WARN-ONLY by design: memories
|
|
28
|
+
* are deliberately-saved user knowledge consumed both via `recall` and via
|
|
29
|
+
* the MEMORY.md index agents read directly, so silent oldest-eviction would
|
|
30
|
+
* be silent data loss. When a save pushes the store past this cap, the save
|
|
31
|
+
* still succeeds but a warning is logged and surfaced through
|
|
32
|
+
* {@link MemoryStore.capStatus} (the `memory_save` tool relays it to the
|
|
33
|
+
* model so it can consolidate or `memory_forget` stale entries).
|
|
34
|
+
*/
|
|
35
|
+
readonly maxMemories?: number;
|
|
36
|
+
}
|
|
37
|
+
/** Default soft cap — see {@link MemoryStoreOptions.maxMemories}. */
|
|
38
|
+
export declare const DEFAULT_MAX_MEMORIES = 500;
|
|
39
|
+
export declare function defaultMemoryDir(): string;
|
|
40
|
+
export declare class MemoryStore {
|
|
41
|
+
readonly dir: string;
|
|
42
|
+
private readonly resolveEmbedder;
|
|
43
|
+
private readonly persistOpt;
|
|
44
|
+
private embedderCache;
|
|
45
|
+
private indexCache;
|
|
46
|
+
private readonly mutex;
|
|
47
|
+
private readonly maxMemories;
|
|
48
|
+
private indexRows;
|
|
49
|
+
constructor(opts?: MemoryStoreOptions);
|
|
50
|
+
private getEmbedder;
|
|
51
|
+
private getIndex;
|
|
52
|
+
get embedderName(): string;
|
|
53
|
+
list(filterType?: MemoryType): Promise<ReadonlyArray<MemoryEntry>>;
|
|
54
|
+
get(name: string): Promise<MemoryEntry | null>;
|
|
55
|
+
save(input: Omit<MemoryFrontmatter, 'createdAt' | 'updatedAt'> & {
|
|
56
|
+
body: string;
|
|
57
|
+
}): Promise<MemoryEntry>;
|
|
58
|
+
update(name: string, patch: {
|
|
59
|
+
body?: string;
|
|
60
|
+
description?: string;
|
|
61
|
+
tags?: ReadonlyArray<string>;
|
|
62
|
+
}): Promise<MemoryEntry | null>;
|
|
63
|
+
forget(name: string): Promise<boolean>;
|
|
64
|
+
/**
|
|
65
|
+
* Soft-cap status (see {@link MemoryStoreOptions.maxMemories}). `over` means
|
|
66
|
+
* the store holds more entries than the cap; nothing is ever evicted.
|
|
67
|
+
*/
|
|
68
|
+
capStatus(): Promise<{
|
|
69
|
+
count: number;
|
|
70
|
+
max: number;
|
|
71
|
+
over: boolean;
|
|
72
|
+
}>;
|
|
73
|
+
/** The actual entry write. NOT serialized — callers hold the mutex. */
|
|
74
|
+
private writeEntry;
|
|
75
|
+
/**
|
|
76
|
+
* Search memories by a free-text query. Uses vector cosine similarity when
|
|
77
|
+
* an EmbeddingProvider is configured (the default is the built-in TF-IDF
|
|
78
|
+
* embedder); falls back to keyword scoring when `mode: 'keyword'` or when
|
|
79
|
+
* no embedder is wired.
|
|
80
|
+
*/
|
|
81
|
+
recall(query: string, opts?: {
|
|
82
|
+
limit?: number;
|
|
83
|
+
type?: MemoryType;
|
|
84
|
+
mode?: RecallMode;
|
|
85
|
+
}): Promise<ReadonlyArray<RankedMemory>>;
|
|
86
|
+
private fileFor;
|
|
87
|
+
/** Hydrate the index-row cache from disk once, then serve it from memory.
|
|
88
|
+
* Callers that mutate it hold the store mutex (writeEntry/forget). */
|
|
89
|
+
private rows;
|
|
90
|
+
/** Render MEMORY.md from the cached rows, name-sorted so the output is
|
|
91
|
+
* deterministic (readdir order, which the old full rebuild inherited,
|
|
92
|
+
* was platform-dependent anyway). */
|
|
93
|
+
private writeIndexFromRows;
|
|
94
|
+
}
|
|
95
|
+
//# sourceMappingURL=store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAEA,OAAO,EAAe,KAAK,iBAAiB,EAAc,MAAM,YAAY,CAAC;AAK7E,OAAO,EAEL,KAAK,WAAW,EAChB,KAAK,iBAAiB,EACtB,KAAK,UAAU,EACf,KAAK,UAAU,EAChB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAgC,KAAK,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEpF,OAAO,EACL,gBAAgB,EAChB,uBAAuB,EACvB,KAAK,WAAW,EAChB,KAAK,iBAAiB,EACtB,KAAK,UAAU,EACf,KAAK,UAAU,GAChB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEtD,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;;;;OASG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,iBAAiB,GAAG,IAAI,GAAG,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC,CAAC;IAChF;;;;;OAKG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC;IACrC;;;;;;;;OAQG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,qEAAqE;AACrE,eAAO,MAAM,oBAAoB,MAAM,CAAC;AAExC,wBAAgB,gBAAgB,IAAI,MAAM,CAEzC;AAED,qBAAa,WAAW;IACtB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAiC;IACjE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsB;IAIjD,OAAO,CAAC,aAAa,CAAuC;IAC5D,OAAO,CAAC,UAAU,CAAoC;IAItD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAwB;IAC9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IAQrC,OAAO,CAAC,SAAS,CAAsC;gBAE3C,IAAI,GAAE,kBAAuB;IAiBzC,OAAO,CAAC,WAAW;IAKnB,OAAO,CAAC,QAAQ;IAahB,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED,IAAI,CAAC,UAAU,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;IAIlE,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAa9C,IAAI,CACF,KAAK,EAAE,IAAI,CAAC,iBAAiB,EAAE,WAAW,GAAG,WAAW,CAAC,GAAG;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAC3E,OAAO,CAAC,WAAW,CAAC;IAIvB,MAAM,CACJ,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,CAAA;KAAE,GAC3E,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAiB9B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAgBtC;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC;IAQzE,uEAAuE;YACzD,UAAU;IAiCxB;;;;;OAKG;IACG,MAAM,CACV,KAAK,EAAE,MAAM,EACb,IAAI,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,UAAU,CAAC;QAAC,IAAI,CAAC,EAAE,UAAU,CAAA;KAAO,GAClE,OAAO,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;IAcvC,OAAO,CAAC,OAAO;IAcf;2EACuE;YACzD,IAAI;IAUlB;;0CAEsC;IACtC,OAAO,CAAC,kBAAkB;CAM3B"}
|