@moxxy/plugin-memory 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/consolidate.d.ts +62 -0
- package/dist/consolidate.d.ts.map +1 -0
- package/dist/consolidate.js +417 -0
- package/dist/consolidate.js.map +1 -0
- package/dist/embedding-cache.d.ts +32 -0
- package/dist/embedding-cache.d.ts.map +1 -0
- package/dist/embedding-cache.js +146 -0
- package/dist/embedding-cache.js.map +1 -0
- package/dist/index.d.ts +31 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +217 -0
- package/dist/index.js.map +1 -0
- package/dist/parse.d.ts +5 -0
- package/dist/parse.d.ts.map +1 -0
- package/dist/parse.js +9 -0
- package/dist/parse.js.map +1 -0
- package/dist/stm.d.ts +20 -0
- package/dist/stm.d.ts.map +1 -0
- package/dist/stm.js +38 -0
- package/dist/stm.js.map +1 -0
- package/dist/store/io.d.ts +13 -0
- package/dist/store/io.d.ts.map +1 -0
- package/dist/store/io.js +124 -0
- package/dist/store/io.js.map +1 -0
- package/dist/store/search.d.ts +10 -0
- package/dist/store/search.d.ts.map +1 -0
- package/dist/store/search.js +160 -0
- package/dist/store/search.js.map +1 -0
- package/dist/store/types.d.ts +33 -0
- package/dist/store/types.d.ts.map +1 -0
- package/dist/store/types.js +11 -0
- package/dist/store/types.js.map +1 -0
- package/dist/store.d.ts +95 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +220 -0
- package/dist/store.js.map +1 -0
- package/dist/tfidf.d.ts +29 -0
- package/dist/tfidf.d.ts.map +1 -0
- package/dist/tfidf.js +103 -0
- package/dist/tfidf.js.map +1 -0
- package/package.json +62 -0
- package/src/consolidate-nudge.test.ts +98 -0
- package/src/consolidate.test.ts +328 -0
- package/src/consolidate.ts +535 -0
- package/src/discovery.test.ts +33 -0
- package/src/embedding-cache.test.ts +268 -0
- package/src/embedding-cache.ts +156 -0
- package/src/index.test.ts +67 -0
- package/src/index.ts +247 -0
- package/src/parse.ts +12 -0
- package/src/stm.test.ts +69 -0
- package/src/stm.ts +48 -0
- package/src/store/io.test.ts +100 -0
- package/src/store/io.ts +138 -0
- package/src/store/search.test.ts +185 -0
- package/src/store/search.ts +197 -0
- package/src/store/types.ts +23 -0
- package/src/store.test.ts +186 -0
- package/src/store.ts +292 -0
- package/src/tfidf.test.ts +59 -0
- package/src/tfidf.ts +105 -0
- package/src/vector-recall.test.ts +88 -0
package/dist/store.js
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
import { promises as fs } from 'node:fs';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import { createMutex } from '@moxxy/sdk';
|
|
4
|
+
import { moxxyPath, writeFileAtomic } from '@moxxy/sdk/server';
|
|
5
|
+
import { renderFrontmatter } from './parse.js';
|
|
6
|
+
import { TfIdfEmbedder } from './tfidf.js';
|
|
7
|
+
import { EmbeddingIndex } from './embedding-cache.js';
|
|
8
|
+
import { memoryFrontmatterSchema, } from './store/types.js';
|
|
9
|
+
import { isEnoent, listEntries, readEntry, safeRead, writeIndex } from './store/io.js';
|
|
10
|
+
import { rankByKeywords, recallVector } from './store/search.js';
|
|
11
|
+
export { memoryTypeSchema, memoryFrontmatterSchema, } from './store/types.js';
|
|
12
|
+
/** Default soft cap — see {@link MemoryStoreOptions.maxMemories}. */
|
|
13
|
+
export const DEFAULT_MAX_MEMORIES = 500;
|
|
14
|
+
export function defaultMemoryDir() {
|
|
15
|
+
return moxxyPath('memory');
|
|
16
|
+
}
|
|
17
|
+
export class MemoryStore {
|
|
18
|
+
dir;
|
|
19
|
+
resolveEmbedder;
|
|
20
|
+
persistOpt;
|
|
21
|
+
// Embedder + index are resolved LAZILY on first recall (memoized), so a host
|
|
22
|
+
// that selects the embedder from a registry after plugins load can pass a
|
|
23
|
+
// resolver here without the store needing to be built after the session.
|
|
24
|
+
embedderCache;
|
|
25
|
+
indexCache;
|
|
26
|
+
// Per-instance mutex. save/update/forget/recall each read-modify-write the
|
|
27
|
+
// entry file, MEMORY.md, and the embedding index; without serialization two
|
|
28
|
+
// overlapping calls clobber MEMORY.md and race the embedding cache.
|
|
29
|
+
mutex = createMutex();
|
|
30
|
+
maxMemories;
|
|
31
|
+
// In-memory MEMORY.md rows (name → frontmatter+path), hydrated lazily from
|
|
32
|
+
// disk ONCE, then maintained incrementally on save/update/forget — so a
|
|
33
|
+
// write no longer re-reads + re-parses every memory file to rebuild the
|
|
34
|
+
// index (the old O(N)-per-write rebuild). Entries written to the dir by
|
|
35
|
+
// other processes appear after the next hydration (new store instance);
|
|
36
|
+
// within one process this store is the only writer, same assumption the
|
|
37
|
+
// mutex already makes.
|
|
38
|
+
indexRows = null;
|
|
39
|
+
constructor(opts = {}) {
|
|
40
|
+
this.dir = opts.dir ?? defaultMemoryDir();
|
|
41
|
+
this.persistOpt = opts.persistEmbeddings;
|
|
42
|
+
this.maxMemories = opts.maxMemories ?? DEFAULT_MAX_MEMORIES;
|
|
43
|
+
const e = opts.embedder;
|
|
44
|
+
if (typeof e === 'function') {
|
|
45
|
+
this.resolveEmbedder = e;
|
|
46
|
+
}
|
|
47
|
+
else if (e === null) {
|
|
48
|
+
this.resolveEmbedder = () => null;
|
|
49
|
+
}
|
|
50
|
+
else if (e !== undefined) {
|
|
51
|
+
this.resolveEmbedder = () => e;
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
const tfidf = new TfIdfEmbedder();
|
|
55
|
+
this.resolveEmbedder = () => tfidf;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
getEmbedder() {
|
|
59
|
+
if (this.embedderCache === undefined)
|
|
60
|
+
this.embedderCache = this.resolveEmbedder();
|
|
61
|
+
return this.embedderCache;
|
|
62
|
+
}
|
|
63
|
+
getIndex() {
|
|
64
|
+
if (this.indexCache === undefined) {
|
|
65
|
+
const emb = this.getEmbedder();
|
|
66
|
+
// TF-IDF's vocab depends on the whole corpus, so per-entry caching is
|
|
67
|
+
// useless — recompute every recall. For neural embedders, caching is
|
|
68
|
+
// a big win since each entry's vector is corpus-independent.
|
|
69
|
+
const isTfIdf = emb instanceof TfIdfEmbedder;
|
|
70
|
+
const persist = this.persistOpt ?? (emb !== null && !isTfIdf);
|
|
71
|
+
this.indexCache = persist && emb ? new EmbeddingIndex(this.dir, emb.name, emb.dim) : null;
|
|
72
|
+
}
|
|
73
|
+
return this.indexCache;
|
|
74
|
+
}
|
|
75
|
+
get embedderName() {
|
|
76
|
+
return this.getEmbedder()?.name ?? 'keyword';
|
|
77
|
+
}
|
|
78
|
+
list(filterType) {
|
|
79
|
+
return listEntries(this.dir, filterType);
|
|
80
|
+
}
|
|
81
|
+
get(name) {
|
|
82
|
+
// fileFor() throws synchronously on a traversal/separator name; surface it
|
|
83
|
+
// as a rejected promise so callers see one consistent (async) failure mode
|
|
84
|
+
// instead of an exception thrown before the promise is even constructed.
|
|
85
|
+
let file;
|
|
86
|
+
try {
|
|
87
|
+
file = this.fileFor(name);
|
|
88
|
+
}
|
|
89
|
+
catch (err) {
|
|
90
|
+
return Promise.reject(err);
|
|
91
|
+
}
|
|
92
|
+
return readEntry(file);
|
|
93
|
+
}
|
|
94
|
+
save(input) {
|
|
95
|
+
return this.mutex.run(() => this.writeEntry(input));
|
|
96
|
+
}
|
|
97
|
+
update(name, patch) {
|
|
98
|
+
// Read-modify-write under the mutex; calls the internal (unserialized)
|
|
99
|
+
// writer so it doesn't deadlock on its own chain.
|
|
100
|
+
return this.mutex.run(async () => {
|
|
101
|
+
const existing = await readEntry(this.fileFor(name));
|
|
102
|
+
if (!existing)
|
|
103
|
+
return null;
|
|
104
|
+
const mergedTags = patch.tags ?? existing.frontmatter.tags;
|
|
105
|
+
return this.writeEntry({
|
|
106
|
+
name: existing.frontmatter.name,
|
|
107
|
+
type: existing.frontmatter.type,
|
|
108
|
+
description: patch.description ?? existing.frontmatter.description,
|
|
109
|
+
...(mergedTags ? { tags: [...mergedTags] } : {}),
|
|
110
|
+
body: patch.body ?? existing.body,
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
forget(name) {
|
|
115
|
+
return this.mutex.run(async () => {
|
|
116
|
+
const filePath = this.fileFor(name);
|
|
117
|
+
try {
|
|
118
|
+
await fs.unlink(filePath);
|
|
119
|
+
const rows = await this.rows();
|
|
120
|
+
rows.delete(name);
|
|
121
|
+
await this.writeIndexFromRows(rows);
|
|
122
|
+
return true;
|
|
123
|
+
}
|
|
124
|
+
catch (err) {
|
|
125
|
+
if (isEnoent(err))
|
|
126
|
+
return false;
|
|
127
|
+
throw err;
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Soft-cap status (see {@link MemoryStoreOptions.maxMemories}). `over` means
|
|
133
|
+
* the store holds more entries than the cap; nothing is ever evicted.
|
|
134
|
+
*/
|
|
135
|
+
async capStatus() {
|
|
136
|
+
// Read the index-row cache under the mutex: writeEntry/forget mutate it
|
|
137
|
+
// while holding the same lock, so a concurrent capStatus() outside the lock
|
|
138
|
+
// could observe a half-updated map or trigger a redundant disk hydration.
|
|
139
|
+
const size = await this.mutex.run(async () => (await this.rows()).size);
|
|
140
|
+
return { count: size, max: this.maxMemories, over: size > this.maxMemories };
|
|
141
|
+
}
|
|
142
|
+
/** The actual entry write. NOT serialized — callers hold the mutex. */
|
|
143
|
+
async writeEntry(input) {
|
|
144
|
+
await fs.mkdir(this.dir, { recursive: true });
|
|
145
|
+
const filePath = this.fileFor(input.name);
|
|
146
|
+
const existing = await safeRead(filePath);
|
|
147
|
+
const now = new Date().toISOString();
|
|
148
|
+
const createdAt = existing?.frontmatter.createdAt ?? now;
|
|
149
|
+
const frontmatter = memoryFrontmatterSchema.parse({
|
|
150
|
+
name: input.name,
|
|
151
|
+
type: input.type,
|
|
152
|
+
description: input.description,
|
|
153
|
+
...(input.tags !== undefined ? { tags: input.tags } : {}),
|
|
154
|
+
createdAt,
|
|
155
|
+
updatedAt: now,
|
|
156
|
+
});
|
|
157
|
+
const content = `${renderFrontmatter(frontmatter)}\n\n${input.body.trimEnd()}\n`;
|
|
158
|
+
await writeFileAtomic(filePath, content);
|
|
159
|
+
// Incremental index maintenance: update THIS entry's row and re-render
|
|
160
|
+
// MEMORY.md from the in-memory rows — no per-write re-read of every file.
|
|
161
|
+
const rows = await this.rows();
|
|
162
|
+
rows.set(frontmatter.name, { frontmatter, path: filePath });
|
|
163
|
+
await this.writeIndexFromRows(rows);
|
|
164
|
+
if (rows.size > this.maxMemories) {
|
|
165
|
+
// Warn-only soft cap — eviction would silently destroy saved knowledge.
|
|
166
|
+
console.warn(`[plugin-memory] memory store holds ${rows.size} entries (soft cap ${this.maxMemories}). ` +
|
|
167
|
+
`Nothing is evicted; consider consolidating or forgetting stale memories.`);
|
|
168
|
+
}
|
|
169
|
+
return { frontmatter, body: input.body.trimEnd(), path: filePath };
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Search memories by a free-text query. Uses vector cosine similarity when
|
|
173
|
+
* an EmbeddingProvider is configured (the default is the built-in TF-IDF
|
|
174
|
+
* embedder); falls back to keyword scoring when `mode: 'keyword'` or when
|
|
175
|
+
* no embedder is wired.
|
|
176
|
+
*/
|
|
177
|
+
async recall(query, opts = {}) {
|
|
178
|
+
const limit = opts.limit ?? 5;
|
|
179
|
+
const mode = opts.mode ?? 'auto';
|
|
180
|
+
const all = await this.list(opts.type);
|
|
181
|
+
if (all.length === 0)
|
|
182
|
+
return [];
|
|
183
|
+
const embedder = this.getEmbedder();
|
|
184
|
+
const useVector = mode === 'vector' || (mode === 'auto' && embedder !== null);
|
|
185
|
+
if (useVector && embedder) {
|
|
186
|
+
return recallVector(all, query, limit, embedder, this.getIndex(), this.mutex);
|
|
187
|
+
}
|
|
188
|
+
return rankByKeywords(all, query, limit);
|
|
189
|
+
}
|
|
190
|
+
fileFor(name) {
|
|
191
|
+
// Belt-and-suspenders containment: the tool schemas restrict `name` to a
|
|
192
|
+
// slug, but the store is a public API and the inproc isolator can't enforce
|
|
193
|
+
// the fs glob — so refuse any name that escapes the memory dir (traversal,
|
|
194
|
+
// absolute path, or embedded separators) rather than unlink/read outside it.
|
|
195
|
+
const root = path.resolve(this.dir);
|
|
196
|
+
const resolved = path.resolve(root, `${name}.md`);
|
|
197
|
+
// The resolved file must live DIRECTLY in `root` (no nested or escaped path).
|
|
198
|
+
if (path.dirname(resolved) !== root) {
|
|
199
|
+
throw new Error(`invalid memory name: ${name}`);
|
|
200
|
+
}
|
|
201
|
+
return resolved;
|
|
202
|
+
}
|
|
203
|
+
/** Hydrate the index-row cache from disk once, then serve it from memory.
|
|
204
|
+
* Callers that mutate it hold the store mutex (writeEntry/forget). */
|
|
205
|
+
async rows() {
|
|
206
|
+
if (!this.indexRows) {
|
|
207
|
+
const entries = await listEntries(this.dir);
|
|
208
|
+
this.indexRows = new Map(entries.map((e) => [e.frontmatter.name, { frontmatter: e.frontmatter, path: e.path }]));
|
|
209
|
+
}
|
|
210
|
+
return this.indexRows;
|
|
211
|
+
}
|
|
212
|
+
/** Render MEMORY.md from the cached rows, name-sorted so the output is
|
|
213
|
+
* deterministic (readdir order, which the old full rebuild inherited,
|
|
214
|
+
* was platform-dependent anyway). */
|
|
215
|
+
writeIndexFromRows(rows) {
|
|
216
|
+
const sorted = [...rows.values()].sort((a, b) => a.frontmatter.name.localeCompare(b.frontmatter.name));
|
|
217
|
+
return writeIndex(this.dir, sorted);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
//# sourceMappingURL=store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,WAAW,EAAsC,MAAM,YAAY,CAAC;AAC7E,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EACL,uBAAuB,GAKxB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAiB,MAAM,eAAe,CAAC;AACtG,OAAO,EAAE,cAAc,EAAE,YAAY,EAAqB,MAAM,mBAAmB,CAAC;AAEpF,OAAO,EACL,gBAAgB,EAChB,uBAAuB,GAKxB,MAAM,kBAAkB,CAAC;AAmC1B,qEAAqE;AACrE,MAAM,CAAC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AAExC,MAAM,UAAU,gBAAgB;IAC9B,OAAO,SAAS,CAAC,QAAQ,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,OAAO,WAAW;IACb,GAAG,CAAS;IACJ,eAAe,CAAiC;IAChD,UAAU,CAAsB;IACjD,6EAA6E;IAC7E,0EAA0E;IAC1E,yEAAyE;IACjE,aAAa,CAAuC;IACpD,UAAU,CAAoC;IACtD,2EAA2E;IAC3E,4EAA4E;IAC5E,oEAAoE;IACnD,KAAK,GAAU,WAAW,EAAE,CAAC;IAC7B,WAAW,CAAS;IACrC,2EAA2E;IAC3E,wEAAwE;IACxE,wEAAwE;IACxE,wEAAwE;IACxE,wEAAwE;IACxE,wEAAwE;IACxE,uBAAuB;IACf,SAAS,GAAiC,IAAI,CAAC;IAEvD,YAAY,OAA2B,EAAE;QACvC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,gBAAgB,EAAE,CAAC;QAC1C,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,oBAAoB,CAAC;QAC5D,MAAM,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC;QACxB,IAAI,OAAO,CAAC,KAAK,UAAU,EAAE,CAAC;YAC5B,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QAC3B,CAAC;aAAM,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;YACtB,IAAI,CAAC,eAAe,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC;QACpC,CAAC;aAAM,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,eAAe,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,GAAG,IAAI,aAAa,EAAE,CAAC;YAClC,IAAI,CAAC,eAAe,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC;QACrC,CAAC;IACH,CAAC;IAEO,WAAW;QACjB,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS;YAAE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;QAClF,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAEO,QAAQ;QACd,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YAC/B,sEAAsE;YACtE,qEAAqE;YACrE,6DAA6D;YAC7D,MAAM,OAAO,GAAG,GAAG,YAAY,aAAa,CAAC;YAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,GAAG,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC;YAC9D,IAAI,CAAC,UAAU,GAAG,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5F,CAAC;QACD,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,WAAW,EAAE,EAAE,IAAI,IAAI,SAAS,CAAC;IAC/C,CAAC;IAED,IAAI,CAAC,UAAuB;QAC1B,OAAO,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAC3C,CAAC;IAED,GAAG,CAAC,IAAY;QACd,2EAA2E;QAC3E,2EAA2E;QAC3E,yEAAyE;QACzE,IAAI,IAAY,CAAC;QACjB,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QACD,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAED,IAAI,CACF,KAA4E;QAE5E,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,CACJ,IAAY,EACZ,KAA4E;QAE5E,uEAAuE;QACvE,kDAAkD;QAClD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;YAC/B,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;YACrD,IAAI,CAAC,QAAQ;gBAAE,OAAO,IAAI,CAAC;YAC3B,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,IAAI,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC;YAC3D,OAAO,IAAI,CAAC,UAAU,CAAC;gBACrB,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC,IAAI;gBAC/B,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC,IAAI;gBAC/B,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,QAAQ,CAAC,WAAW,CAAC,WAAW;gBAClE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChD,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI;aAClC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,IAAY;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;YAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,CAAC;gBACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC1B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAClB,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;gBACpC,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,QAAQ,CAAC,GAAG,CAAC;oBAAE,OAAO,KAAK,CAAC;gBAChC,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,SAAS;QACb,wEAAwE;QACxE,4EAA4E;QAC5E,0EAA0E;QAC1E,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QACxE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAC/E,CAAC;IAED,uEAAuE;IAC/D,KAAK,CAAC,UAAU,CACtB,KAA4E;QAE5E,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,SAAS,GAAG,QAAQ,EAAE,WAAW,CAAC,SAAS,IAAI,GAAG,CAAC;QACzD,MAAM,WAAW,GAAG,uBAAuB,CAAC,KAAK,CAAC;YAChD,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzD,SAAS;YACT,SAAS,EAAE,GAAG;SACf,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,GAAG,iBAAiB,CAAC,WAAW,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC;QACjF,MAAM,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACzC,uEAAuE;QACvE,0EAA0E;QAC1E,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YACjC,wEAAwE;YACxE,OAAO,CAAC,IAAI,CACV,sCAAsC,IAAI,CAAC,IAAI,sBAAsB,IAAI,CAAC,WAAW,KAAK;gBACxF,0EAA0E,CAC7E,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IACrE,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,MAAM,CACV,KAAa,EACb,OAAiE,EAAE;QAEnE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;QAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEhC,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,IAAI,KAAK,QAAQ,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,QAAQ,KAAK,IAAI,CAAC,CAAC;QAC9E,IAAI,SAAS,IAAI,QAAQ,EAAE,CAAC;YAC1B,OAAO,YAAY,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAChF,CAAC;QACD,OAAO,cAAc,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAC3C,CAAC;IAEO,OAAO,CAAC,IAAY;QAC1B,yEAAyE;QACzE,4EAA4E;QAC5E,2EAA2E;QAC3E,6EAA6E;QAC7E,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC;QAClD,8EAA8E;QAC9E,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;2EACuE;IAC/D,KAAK,CAAC,IAAI;QAChB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC5C,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CACtB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CACvF,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;0CAEsC;IAC9B,kBAAkB,CAAC,IAA2B;QACpD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC9C,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CACrD,CAAC;QACF,OAAO,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACtC,CAAC;CACF"}
|
package/dist/tfidf.d.ts
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { EmbeddingProvider } from '@moxxy/sdk';
|
|
2
|
+
/**
|
|
3
|
+
* Built-in zero-dependency TF-IDF embedder. Computes a sparse-but-stored-dense
|
|
4
|
+
* vector per text. Builds vocabulary lazily from the corpus you provide via
|
|
5
|
+
* `fit()`. Once fit, `embed()` produces vectors over that fixed vocabulary.
|
|
6
|
+
*
|
|
7
|
+
* Quality is meh by neural-embedding standards but: deterministic, runs in
|
|
8
|
+
* <1ms for hundreds of entries, no network, no model download, no API key.
|
|
9
|
+
* Real semantic embeddings can be wired by passing a different EmbeddingProvider
|
|
10
|
+
* to MemoryStore.
|
|
11
|
+
*/
|
|
12
|
+
export declare class TfIdfEmbedder implements EmbeddingProvider {
|
|
13
|
+
readonly name = "tfidf";
|
|
14
|
+
private vocab;
|
|
15
|
+
private vocabIndex;
|
|
16
|
+
private idf;
|
|
17
|
+
get dim(): number | 'dynamic';
|
|
18
|
+
/**
|
|
19
|
+
* Compute vocabulary + IDF weights from the corpus. Call this whenever the
|
|
20
|
+
* source corpus changes; embed() must be called only after fit().
|
|
21
|
+
*/
|
|
22
|
+
fit(corpus: ReadonlyArray<string>): void;
|
|
23
|
+
embedSync(text: string): number[];
|
|
24
|
+
embed(texts: ReadonlyArray<string>): Promise<ReadonlyArray<ReadonlyArray<number>>>;
|
|
25
|
+
}
|
|
26
|
+
export declare function cosineSimilarity(a: ReadonlyArray<number>, b: ReadonlyArray<number>): number;
|
|
27
|
+
export declare function l2Normalize(v: number[]): number[];
|
|
28
|
+
export declare function tokenize(text: string): string[];
|
|
29
|
+
//# sourceMappingURL=tfidf.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tfidf.d.ts","sourceRoot":"","sources":["../src/tfidf.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEpD;;;;;;;;;GASG;AACH,qBAAa,aAAc,YAAW,iBAAiB;IACrD,QAAQ,CAAC,IAAI,WAAW;IACxB,OAAO,CAAC,KAAK,CAA6B;IAC1C,OAAO,CAAC,UAAU,CAA6B;IAC/C,OAAO,CAAC,GAAG,CAA6B;IAExC,IAAI,GAAG,IAAI,MAAM,GAAG,SAAS,CAE5B;IAED;;;OAGG;IACH,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,IAAI;IAwBxC,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAiB3B,KAAK,CAAC,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;CAGzF;AAED,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,MAAM,CAK3F;AAED,wBAAgB,WAAW,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAMjD;AAED,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAO/C"}
|
package/dist/tfidf.js
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Built-in zero-dependency TF-IDF embedder. Computes a sparse-but-stored-dense
|
|
3
|
+
* vector per text. Builds vocabulary lazily from the corpus you provide via
|
|
4
|
+
* `fit()`. Once fit, `embed()` produces vectors over that fixed vocabulary.
|
|
5
|
+
*
|
|
6
|
+
* Quality is meh by neural-embedding standards but: deterministic, runs in
|
|
7
|
+
* <1ms for hundreds of entries, no network, no model download, no API key.
|
|
8
|
+
* Real semantic embeddings can be wired by passing a different EmbeddingProvider
|
|
9
|
+
* to MemoryStore.
|
|
10
|
+
*/
|
|
11
|
+
export class TfIdfEmbedder {
|
|
12
|
+
name = 'tfidf';
|
|
13
|
+
vocab = [];
|
|
14
|
+
vocabIndex = new Map();
|
|
15
|
+
idf = [];
|
|
16
|
+
get dim() {
|
|
17
|
+
return this.vocab.length || 'dynamic';
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Compute vocabulary + IDF weights from the corpus. Call this whenever the
|
|
21
|
+
* source corpus changes; embed() must be called only after fit().
|
|
22
|
+
*/
|
|
23
|
+
fit(corpus) {
|
|
24
|
+
const docs = corpus.map(tokenize);
|
|
25
|
+
const df = new Map();
|
|
26
|
+
for (const doc of docs) {
|
|
27
|
+
const seen = new Set();
|
|
28
|
+
for (const t of doc) {
|
|
29
|
+
if (seen.has(t))
|
|
30
|
+
continue;
|
|
31
|
+
seen.add(t);
|
|
32
|
+
df.set(t, (df.get(t) ?? 0) + 1);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
// Filter: keep tokens appearing at least once and at most in 95% of docs.
|
|
36
|
+
const N = Math.max(1, docs.length);
|
|
37
|
+
const maxDf = Math.max(1, Math.ceil(N * 0.95));
|
|
38
|
+
const kept = [];
|
|
39
|
+
for (const [token, count] of df) {
|
|
40
|
+
if (count <= maxDf)
|
|
41
|
+
kept.push([token, count]);
|
|
42
|
+
}
|
|
43
|
+
kept.sort((a, b) => a[0].localeCompare(b[0]));
|
|
44
|
+
this.vocab = kept.map((x) => x[0]);
|
|
45
|
+
this.vocabIndex = new Map(this.vocab.map((t, i) => [t, i]));
|
|
46
|
+
this.idf = kept.map(([, dfi]) => Math.log((N + 1) / (dfi + 1)) + 1);
|
|
47
|
+
}
|
|
48
|
+
embedSync(text) {
|
|
49
|
+
if (this.vocab.length === 0)
|
|
50
|
+
return [];
|
|
51
|
+
const tokens = tokenize(text);
|
|
52
|
+
const tf = new Map();
|
|
53
|
+
for (const t of tokens) {
|
|
54
|
+
const idx = this.vocabIndex.get(t);
|
|
55
|
+
if (idx === undefined)
|
|
56
|
+
continue;
|
|
57
|
+
tf.set(idx, (tf.get(idx) ?? 0) + 1);
|
|
58
|
+
}
|
|
59
|
+
const vec = new Array(this.vocab.length).fill(0);
|
|
60
|
+
const docLen = tokens.length || 1;
|
|
61
|
+
for (const [idx, count] of tf) {
|
|
62
|
+
vec[idx] = (count / docLen) * (this.idf[idx] ?? 1);
|
|
63
|
+
}
|
|
64
|
+
return l2Normalize(vec);
|
|
65
|
+
}
|
|
66
|
+
async embed(texts) {
|
|
67
|
+
return texts.map((t) => this.embedSync(t));
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
export function cosineSimilarity(a, b) {
|
|
71
|
+
const n = Math.min(a.length, b.length);
|
|
72
|
+
let dot = 0;
|
|
73
|
+
for (let i = 0; i < n; i++)
|
|
74
|
+
dot += (a[i] ?? 0) * (b[i] ?? 0);
|
|
75
|
+
return dot;
|
|
76
|
+
}
|
|
77
|
+
export function l2Normalize(v) {
|
|
78
|
+
let sum = 0;
|
|
79
|
+
for (const x of v)
|
|
80
|
+
sum += x * x;
|
|
81
|
+
if (sum === 0)
|
|
82
|
+
return v;
|
|
83
|
+
const norm = Math.sqrt(sum);
|
|
84
|
+
return v.map((x) => x / norm);
|
|
85
|
+
}
|
|
86
|
+
export function tokenize(text) {
|
|
87
|
+
return text
|
|
88
|
+
.toLowerCase()
|
|
89
|
+
.normalize('NFKD')
|
|
90
|
+
.replace(/[̀-ͯ]/g, '')
|
|
91
|
+
.split(/[^a-z0-9_-]+/)
|
|
92
|
+
.filter((t) => t.length >= 2 && !STOPWORDS.has(t));
|
|
93
|
+
}
|
|
94
|
+
const STOPWORDS = new Set([
|
|
95
|
+
'the', 'a', 'an', 'and', 'or', 'but', 'is', 'are', 'was', 'were', 'be',
|
|
96
|
+
'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
|
|
97
|
+
'should', 'could', 'may', 'might', 'must', 'shall', 'to', 'of', 'in', 'for',
|
|
98
|
+
'on', 'at', 'by', 'with', 'from', 'as', 'into', 'through', 'about', 'between',
|
|
99
|
+
'this', 'that', 'these', 'those', 'it', 'its', 'they', 'them', 'their', 'we',
|
|
100
|
+
'us', 'our', 'you', 'your', 'he', 'she', 'his', 'her', 'i', 'me', 'my',
|
|
101
|
+
'so', 'than', 'too', 'very', 'just', 'not', 'no', 'yes', 'if', 'then',
|
|
102
|
+
]);
|
|
103
|
+
//# sourceMappingURL=tfidf.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tfidf.js","sourceRoot":"","sources":["../src/tfidf.ts"],"names":[],"mappings":"AAEA;;;;;;;;;GASG;AACH,MAAM,OAAO,aAAa;IACf,IAAI,GAAG,OAAO,CAAC;IAChB,KAAK,GAA0B,EAAE,CAAC;IAClC,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,GAAG,GAA0B,EAAE,CAAC;IAExC,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,SAAS,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,MAA6B;QAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAClC,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;QACrC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;gBACpB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;oBAAE,SAAS;gBAC1B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACZ,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QACD,0EAA0E;QAC1E,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;QAC/C,MAAM,IAAI,GAA4B,EAAE,CAAC;QACzC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YAChC,IAAI,KAAK,IAAI,KAAK;gBAAE,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACtE,CAAC;IAED,SAAS,CAAC,IAAY;QACpB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC9B,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;QACrC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,GAAG,KAAK,SAAS;gBAAE,SAAS;YAChC,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACtC,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,KAAK,CAAS,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;QAClC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9B,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAA4B;QACtC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC;CACF;AAED,MAAM,UAAU,gBAAgB,CAAC,CAAwB,EAAE,CAAwB;IACjF,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IACvC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7D,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,CAAW;IACrC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,CAAC,IAAI,CAAC;QAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,GAAG,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACxB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5B,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,OAAO,IAAI;SACR,WAAW,EAAE;SACb,SAAS,CAAC,MAAM,CAAC;SACjB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;SACrB,KAAK,CAAC,cAAc,CAAC;SACrB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;IACxB,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI;IACtE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO;IAC3E,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK;IAC3E,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS;IAC7E,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI;IAC5E,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI;IACtE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM;CACtE,CAAC,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@moxxy/plugin-memory",
|
|
3
|
+
"version": "0.27.0",
|
|
4
|
+
"description": "Journal-based long-term memory for moxxy. Markdown files at ~/.moxxy/memory/ with type/description frontmatter + an index. Plus STM selectors over the event log.",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"moxxy",
|
|
7
|
+
"agent",
|
|
8
|
+
"memory",
|
|
9
|
+
"recall",
|
|
10
|
+
"embeddings"
|
|
11
|
+
],
|
|
12
|
+
"homepage": "https://moxxy.ai",
|
|
13
|
+
"bugs": {
|
|
14
|
+
"url": "https://github.com/moxxy-ai/moxxy/issues"
|
|
15
|
+
},
|
|
16
|
+
"repository": {
|
|
17
|
+
"type": "git",
|
|
18
|
+
"url": "git+https://github.com/moxxy-ai/moxxy.git",
|
|
19
|
+
"directory": "packages/plugin-memory"
|
|
20
|
+
},
|
|
21
|
+
"author": "Michal Makowski <michal.makowski97@gmail.com>",
|
|
22
|
+
"license": "MIT",
|
|
23
|
+
"publishConfig": {
|
|
24
|
+
"access": "public"
|
|
25
|
+
},
|
|
26
|
+
"type": "module",
|
|
27
|
+
"main": "./dist/index.js",
|
|
28
|
+
"types": "./dist/index.d.ts",
|
|
29
|
+
"exports": {
|
|
30
|
+
".": {
|
|
31
|
+
"types": "./dist/index.d.ts",
|
|
32
|
+
"import": "./dist/index.js"
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
"files": [
|
|
36
|
+
"dist",
|
|
37
|
+
"src"
|
|
38
|
+
],
|
|
39
|
+
"moxxy": {
|
|
40
|
+
"plugin": {
|
|
41
|
+
"entry": "./dist/index.js",
|
|
42
|
+
"kind": "tools"
|
|
43
|
+
}
|
|
44
|
+
},
|
|
45
|
+
"dependencies": {
|
|
46
|
+
"zod": "^3.24.0",
|
|
47
|
+
"@moxxy/sdk": "0.27.0"
|
|
48
|
+
},
|
|
49
|
+
"devDependencies": {
|
|
50
|
+
"@types/node": "^22.10.0",
|
|
51
|
+
"typescript": "^5.7.3",
|
|
52
|
+
"vitest": "^2.1.8",
|
|
53
|
+
"@moxxy/tsconfig": "0.0.0",
|
|
54
|
+
"@moxxy/vitest-preset": "0.0.0"
|
|
55
|
+
},
|
|
56
|
+
"scripts": {
|
|
57
|
+
"build": "tsc -p tsconfig.json",
|
|
58
|
+
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
59
|
+
"test": "vitest run",
|
|
60
|
+
"clean": "rm -rf dist .turbo"
|
|
61
|
+
}
|
|
62
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
2
|
+
import { promises as fs } from 'node:fs';
|
|
3
|
+
import * as os from 'node:os';
|
|
4
|
+
import * as path from 'node:path';
|
|
5
|
+
import type { LLMProvider, ProviderEvent } from '@moxxy/sdk';
|
|
6
|
+
import { MemoryStore } from './store.js';
|
|
7
|
+
import { buildMemoryConsolidatePlugin } from './consolidate.js';
|
|
8
|
+
|
|
9
|
+
let tmp: string;
|
|
10
|
+
beforeEach(async () => {
|
|
11
|
+
tmp = await fs.mkdtemp(path.join(os.tmpdir(), 'mox-nudge-'));
|
|
12
|
+
});
|
|
13
|
+
afterEach(async () => {
|
|
14
|
+
await fs.rm(tmp, { recursive: true, force: true });
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
const stubProvider: LLMProvider = {
|
|
18
|
+
name: 'stub',
|
|
19
|
+
models: [],
|
|
20
|
+
async *stream(): AsyncIterable<ProviderEvent> {},
|
|
21
|
+
async countTokens() {
|
|
22
|
+
return 0;
|
|
23
|
+
},
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
const baseReq = (system?: string): import('@moxxy/sdk').ProviderRequest => ({
|
|
27
|
+
model: 'stub',
|
|
28
|
+
messages: [],
|
|
29
|
+
...(system !== undefined ? { system } : {}),
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
const ctx = (): import('@moxxy/sdk').TurnContext => ({
|
|
33
|
+
sessionId: 's' as never,
|
|
34
|
+
cwd: '/tmp',
|
|
35
|
+
log: { length: 0, at: () => undefined, slice: () => [], ofType: () => [], byTurn: () => [], toJSON: () => [] },
|
|
36
|
+
env: {},
|
|
37
|
+
turnId: 't' as never,
|
|
38
|
+
iteration: 0,
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
async function fillMemories(store: MemoryStore, n: number): Promise<void> {
|
|
42
|
+
for (let i = 0; i < n; i++) {
|
|
43
|
+
await store.save({
|
|
44
|
+
name: `entry-${i}`,
|
|
45
|
+
type: 'fact',
|
|
46
|
+
description: `Entry ${i}`,
|
|
47
|
+
body: `body ${i}`,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
describe('auto-consolidation nudge hook', () => {
|
|
53
|
+
it('appends a hint to system prompt when memory count exceeds threshold', async () => {
|
|
54
|
+
const store = new MemoryStore({ dir: tmp, embedder: null });
|
|
55
|
+
await fillMemories(store, 5);
|
|
56
|
+
const plugin = buildMemoryConsolidatePlugin(store, () => stubProvider, { autoNudgeThreshold: 3 });
|
|
57
|
+
const result = await plugin.hooks?.onBeforeProviderCall?.(baseReq('be terse'), ctx());
|
|
58
|
+
expect(result).toBeDefined();
|
|
59
|
+
expect(result!.system).toContain('be terse');
|
|
60
|
+
expect(result!.system).toContain('memory_consolidate');
|
|
61
|
+
expect(result!.system).toContain('5 entries');
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it('does NOT append a hint when memory count is below threshold', async () => {
|
|
65
|
+
const store = new MemoryStore({ dir: tmp, embedder: null });
|
|
66
|
+
await fillMemories(store, 2);
|
|
67
|
+
const plugin = buildMemoryConsolidatePlugin(store, () => stubProvider, { autoNudgeThreshold: 10 });
|
|
68
|
+
const result = await plugin.hooks?.onBeforeProviderCall?.(baseReq(), ctx());
|
|
69
|
+
expect(result).toBeUndefined();
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
it('nudges at most once per session (no repeat fires)', async () => {
|
|
73
|
+
const store = new MemoryStore({ dir: tmp, embedder: null });
|
|
74
|
+
await fillMemories(store, 5);
|
|
75
|
+
const plugin = buildMemoryConsolidatePlugin(store, () => stubProvider, { autoNudgeThreshold: 3 });
|
|
76
|
+
const first = await plugin.hooks?.onBeforeProviderCall?.(baseReq(), ctx());
|
|
77
|
+
const second = await plugin.hooks?.onBeforeProviderCall?.(baseReq(), ctx());
|
|
78
|
+
expect(first).toBeDefined();
|
|
79
|
+
expect(second).toBeUndefined();
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it('threshold:0 disables the nudge entirely', async () => {
|
|
83
|
+
const store = new MemoryStore({ dir: tmp, embedder: null });
|
|
84
|
+
const plugin = buildMemoryConsolidatePlugin(store, () => stubProvider, { autoNudgeThreshold: 0 });
|
|
85
|
+
// With threshold 0, no hooks are registered at all
|
|
86
|
+
expect(plugin.hooks?.onBeforeProviderCall).toBeUndefined();
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it('default threshold is 30', async () => {
|
|
90
|
+
const store = new MemoryStore({ dir: tmp, embedder: null });
|
|
91
|
+
await fillMemories(store, 30);
|
|
92
|
+
const plugin = buildMemoryConsolidatePlugin(store, () => stubProvider);
|
|
93
|
+
// 30 is not greater than 30 → no nudge
|
|
94
|
+
const result = await plugin.hooks?.onBeforeProviderCall?.(baseReq(), ctx());
|
|
95
|
+
expect(result).toBeUndefined();
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
});
|