@moxxy/plugin-memory 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/consolidate.d.ts +62 -0
- package/dist/consolidate.d.ts.map +1 -0
- package/dist/consolidate.js +417 -0
- package/dist/consolidate.js.map +1 -0
- package/dist/embedding-cache.d.ts +32 -0
- package/dist/embedding-cache.d.ts.map +1 -0
- package/dist/embedding-cache.js +146 -0
- package/dist/embedding-cache.js.map +1 -0
- package/dist/index.d.ts +31 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +217 -0
- package/dist/index.js.map +1 -0
- package/dist/parse.d.ts +5 -0
- package/dist/parse.d.ts.map +1 -0
- package/dist/parse.js +9 -0
- package/dist/parse.js.map +1 -0
- package/dist/stm.d.ts +20 -0
- package/dist/stm.d.ts.map +1 -0
- package/dist/stm.js +38 -0
- package/dist/stm.js.map +1 -0
- package/dist/store/io.d.ts +13 -0
- package/dist/store/io.d.ts.map +1 -0
- package/dist/store/io.js +124 -0
- package/dist/store/io.js.map +1 -0
- package/dist/store/search.d.ts +10 -0
- package/dist/store/search.d.ts.map +1 -0
- package/dist/store/search.js +160 -0
- package/dist/store/search.js.map +1 -0
- package/dist/store/types.d.ts +33 -0
- package/dist/store/types.d.ts.map +1 -0
- package/dist/store/types.js +11 -0
- package/dist/store/types.js.map +1 -0
- package/dist/store.d.ts +95 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +220 -0
- package/dist/store.js.map +1 -0
- package/dist/tfidf.d.ts +29 -0
- package/dist/tfidf.d.ts.map +1 -0
- package/dist/tfidf.js +103 -0
- package/dist/tfidf.js.map +1 -0
- package/package.json +62 -0
- package/src/consolidate-nudge.test.ts +98 -0
- package/src/consolidate.test.ts +328 -0
- package/src/consolidate.ts +535 -0
- package/src/discovery.test.ts +33 -0
- package/src/embedding-cache.test.ts +268 -0
- package/src/embedding-cache.ts +156 -0
- package/src/index.test.ts +67 -0
- package/src/index.ts +247 -0
- package/src/parse.ts +12 -0
- package/src/stm.test.ts +69 -0
- package/src/stm.ts +48 -0
- package/src/store/io.test.ts +100 -0
- package/src/store/io.ts +138 -0
- package/src/store/search.test.ts +185 -0
- package/src/store/search.ts +197 -0
- package/src/store/types.ts +23 -0
- package/src/store.test.ts +186 -0
- package/src/store.ts +292 -0
- package/src/tfidf.test.ts +59 -0
- package/src/tfidf.ts +105 -0
- package/src/vector-recall.test.ts +88 -0
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
2
|
+
import { promises as fs } from 'node:fs';
|
|
3
|
+
import * as os from 'node:os';
|
|
4
|
+
import * as path from 'node:path';
|
|
5
|
+
import type { EmbeddingProvider, Mutex } from '@moxxy/sdk';
|
|
6
|
+
import { createMutex } from '@moxxy/sdk';
|
|
7
|
+
import { rankByKeywords, recallVector } from './search.js';
|
|
8
|
+
import { EmbeddingIndex } from '../embedding-cache.js';
|
|
9
|
+
import type { MemoryEntry, MemoryType } from './types.js';
|
|
10
|
+
|
|
11
|
+
function entry(
|
|
12
|
+
name: string,
|
|
13
|
+
description: string,
|
|
14
|
+
body: string,
|
|
15
|
+
opts: { type?: MemoryType; tags?: string[] } = {},
|
|
16
|
+
): MemoryEntry {
|
|
17
|
+
return {
|
|
18
|
+
frontmatter: {
|
|
19
|
+
name,
|
|
20
|
+
type: opts.type ?? 'fact',
|
|
21
|
+
description,
|
|
22
|
+
...(opts.tags ? { tags: opts.tags } : {}),
|
|
23
|
+
},
|
|
24
|
+
body,
|
|
25
|
+
path: `/tmp/${name}.md`,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
describe('rankByKeywords', () => {
|
|
30
|
+
it('counts repeated occurrences in the body (no array-split regression)', () => {
|
|
31
|
+
// "deploy" appears 3x in the body; the non-allocating indexOf count must
|
|
32
|
+
// match the old `split(t).length - 1` behavior exactly.
|
|
33
|
+
const e = entry('ci', 'continuous integration', 'deploy then deploy and deploy again');
|
|
34
|
+
const ranked = rankByKeywords([e], 'deploy', 5);
|
|
35
|
+
expect(ranked).toHaveLength(1);
|
|
36
|
+
expect(ranked[0]!.score).toBe(3);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it('weights name and description hits above body-only hits', () => {
|
|
40
|
+
const named = entry('deploy-runbook', 'how to ship', 'steps here');
|
|
41
|
+
const described = entry('runbook', 'deploy guide', 'steps here');
|
|
42
|
+
const bodyOnly = entry('notes', 'misc', 'we deploy on fridays');
|
|
43
|
+
const ranked = rankByKeywords([bodyOnly, described, named], 'deploy', 5);
|
|
44
|
+
// name match (+3) > description match (+2) > body-only (+1)
|
|
45
|
+
expect(ranked.map((r) => r.entry.frontmatter.name)).toEqual([
|
|
46
|
+
'deploy-runbook',
|
|
47
|
+
'runbook',
|
|
48
|
+
'notes',
|
|
49
|
+
]);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it('sums scores across multiple query tokens and ranks by total', () => {
|
|
53
|
+
const both = entry('a', 'alpha beta', 'alpha beta gamma');
|
|
54
|
+
const one = entry('b', 'just alpha', 'nothing else');
|
|
55
|
+
const ranked = rankByKeywords([one, both], 'alpha beta', 5);
|
|
56
|
+
expect(ranked[0]!.entry.frontmatter.name).toBe('a');
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it('drops zero-score entries and respects the limit', () => {
|
|
60
|
+
const hit = entry('x', 'has the word widget', 'widget widget');
|
|
61
|
+
const miss = entry('y', 'unrelated', 'nothing relevant');
|
|
62
|
+
const ranked = rankByKeywords([hit, miss], 'widget', 1);
|
|
63
|
+
expect(ranked).toHaveLength(1);
|
|
64
|
+
expect(ranked[0]!.entry.frontmatter.name).toBe('x');
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it('an empty query matches every entry with score 1', () => {
|
|
68
|
+
const ranked = rankByKeywords([entry('a', 'd', 'b'), entry('c', 'e', 'f')], ' ', 5);
|
|
69
|
+
expect(ranked).toHaveLength(2);
|
|
70
|
+
expect(ranked.every((r) => r.score === 1)).toBe(true);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it('searches tags too', () => {
|
|
74
|
+
const e = entry('a', 'desc', 'body', { tags: ['kubernetes', 'infra'] });
|
|
75
|
+
const ranked = rankByKeywords([e], 'kubernetes', 5);
|
|
76
|
+
expect(ranked).toHaveLength(1);
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
describe('recallVector dimension-drift hardening', () => {
|
|
81
|
+
let tmp: string;
|
|
82
|
+
beforeEach(async () => {
|
|
83
|
+
tmp = await fs.mkdtemp(path.join(os.tmpdir(), 'mox-search-'));
|
|
84
|
+
});
|
|
85
|
+
afterEach(async () => {
|
|
86
|
+
await fs.rm(tmp, { recursive: true, force: true });
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it('degrades (no crash) when a misbehaving embedder under-returns vectors — no-cache path', async () => {
|
|
90
|
+
const mutex: Mutex = createMutex();
|
|
91
|
+
// Asked for corpus.length + 1 vectors, returns only one — the query vector
|
|
92
|
+
// is absent. Must NOT throw a TypeError on `vec.length`/`query.length`.
|
|
93
|
+
const liar: EmbeddingProvider = {
|
|
94
|
+
name: 'liar',
|
|
95
|
+
dim: 2,
|
|
96
|
+
async embed() {
|
|
97
|
+
return [[1, 0]];
|
|
98
|
+
},
|
|
99
|
+
};
|
|
100
|
+
const all = [entry('a', 'da', 'ba'), entry('b', 'db', 'bb')];
|
|
101
|
+
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
102
|
+
try {
|
|
103
|
+
const ranked = await recallVector(all, 'q', 5, liar, null, mutex);
|
|
104
|
+
expect(Array.isArray(ranked)).toBe(true);
|
|
105
|
+
} finally {
|
|
106
|
+
warn.mockRestore();
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
it('returns empty (no crash) when the embedder returns no vectors at all', async () => {
|
|
111
|
+
const mutex: Mutex = createMutex();
|
|
112
|
+
const empty: EmbeddingProvider = {
|
|
113
|
+
name: 'empty',
|
|
114
|
+
dim: 2,
|
|
115
|
+
async embed() {
|
|
116
|
+
return [];
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
120
|
+
try {
|
|
121
|
+
const ranked = await recallVector([entry('a', 'd', 'b')], 'q', 5, empty, null, mutex);
|
|
122
|
+
expect(ranked).toEqual([]);
|
|
123
|
+
} finally {
|
|
124
|
+
warn.mockRestore();
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it('does not poison the persistent cache with an undefined vector when the embedder under-returns', async () => {
|
|
129
|
+
const mutex: Mutex = createMutex();
|
|
130
|
+
const liar: EmbeddingProvider = {
|
|
131
|
+
name: 'liar',
|
|
132
|
+
dim: 2,
|
|
133
|
+
async embed() {
|
|
134
|
+
return [[1, 0]]; // misses=2 + query=1 requested; only 1 returned
|
|
135
|
+
},
|
|
136
|
+
};
|
|
137
|
+
const index = new EmbeddingIndex(tmp, 'liar', 2);
|
|
138
|
+
const all = [entry('a', 'da', 'ba'), entry('b', 'db', 'bb')];
|
|
139
|
+
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
140
|
+
try {
|
|
141
|
+
await recallVector(all, 'q', 5, liar, index, mutex);
|
|
142
|
+
} finally {
|
|
143
|
+
warn.mockRestore();
|
|
144
|
+
}
|
|
145
|
+
// Whatever landed on disk must be well-formed: every cached vector is a real
|
|
146
|
+
// numeric array, never `null`/`undefined` (a poisoned cache breaks all recall).
|
|
147
|
+
const raw = await fs
|
|
148
|
+
.readFile(path.join(tmp, '.embeddings.json'), 'utf8')
|
|
149
|
+
.catch(() => '{"entries":{}}');
|
|
150
|
+
const parsed = JSON.parse(raw) as { entries?: Record<string, { vector?: unknown }> };
|
|
151
|
+
for (const e of Object.values(parsed.entries ?? {})) {
|
|
152
|
+
expect(Array.isArray(e.vector)).toBe(true);
|
|
153
|
+
}
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
it('skips a cached entry whose vector dim no longer matches the query (no silently-wrong score)', async () => {
|
|
157
|
+
const mutex: Mutex = createMutex();
|
|
158
|
+
// Pre-seed the persistent cache with a dim-3 vector for `stale`.
|
|
159
|
+
const seed = new EmbeddingIndex(tmp, 'drift');
|
|
160
|
+
const stale = entry('stale', 'stale desc', 'stale body');
|
|
161
|
+
const corpusText = ['stale', 'stale desc', '', 'stale body'].join('\n');
|
|
162
|
+
seed.set('stale', corpusText, [1, 0, 0]);
|
|
163
|
+
await seed.flush();
|
|
164
|
+
|
|
165
|
+
// An embedder that now returns dim-2 vectors (model/dim drift). The cached
|
|
166
|
+
// `stale` entry is dim-3; the fresh query is dim-2.
|
|
167
|
+
const driftEmbedder: EmbeddingProvider = {
|
|
168
|
+
name: 'drift',
|
|
169
|
+
dim: 2,
|
|
170
|
+
async embed(texts) {
|
|
171
|
+
return texts.map(() => [1, 0]);
|
|
172
|
+
},
|
|
173
|
+
};
|
|
174
|
+
const index = new EmbeddingIndex(tmp, 'drift'); // no dim → cache not invalidated on dim
|
|
175
|
+
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
176
|
+
try {
|
|
177
|
+
const ranked = await recallVector([stale], 'q', 5, driftEmbedder, index, mutex);
|
|
178
|
+
// The mismatched-dim entry is dropped, not ranked on a truncated basis.
|
|
179
|
+
expect(ranked).toHaveLength(0);
|
|
180
|
+
expect(warn).toHaveBeenCalled();
|
|
181
|
+
} finally {
|
|
182
|
+
warn.mockRestore();
|
|
183
|
+
}
|
|
184
|
+
});
|
|
185
|
+
});
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import type { EmbeddingProvider, Mutex } from '@moxxy/sdk';
|
|
2
|
+
import { TfIdfEmbedder, cosineSimilarity, tokenize } from '../tfidf.js';
|
|
3
|
+
import type { EmbeddingIndex } from '../embedding-cache.js';
|
|
4
|
+
import type { MemoryEntry } from './types.js';
|
|
5
|
+
|
|
6
|
+
export interface RankedMemory {
|
|
7
|
+
readonly entry: MemoryEntry;
|
|
8
|
+
readonly score: number;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export async function recallVector(
|
|
12
|
+
all: ReadonlyArray<MemoryEntry>,
|
|
13
|
+
query: string,
|
|
14
|
+
limit: number,
|
|
15
|
+
embedder: EmbeddingProvider,
|
|
16
|
+
index: EmbeddingIndex | null,
|
|
17
|
+
mutex: Mutex,
|
|
18
|
+
): Promise<ReadonlyArray<RankedMemory>> {
|
|
19
|
+
const corpus = all.map((e) => entryForEmbedding(e));
|
|
20
|
+
|
|
21
|
+
// TF-IDF special-cases the persistent cache (vocab is corpus-wide).
|
|
22
|
+
if (embedder instanceof TfIdfEmbedder) {
|
|
23
|
+
embedder.fit([...corpus, query]);
|
|
24
|
+
return rankAllFresh(all, corpus, query, limit, embedder);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// Neural embedders: consult the persistent cache, only embed misses + query.
|
|
28
|
+
if (index) {
|
|
29
|
+
// The index load->lookup->set->prune->flush cycle mutates the shared
|
|
30
|
+
// on-disk cache, so it must run under the store's write mutex — otherwise
|
|
31
|
+
// two concurrent recalls (or a recall racing forget()'s rebuildIndex)
|
|
32
|
+
// read the same snapshot and clobber each other's writes. Only the cache
|
|
33
|
+
// bookkeeping is serialized; the pure cosine ranking stays outside.
|
|
34
|
+
const { vectors, queryVec } = await mutex.run(async () => {
|
|
35
|
+
await index.load();
|
|
36
|
+
const cached: Array<ReadonlyArray<number> | null> = [];
|
|
37
|
+
const misses: { index: number; text: string }[] = [];
|
|
38
|
+
for (let i = 0; i < all.length; i++) {
|
|
39
|
+
const hit = index.lookup(all[i]!.frontmatter.name, corpus[i]!);
|
|
40
|
+
cached.push(hit);
|
|
41
|
+
if (!hit) misses.push({ index: i, text: corpus[i]! });
|
|
42
|
+
}
|
|
43
|
+
const queryIdx = misses.length;
|
|
44
|
+
const toEmbed = [...misses.map((m) => m.text), query];
|
|
45
|
+
const fresh = await embedder.embed(toEmbed);
|
|
46
|
+
const qVec = fresh[queryIdx];
|
|
47
|
+
// Map each missed corpus index to its freshly-embedded vector so the
|
|
48
|
+
// stitch loop below stays O(1) per entry instead of scanning `misses`.
|
|
49
|
+
// A misbehaving embedder may under-return; only valid vectors get mapped,
|
|
50
|
+
// and missing ones stay absent so rankCosine drops them (never crashes).
|
|
51
|
+
const freshByEntryIndex = new Map<number, ReadonlyArray<number>>();
|
|
52
|
+
for (const [j, m] of misses.entries()) {
|
|
53
|
+
const v = fresh[j];
|
|
54
|
+
if (Array.isArray(v)) freshByEntryIndex.set(m.index, v);
|
|
55
|
+
}
|
|
56
|
+
// Stitch results: cached + freshly-embedded. Holes (undefined) survive to
|
|
57
|
+
// rankCosine, which skips them rather than throwing.
|
|
58
|
+
const vecs: Array<ReadonlyArray<number> | undefined> = [];
|
|
59
|
+
for (let i = 0; i < all.length; i++) {
|
|
60
|
+
vecs.push(cached[i] ?? freshByEntryIndex.get(i));
|
|
61
|
+
}
|
|
62
|
+
// Persist ONLY valid fresh vectors — never write an `undefined`/non-array
|
|
63
|
+
// vector to the on-disk cache, which would poison it permanently (every
|
|
64
|
+
// future recall would read back a corrupt entry).
|
|
65
|
+
for (const [j, m] of misses.entries()) {
|
|
66
|
+
const v = fresh[j];
|
|
67
|
+
if (Array.isArray(v)) index.set(all[m.index]!.frontmatter.name, m.text, v);
|
|
68
|
+
}
|
|
69
|
+
index.prune(all.map((e) => e.frontmatter.name));
|
|
70
|
+
await index.flush();
|
|
71
|
+
return { vectors: vecs, queryVec: qVec };
|
|
72
|
+
});
|
|
73
|
+
return rankCosine(all, vectors, queryVec, limit);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// No cache configured — embed everything every time.
|
|
77
|
+
return rankAllFresh(all, corpus, query, limit, embedder);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// Embed `[...corpus, query]` in one batch, then cosine-rank the corpus against
|
|
81
|
+
// the (last) query vector. Shared by the TF-IDF and no-cache branches.
|
|
82
|
+
async function rankAllFresh(
|
|
83
|
+
all: ReadonlyArray<MemoryEntry>,
|
|
84
|
+
corpus: ReadonlyArray<string>,
|
|
85
|
+
query: string,
|
|
86
|
+
limit: number,
|
|
87
|
+
embedder: EmbeddingProvider,
|
|
88
|
+
): Promise<ReadonlyArray<RankedMemory>> {
|
|
89
|
+
const vectors = await embedder.embed([...corpus, query]);
|
|
90
|
+
// The query is embedded at index `corpus.length`. Address it by that fixed
|
|
91
|
+
// position (not `vectors.length - 1`): if the embedder under-returns and drops
|
|
92
|
+
// the query, `length - 1` would silently grab the last CORPUS vector and use
|
|
93
|
+
// it as the query. `vectors[corpus.length]` is `undefined` in that case, which
|
|
94
|
+
// rankCosine handles by returning an empty result — degrade, never mislead.
|
|
95
|
+
const queryVec = vectors[corpus.length];
|
|
96
|
+
return rankCosine(all, vectors.slice(0, all.length), queryVec, limit);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export function rankByKeywords(
|
|
100
|
+
all: ReadonlyArray<MemoryEntry>,
|
|
101
|
+
query: string,
|
|
102
|
+
limit: number,
|
|
103
|
+
): ReadonlyArray<RankedMemory> {
|
|
104
|
+
const tokens = tokenize(query);
|
|
105
|
+
return all
|
|
106
|
+
.map((entry) => ({ entry, score: scoreEntry(entry, tokens) }))
|
|
107
|
+
.filter((r) => r.score > 0)
|
|
108
|
+
.sort((a, b) => b.score - a.score)
|
|
109
|
+
.slice(0, limit);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function rankCosine(
|
|
113
|
+
entries: ReadonlyArray<MemoryEntry>,
|
|
114
|
+
vectors: ReadonlyArray<ReadonlyArray<number> | undefined>,
|
|
115
|
+
query: ReadonlyArray<number> | undefined,
|
|
116
|
+
limit: number,
|
|
117
|
+
): ReadonlyArray<RankedMemory> {
|
|
118
|
+
// A misbehaving/hostile embedder can return fewer vectors than requested (or
|
|
119
|
+
// a non-array element) — the EmbeddingProvider.embed contract promises order
|
|
120
|
+
// but not count. Without this guard `query.length`/`vec.length` throws an
|
|
121
|
+
// opaque TypeError and recall crashes instead of degrading. A bad embedder
|
|
122
|
+
// must yield an empty/partial result set, never crash memory_recall.
|
|
123
|
+
if (!Array.isArray(query) || query.length === 0) {
|
|
124
|
+
console.warn(
|
|
125
|
+
'[plugin-memory] embedder returned no usable query vector; skipping vector recall',
|
|
126
|
+
);
|
|
127
|
+
return [];
|
|
128
|
+
}
|
|
129
|
+
const ranked: RankedMemory[] = [];
|
|
130
|
+
for (let i = 0; i < entries.length; i++) {
|
|
131
|
+
const vec = vectors[i];
|
|
132
|
+
// cosineSimilarity silently truncates to the shorter vector, so a stale
|
|
133
|
+
// cached vector or a provider quirk of the wrong dimensionality would
|
|
134
|
+
// produce a plausible-but-wrong score (invisible corruption). A missing
|
|
135
|
+
// vector (embedder under-returned) is the same hazard. Skip the entry
|
|
136
|
+
// loudly instead of ranking it on a mismatched or absent basis.
|
|
137
|
+
if (!Array.isArray(vec) || vec.length !== query.length) {
|
|
138
|
+
console.warn(
|
|
139
|
+
`[plugin-memory] skipping '${entries[i]!.frontmatter.name}' in recall: ` +
|
|
140
|
+
`vector dim ${Array.isArray(vec) ? vec.length : 'missing'} != query dim ${query.length} (cache/embedder drift)`,
|
|
141
|
+
);
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
const score = cosineSimilarity(vec, query);
|
|
145
|
+
if (score > 0) ranked.push({ entry: entries[i]!, score });
|
|
146
|
+
}
|
|
147
|
+
ranked.sort((a, b) => b.score - a.score);
|
|
148
|
+
return ranked.slice(0, limit);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// Count non-overlapping occurrences of `needle` in `haystack` without
|
|
152
|
+
// allocating the intermediate array `haystack.split(needle)` would build.
|
|
153
|
+
// Identical result to `split(needle).length - 1` for the non-empty tokens
|
|
154
|
+
// `tokenize` yields. Tokens are `[a-z0-9_-]+`, so there are no overlap or
|
|
155
|
+
// empty-needle edge cases to worry about.
|
|
156
|
+
function countOccurrences(haystack: string, needle: string): number {
|
|
157
|
+
let n = 0;
|
|
158
|
+
let i = haystack.indexOf(needle);
|
|
159
|
+
while (i !== -1) {
|
|
160
|
+
n += 1;
|
|
161
|
+
i = haystack.indexOf(needle, i + needle.length);
|
|
162
|
+
}
|
|
163
|
+
return n;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function entryForEmbedding(entry: MemoryEntry): string {
|
|
167
|
+
return [
|
|
168
|
+
entry.frontmatter.name,
|
|
169
|
+
entry.frontmatter.description,
|
|
170
|
+
(entry.frontmatter.tags ?? []).join(' '),
|
|
171
|
+
entry.body,
|
|
172
|
+
].join('\n');
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function scoreEntry(entry: MemoryEntry, tokens: ReadonlyArray<string>): number {
|
|
176
|
+
if (tokens.length === 0) return 1;
|
|
177
|
+
const haystack = (
|
|
178
|
+
entry.frontmatter.name +
|
|
179
|
+
' ' +
|
|
180
|
+
entry.frontmatter.description +
|
|
181
|
+
' ' +
|
|
182
|
+
(entry.frontmatter.tags ?? []).join(' ') +
|
|
183
|
+
' ' +
|
|
184
|
+
entry.body
|
|
185
|
+
).toLowerCase();
|
|
186
|
+
let score = 0;
|
|
187
|
+
for (const t of tokens) {
|
|
188
|
+
if (!t) continue;
|
|
189
|
+
const matches = countOccurrences(haystack, t);
|
|
190
|
+
if (matches > 0) {
|
|
191
|
+
score += matches;
|
|
192
|
+
if (entry.frontmatter.name.toLowerCase().includes(t)) score += 3;
|
|
193
|
+
if (entry.frontmatter.description.toLowerCase().includes(t)) score += 2;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
return score;
|
|
197
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
|
|
3
|
+
export const memoryTypeSchema = z.enum(['fact', 'preference', 'project', 'reference']);
|
|
4
|
+
export type MemoryType = z.infer<typeof memoryTypeSchema>;
|
|
5
|
+
|
|
6
|
+
export const memoryFrontmatterSchema = z.object({
|
|
7
|
+
name: z.string().min(1).max(120).regex(/^[a-z0-9][a-z0-9-]*$/, 'name must be slug-like'),
|
|
8
|
+
type: memoryTypeSchema,
|
|
9
|
+
description: z.string().min(1).max(280),
|
|
10
|
+
tags: z.array(z.string().min(1)).optional(),
|
|
11
|
+
createdAt: z.string().optional(),
|
|
12
|
+
updatedAt: z.string().optional(),
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
export type MemoryFrontmatter = z.infer<typeof memoryFrontmatterSchema>;
|
|
16
|
+
|
|
17
|
+
export interface MemoryEntry {
|
|
18
|
+
readonly frontmatter: MemoryFrontmatter;
|
|
19
|
+
readonly body: string;
|
|
20
|
+
readonly path: string;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export type RecallMode = 'auto' | 'vector' | 'keyword';
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
2
|
+
import { promises as fs } from 'node:fs';
|
|
3
|
+
import * as os from 'node:os';
|
|
4
|
+
import * as path from 'node:path';
|
|
5
|
+
import { MemoryStore } from './store.js';
|
|
6
|
+
|
|
7
|
+
let tmp: string;
|
|
8
|
+
const newStore = () => new MemoryStore({ dir: tmp });
|
|
9
|
+
|
|
10
|
+
beforeEach(async () => {
|
|
11
|
+
tmp = await fs.mkdtemp(path.join(os.tmpdir(), 'mox-mem-'));
|
|
12
|
+
});
|
|
13
|
+
afterEach(async () => {
|
|
14
|
+
await fs.rm(tmp, { recursive: true, force: true });
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
describe('MemoryStore', () => {
|
|
18
|
+
it('saves and round-trips an entry', async () => {
|
|
19
|
+
const store = newStore();
|
|
20
|
+
await store.save({ name: 'foo', type: 'fact', description: 'foo desc', body: 'foo body' });
|
|
21
|
+
const got = await store.get('foo');
|
|
22
|
+
expect(got?.frontmatter.name).toBe('foo');
|
|
23
|
+
expect(got?.frontmatter.type).toBe('fact');
|
|
24
|
+
expect(got?.body).toBe('foo body');
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
it('emits valid Markdown with frontmatter on disk', async () => {
|
|
28
|
+
const store = newStore();
|
|
29
|
+
await store.save({ name: 'foo', type: 'fact', description: 'desc', body: 'body' });
|
|
30
|
+
const raw = await fs.readFile(path.join(tmp, 'foo.md'), 'utf8');
|
|
31
|
+
expect(raw).toContain('---');
|
|
32
|
+
expect(raw).toContain('name: foo');
|
|
33
|
+
expect(raw).toContain('type: fact');
|
|
34
|
+
expect(raw).toMatch(/createdAt: .+/);
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it('serializes concurrent saves so MEMORY.md keeps every entry', async () => {
|
|
38
|
+
const store = newStore();
|
|
39
|
+
// Without the per-instance mutex, each save's rebuildIndex can read the
|
|
40
|
+
// entry list before a sibling save has written its file, so a concurrent
|
|
41
|
+
// save's row is dropped from MEMORY.md (and the writes race the file).
|
|
42
|
+
await Promise.all([
|
|
43
|
+
store.save({ name: 'a', type: 'fact', description: 'A.', body: 'a' }),
|
|
44
|
+
store.save({ name: 'b', type: 'fact', description: 'B.', body: 'b' }),
|
|
45
|
+
store.save({ name: 'c', type: 'fact', description: 'C.', body: 'c' }),
|
|
46
|
+
]);
|
|
47
|
+
const idx = await fs.readFile(path.join(tmp, 'MEMORY.md'), 'utf8');
|
|
48
|
+
expect(idx).toContain('[a](a.md)');
|
|
49
|
+
expect(idx).toContain('[b](b.md)');
|
|
50
|
+
expect(idx).toContain('[c](c.md)');
|
|
51
|
+
expect(await store.list()).toHaveLength(3);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it('rebuilds the MEMORY.md index after each save', async () => {
|
|
55
|
+
const store = newStore();
|
|
56
|
+
await store.save({ name: 'a', type: 'fact', description: 'A.', body: '...' });
|
|
57
|
+
await store.save({ name: 'b', type: 'preference', description: 'B.', body: '...' });
|
|
58
|
+
const idx = await fs.readFile(path.join(tmp, 'MEMORY.md'), 'utf8');
|
|
59
|
+
expect(idx).toContain('## fact');
|
|
60
|
+
expect(idx).toContain('## preference');
|
|
61
|
+
expect(idx).toContain('[a](a.md)');
|
|
62
|
+
expect(idx).toContain('[b](b.md)');
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it('list filters by type', async () => {
|
|
66
|
+
const store = newStore();
|
|
67
|
+
await store.save({ name: 'a', type: 'fact', description: 'A', body: '.' });
|
|
68
|
+
await store.save({ name: 'b', type: 'preference', description: 'B', body: '.' });
|
|
69
|
+
const facts = await store.list('fact');
|
|
70
|
+
expect(facts).toHaveLength(1);
|
|
71
|
+
expect(facts[0]!.frontmatter.name).toBe('a');
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it('update preserves createdAt but bumps updatedAt', async () => {
|
|
75
|
+
const store = newStore();
|
|
76
|
+
await store.save({ name: 'foo', type: 'fact', description: 'd', body: 'orig' });
|
|
77
|
+
const first = (await store.get('foo'))!;
|
|
78
|
+
await new Promise((r) => setTimeout(r, 10));
|
|
79
|
+
await store.update('foo', { body: 'updated' });
|
|
80
|
+
const second = (await store.get('foo'))!;
|
|
81
|
+
expect(second.frontmatter.createdAt).toBe(first.frontmatter.createdAt);
|
|
82
|
+
expect(second.frontmatter.updatedAt).not.toBe(first.frontmatter.updatedAt);
|
|
83
|
+
expect(second.body).toBe('updated');
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it('forget deletes the file and updates the index', async () => {
|
|
87
|
+
const store = newStore();
|
|
88
|
+
await store.save({ name: 'a', type: 'fact', description: 'A', body: '.' });
|
|
89
|
+
expect(await store.forget('a')).toBe(true);
|
|
90
|
+
expect(await store.forget('a')).toBe(false);
|
|
91
|
+
const idx = await fs.readFile(path.join(tmp, 'MEMORY.md'), 'utf8').catch(() => '');
|
|
92
|
+
expect(idx).not.toContain('[a]');
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it('returns empty list when dir does not exist', async () => {
|
|
96
|
+
const store = new MemoryStore({ dir: path.join(tmp, 'nope') });
|
|
97
|
+
expect(await store.list()).toEqual([]);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it('recall ranks by token frequency + name/description matches', async () => {
|
|
101
|
+
const store = newStore();
|
|
102
|
+
await store.save({
|
|
103
|
+
name: 'team-likes-trpc',
|
|
104
|
+
type: 'preference',
|
|
105
|
+
description: 'The team prefers tRPC over REST.',
|
|
106
|
+
body: 'When generating new endpoints, scaffold a tRPC router.',
|
|
107
|
+
});
|
|
108
|
+
await store.save({
|
|
109
|
+
name: 'prod-postgres',
|
|
110
|
+
type: 'project',
|
|
111
|
+
description: 'Production runs Postgres 16.',
|
|
112
|
+
body: 'All migrations target Postgres 16. Use `pg_dump` for backups.',
|
|
113
|
+
});
|
|
114
|
+
const matches = await store.recall('trpc endpoints');
|
|
115
|
+
expect(matches[0]!.entry.frontmatter.name).toBe('team-likes-trpc');
|
|
116
|
+
const pg = await store.recall('postgres');
|
|
117
|
+
expect(pg[0]!.entry.frontmatter.name).toBe('prod-postgres');
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it('rejects invalid frontmatter at save time via the schema', async () => {
|
|
121
|
+
const store = newStore();
|
|
122
|
+
await expect(
|
|
123
|
+
store.save({ name: 'Bad Name', type: 'fact', description: 'd', body: '.' } as never),
|
|
124
|
+
).rejects.toThrow();
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
it('maintains the index incrementally — a save does not re-read every memory file', async () => {
|
|
128
|
+
const store = newStore();
|
|
129
|
+
// First save hydrates the row cache from disk (one readdir + reads).
|
|
130
|
+
await store.save({ name: 'a', type: 'fact', description: 'A', body: 'a' });
|
|
131
|
+
const readdirSpy = vi.spyOn(fs, 'readdir');
|
|
132
|
+
const readFileSpy = vi.spyOn(fs, 'readFile');
|
|
133
|
+
try {
|
|
134
|
+
await store.save({ name: 'b', type: 'fact', description: 'B', body: 'b' });
|
|
135
|
+
await store.save({ name: 'c', type: 'preference', description: 'C', body: 'c' });
|
|
136
|
+
// No directory re-scan; the only reads are each entry's own
|
|
137
|
+
// read-modify-write (createdAt preservation), not a full re-index.
|
|
138
|
+
expect(readdirSpy).not.toHaveBeenCalled();
|
|
139
|
+
expect(readFileSpy.mock.calls.length).toBeLessThanOrEqual(2);
|
|
140
|
+
} finally {
|
|
141
|
+
readdirSpy.mockRestore();
|
|
142
|
+
readFileSpy.mockRestore();
|
|
143
|
+
}
|
|
144
|
+
const idx = await fs.readFile(path.join(tmp, 'MEMORY.md'), 'utf8');
|
|
145
|
+
expect(idx).toContain('[a](a.md)');
|
|
146
|
+
expect(idx).toContain('[b](b.md)');
|
|
147
|
+
expect(idx).toContain('[c](c.md)');
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
it('incremental index survives forget and a fresh store rehydrates from disk', async () => {
|
|
151
|
+
const store = newStore();
|
|
152
|
+
await store.save({ name: 'a', type: 'fact', description: 'A', body: 'a' });
|
|
153
|
+
await store.save({ name: 'b', type: 'fact', description: 'B', body: 'b' });
|
|
154
|
+
await store.forget('a');
|
|
155
|
+
const idx = await fs.readFile(path.join(tmp, 'MEMORY.md'), 'utf8');
|
|
156
|
+
expect(idx).not.toContain('[a](a.md)');
|
|
157
|
+
expect(idx).toContain('[b](b.md)');
|
|
158
|
+
// A new instance (cold cache) sees the same state.
|
|
159
|
+
const fresh = newStore();
|
|
160
|
+
await fresh.save({ name: 'c', type: 'fact', description: 'C', body: 'c' });
|
|
161
|
+
const idx2 = await fs.readFile(path.join(tmp, 'MEMORY.md'), 'utf8');
|
|
162
|
+
expect(idx2).not.toContain('[a](a.md)');
|
|
163
|
+
expect(idx2).toContain('[b](b.md)');
|
|
164
|
+
expect(idx2).toContain('[c](c.md)');
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
it('soft cap is warn-only: saves past maxMemories succeed, capStatus flips, nothing evicted', async () => {
|
|
168
|
+
const store = new MemoryStore({ dir: tmp, maxMemories: 2 });
|
|
169
|
+
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
170
|
+
try {
|
|
171
|
+
await store.save({ name: 'a', type: 'fact', description: 'A', body: 'a' });
|
|
172
|
+
await store.save({ name: 'b', type: 'fact', description: 'B', body: 'b' });
|
|
173
|
+
expect((await store.capStatus()).over).toBe(false);
|
|
174
|
+
expect(warn).not.toHaveBeenCalled();
|
|
175
|
+
|
|
176
|
+
await store.save({ name: 'c', type: 'fact', description: 'C', body: 'c' });
|
|
177
|
+
const cap = await store.capStatus();
|
|
178
|
+
expect(cap).toEqual({ count: 3, max: 2, over: true });
|
|
179
|
+
expect(warn).toHaveBeenCalledOnce();
|
|
180
|
+
// Warn-only: every entry is still on disk and listed.
|
|
181
|
+
expect((await store.list()).map((e) => e.frontmatter.name).sort()).toEqual(['a', 'b', 'c']);
|
|
182
|
+
} finally {
|
|
183
|
+
warn.mockRestore();
|
|
184
|
+
}
|
|
185
|
+
});
|
|
186
|
+
});
|