@teammates/recall 0.4.1 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/embeddings.test.d.ts +1 -0
- package/dist/embeddings.test.js +81 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +3 -1
- package/dist/indexer.d.ts +5 -0
- package/dist/indexer.js +24 -3
- package/dist/indexer.test.js +65 -3
- package/dist/memory-index.d.ts +34 -0
- package/dist/memory-index.js +118 -0
- package/dist/memory-index.test.d.ts +1 -0
- package/dist/memory-index.test.js +96 -0
- package/dist/query-expansion.d.ts +20 -0
- package/dist/query-expansion.js +92 -0
- package/dist/query-expansion.test.d.ts +1 -0
- package/dist/query-expansion.test.js +79 -0
- package/dist/search.d.ts +20 -0
- package/dist/search.js +50 -1
- package/dist/search.test.js +263 -19
- package/package.json +1 -1
- package/src/embeddings.test.ts +106 -0
- package/src/index.ts +9 -1
- package/src/indexer.test.ts +78 -3
- package/src/indexer.ts +26 -3
- package/src/memory-index.test.ts +149 -0
- package/src/memory-index.ts +151 -0
- package/src/query-expansion.test.ts +90 -0
- package/src/query-expansion.ts +105 -0
- package/src/search.test.ts +386 -49
- package/src/search.ts +67 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { describe, expect, it, vi } from "vitest";
|
|
2
|
+
import { LocalEmbeddings } from "./embeddings.js";
|
|
3
|
+
// Mock @huggingface/transformers to avoid loading the real model
|
|
4
|
+
vi.mock("@huggingface/transformers", () => ({
|
|
5
|
+
pipeline: vi.fn(async () => {
|
|
6
|
+
// Return a fake extractor function
|
|
7
|
+
return async (texts, _opts) => ({
|
|
8
|
+
tolist: () => texts.map(() => new Array(384).fill(0.1)),
|
|
9
|
+
});
|
|
10
|
+
}),
|
|
11
|
+
}));
|
|
12
|
+
describe("LocalEmbeddings", () => {
|
|
13
|
+
it("has maxTokens set to 256", () => {
|
|
14
|
+
const emb = new LocalEmbeddings();
|
|
15
|
+
expect(emb.maxTokens).toBe(256);
|
|
16
|
+
});
|
|
17
|
+
describe("createEmbeddings", () => {
|
|
18
|
+
it("returns embeddings for a single string input", async () => {
|
|
19
|
+
const emb = new LocalEmbeddings();
|
|
20
|
+
const result = await emb.createEmbeddings("hello world");
|
|
21
|
+
expect(result.status).toBe("success");
|
|
22
|
+
expect(result.output).toHaveLength(1);
|
|
23
|
+
expect(result.output[0]).toHaveLength(384);
|
|
24
|
+
});
|
|
25
|
+
it("returns embeddings for an array of strings", async () => {
|
|
26
|
+
const emb = new LocalEmbeddings();
|
|
27
|
+
const result = await emb.createEmbeddings(["hello", "world"]);
|
|
28
|
+
expect(result.status).toBe("success");
|
|
29
|
+
expect(result.output).toHaveLength(2);
|
|
30
|
+
expect(result.output[0]).toHaveLength(384);
|
|
31
|
+
expect(result.output[1]).toHaveLength(384);
|
|
32
|
+
});
|
|
33
|
+
it("returns empty output for empty string input", async () => {
|
|
34
|
+
const emb = new LocalEmbeddings();
|
|
35
|
+
const result = await emb.createEmbeddings("");
|
|
36
|
+
expect(result.status).toBe("success");
|
|
37
|
+
expect(result.output).toHaveLength(0);
|
|
38
|
+
});
|
|
39
|
+
it("returns empty output for whitespace-only input", async () => {
|
|
40
|
+
const emb = new LocalEmbeddings();
|
|
41
|
+
const result = await emb.createEmbeddings(" ");
|
|
42
|
+
expect(result.status).toBe("success");
|
|
43
|
+
expect(result.output).toHaveLength(0);
|
|
44
|
+
});
|
|
45
|
+
it("filters out empty strings from array input", async () => {
|
|
46
|
+
const emb = new LocalEmbeddings();
|
|
47
|
+
const result = await emb.createEmbeddings(["hello", "", " ", "world"]);
|
|
48
|
+
expect(result.status).toBe("success");
|
|
49
|
+
expect(result.output).toHaveLength(2);
|
|
50
|
+
});
|
|
51
|
+
it("returns error status when extractor throws", async () => {
|
|
52
|
+
const { pipeline } = await import("@huggingface/transformers");
|
|
53
|
+
// Override mock to throw
|
|
54
|
+
pipeline.mockImplementationOnce(async () => {
|
|
55
|
+
return async () => {
|
|
56
|
+
throw new Error("Model load failed");
|
|
57
|
+
};
|
|
58
|
+
});
|
|
59
|
+
const emb = new LocalEmbeddings("bad-model");
|
|
60
|
+
const result = await emb.createEmbeddings("test");
|
|
61
|
+
expect(result.status).toBe("error");
|
|
62
|
+
expect(result.message).toBe("Model load failed");
|
|
63
|
+
});
|
|
64
|
+
it("reuses the extractor on subsequent calls (lazy init)", async () => {
|
|
65
|
+
const { pipeline } = await import("@huggingface/transformers");
|
|
66
|
+
pipeline.mockClear();
|
|
67
|
+
const emb = new LocalEmbeddings();
|
|
68
|
+
await emb.createEmbeddings("first");
|
|
69
|
+
await emb.createEmbeddings("second");
|
|
70
|
+
// pipeline() should only be called once (lazy singleton)
|
|
71
|
+
expect(pipeline).toHaveBeenCalledTimes(1);
|
|
72
|
+
});
|
|
73
|
+
it("accepts a custom model name", async () => {
|
|
74
|
+
const { pipeline } = await import("@huggingface/transformers");
|
|
75
|
+
pipeline.mockClear();
|
|
76
|
+
const emb = new LocalEmbeddings("custom/model-name");
|
|
77
|
+
await emb.createEmbeddings("test");
|
|
78
|
+
expect(pipeline).toHaveBeenCalledWith("feature-extraction", "custom/model-name", expect.objectContaining({ dtype: "fp32" }));
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
});
|
package/dist/index.d.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
1
|
export { LocalEmbeddings } from "./embeddings.js";
|
|
2
2
|
export { Indexer, type IndexerConfig } from "./indexer.js";
|
|
3
|
-
export {
|
|
3
|
+
export { matchMemoryCatalog, scanMemoryCatalog } from "./memory-index.js";
|
|
4
|
+
export { buildQueryVariations, extractKeywords } from "./query-expansion.js";
|
|
5
|
+
export { type MultiSearchOptions, type SearchOptions, type SearchResult, multiSearch, search, } from "./search.js";
|
package/dist/index.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
1
|
export { LocalEmbeddings } from "./embeddings.js";
|
|
2
2
|
export { Indexer } from "./indexer.js";
|
|
3
|
-
export {
|
|
3
|
+
export { matchMemoryCatalog, scanMemoryCatalog } from "./memory-index.js";
|
|
4
|
+
export { buildQueryVariations, extractKeywords } from "./query-expansion.js";
|
|
5
|
+
export { multiSearch, search, } from "./search.js";
|
package/dist/indexer.d.ts
CHANGED
|
@@ -47,6 +47,11 @@ export declare class Indexer {
|
|
|
47
47
|
* Upserts new/changed files without a full rebuild.
|
|
48
48
|
*/
|
|
49
49
|
syncTeammate(teammate: string): Promise<number>;
|
|
50
|
+
/**
|
|
51
|
+
* Delete a document from a teammate's index by URI.
|
|
52
|
+
* Used to purge stale daily logs after they age out on disk.
|
|
53
|
+
*/
|
|
54
|
+
deleteDocument(teammate: string, uri: string): Promise<void>;
|
|
50
55
|
/**
|
|
51
56
|
* Sync indexes for all teammates.
|
|
52
57
|
*/
|
package/dist/indexer.js
CHANGED
|
@@ -54,16 +54,20 @@ export class Indexer {
|
|
|
54
54
|
catch {
|
|
55
55
|
// No WISDOM.md
|
|
56
56
|
}
|
|
57
|
-
// memory/*.md — typed memories
|
|
57
|
+
// memory/*.md — typed memories + daily logs (day 2+)
|
|
58
|
+
// Today's daily log is excluded (still being written). Older dailies are
|
|
59
|
+
// indexed so recall can surface high-resolution episodic context beyond
|
|
60
|
+
// the 7-day prompt window. Dailies older than 30 days are purged elsewhere.
|
|
58
61
|
const memoryDir = path.join(teammateDir, "memory");
|
|
62
|
+
const today = new Date().toISOString().slice(0, 10);
|
|
59
63
|
try {
|
|
60
64
|
const memoryEntries = await fs.readdir(memoryDir);
|
|
61
65
|
for (const entry of memoryEntries) {
|
|
62
66
|
if (!entry.endsWith(".md"))
|
|
63
67
|
continue;
|
|
64
68
|
const stem = path.basename(entry, ".md");
|
|
65
|
-
// Skip daily
|
|
66
|
-
if (
|
|
69
|
+
// Skip today's daily log — it's still being written and already in prompt context
|
|
70
|
+
if (stem === today)
|
|
67
71
|
continue;
|
|
68
72
|
files.push({
|
|
69
73
|
uri: `${teammate}/memory/${entry}`,
|
|
@@ -194,6 +198,23 @@ export class Indexer {
|
|
|
194
198
|
}
|
|
195
199
|
return count;
|
|
196
200
|
}
|
|
201
|
+
/**
|
|
202
|
+
* Delete a document from a teammate's index by URI.
|
|
203
|
+
* Used to purge stale daily logs after they age out on disk.
|
|
204
|
+
*/
|
|
205
|
+
async deleteDocument(teammate, uri) {
|
|
206
|
+
const indexPath = this.indexPath(teammate);
|
|
207
|
+
const index = new LocalDocumentIndex({
|
|
208
|
+
folderPath: indexPath,
|
|
209
|
+
embeddings: this._embeddings,
|
|
210
|
+
});
|
|
211
|
+
if (!(await index.isIndexCreated()))
|
|
212
|
+
return;
|
|
213
|
+
const docId = await index.getDocumentId(uri);
|
|
214
|
+
if (docId) {
|
|
215
|
+
await index.deleteDocument(uri);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
197
218
|
/**
|
|
198
219
|
* Sync indexes for all teammates.
|
|
199
220
|
*/
|
package/dist/indexer.test.js
CHANGED
|
@@ -83,17 +83,20 @@ describe("Indexer", () => {
|
|
|
83
83
|
expect(uris).toContain("beacon/memory/feedback_testing.md");
|
|
84
84
|
expect(uris).toContain("beacon/memory/project_goals.md");
|
|
85
85
|
});
|
|
86
|
-
it("
|
|
86
|
+
it("includes older daily logs but skips today's", async () => {
|
|
87
87
|
const memDir = join(testDir, "beacon", "memory");
|
|
88
88
|
await mkdir(memDir, { recursive: true });
|
|
89
|
+
const today = new Date().toISOString().slice(0, 10);
|
|
90
|
+
await writeFile(join(memDir, `${today}.md`), "# Today");
|
|
89
91
|
await writeFile(join(memDir, "2026-03-14.md"), "# Day 1");
|
|
90
92
|
await writeFile(join(memDir, "2026-03-15.md"), "# Day 2");
|
|
91
93
|
await writeFile(join(memDir, "feedback_testing.md"), "# Feedback");
|
|
92
94
|
const indexer = createIndexer(testDir);
|
|
93
95
|
const { files } = await indexer.collectFiles("beacon");
|
|
94
96
|
const uris = files.map((f) => f.uri);
|
|
95
|
-
expect(uris).not.toContain(
|
|
96
|
-
expect(uris).
|
|
97
|
+
expect(uris).not.toContain(`beacon/memory/${today}.md`);
|
|
98
|
+
expect(uris).toContain("beacon/memory/2026-03-14.md");
|
|
99
|
+
expect(uris).toContain("beacon/memory/2026-03-15.md");
|
|
97
100
|
expect(uris).toContain("beacon/memory/feedback_testing.md");
|
|
98
101
|
});
|
|
99
102
|
it("collects weekly summaries from memory/weekly/", async () => {
|
|
@@ -181,6 +184,65 @@ describe("Indexer", () => {
|
|
|
181
184
|
expect(results.get("scribe")).toBe(0); // no indexable files
|
|
182
185
|
});
|
|
183
186
|
});
|
|
187
|
+
describe("upsertFile", () => {
|
|
188
|
+
it("upserts a single file into a new index", async () => {
|
|
189
|
+
const beacon = join(testDir, "beacon");
|
|
190
|
+
await mkdir(beacon, { recursive: true });
|
|
191
|
+
const filePath = join(beacon, "WISDOM.md");
|
|
192
|
+
await writeFile(filePath, "# Upsert test wisdom");
|
|
193
|
+
const indexer = createIndexer(testDir);
|
|
194
|
+
await indexer.upsertFile("beacon", filePath);
|
|
195
|
+
// Verify index was created by syncing (which reads the index)
|
|
196
|
+
const count = await indexer.syncTeammate("beacon");
|
|
197
|
+
expect(count).toBeGreaterThanOrEqual(1);
|
|
198
|
+
});
|
|
199
|
+
it("upserts into an existing index without rebuilding", async () => {
|
|
200
|
+
const beacon = join(testDir, "beacon");
|
|
201
|
+
const memDir = join(beacon, "memory");
|
|
202
|
+
await mkdir(memDir, { recursive: true });
|
|
203
|
+
await writeFile(join(beacon, "WISDOM.md"), "# Wisdom");
|
|
204
|
+
const indexer = createIndexer(testDir);
|
|
205
|
+
// Build initial index
|
|
206
|
+
await indexer.indexTeammate("beacon");
|
|
207
|
+
// Upsert a new file
|
|
208
|
+
const newFile = join(memDir, "feedback_test.md");
|
|
209
|
+
await writeFile(newFile, "# New feedback content");
|
|
210
|
+
await indexer.upsertFile("beacon", newFile);
|
|
211
|
+
// Sync should see both files
|
|
212
|
+
const count = await indexer.syncTeammate("beacon");
|
|
213
|
+
expect(count).toBe(2);
|
|
214
|
+
});
|
|
215
|
+
it("skips empty files", async () => {
|
|
216
|
+
const beacon = join(testDir, "beacon");
|
|
217
|
+
await mkdir(beacon, { recursive: true });
|
|
218
|
+
const filePath = join(beacon, "WISDOM.md");
|
|
219
|
+
await writeFile(filePath, " "); // whitespace only
|
|
220
|
+
const indexer = createIndexer(testDir);
|
|
221
|
+
// Should not throw, just skip
|
|
222
|
+
await indexer.upsertFile("beacon", filePath);
|
|
223
|
+
});
|
|
224
|
+
});
|
|
225
|
+
describe("syncAll", () => {
|
|
226
|
+
it("syncs all discovered teammates", async () => {
|
|
227
|
+
const beacon = join(testDir, "beacon");
|
|
228
|
+
const scribe = join(testDir, "scribe");
|
|
229
|
+
await mkdir(beacon, { recursive: true });
|
|
230
|
+
await mkdir(scribe, { recursive: true });
|
|
231
|
+
await writeFile(join(beacon, "SOUL.md"), "# Beacon");
|
|
232
|
+
await writeFile(join(beacon, "WISDOM.md"), "# Beacon wisdom");
|
|
233
|
+
await writeFile(join(scribe, "SOUL.md"), "# Scribe");
|
|
234
|
+
await writeFile(join(scribe, "WISDOM.md"), "# Scribe wisdom");
|
|
235
|
+
const indexer = createIndexer(testDir);
|
|
236
|
+
const results = await indexer.syncAll();
|
|
237
|
+
expect(results.get("beacon")).toBe(1);
|
|
238
|
+
expect(results.get("scribe")).toBe(1);
|
|
239
|
+
});
|
|
240
|
+
it("returns empty map when no teammates exist", async () => {
|
|
241
|
+
const indexer = createIndexer(testDir);
|
|
242
|
+
const results = await indexer.syncAll();
|
|
243
|
+
expect(results.size).toBe(0);
|
|
244
|
+
});
|
|
245
|
+
});
|
|
184
246
|
describe("syncTeammate", () => {
|
|
185
247
|
it("falls back to full index when no index exists", async () => {
|
|
186
248
|
const beacon = join(testDir, "beacon");
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Memory frontmatter scanning for Pass 1 recall queries.
|
|
3
|
+
*
|
|
4
|
+
* Reads the teammate's memory file catalog (name + description from frontmatter)
|
|
5
|
+
* and does fast text matching against the task prompt. This is a lightweight,
|
|
6
|
+
* no-embedding relevance signal — "here's a menu of what I might know about."
|
|
7
|
+
*/
|
|
8
|
+
import type { SearchResult } from "./search.js";
|
|
9
|
+
interface MemoryEntry {
|
|
10
|
+
/** Relative URI (e.g. "beacon/memory/project_goals.md") */
|
|
11
|
+
uri: string;
|
|
12
|
+
/** Absolute file path */
|
|
13
|
+
absolutePath: string;
|
|
14
|
+
/** Frontmatter name field */
|
|
15
|
+
name: string;
|
|
16
|
+
/** Frontmatter description field */
|
|
17
|
+
description: string;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Scan a teammate's memory directory and build a catalog of memory entries
|
|
21
|
+
* with their frontmatter metadata.
|
|
22
|
+
*/
|
|
23
|
+
export declare function scanMemoryCatalog(teammatesDir: string, teammate: string): Promise<MemoryEntry[]>;
|
|
24
|
+
/**
|
|
25
|
+
* Match task prompt text against memory catalog entries.
|
|
26
|
+
* Returns memory files whose name or description has significant word overlap
|
|
27
|
+
* with the task prompt. Each match is returned as a SearchResult with the
|
|
28
|
+
* file's full content.
|
|
29
|
+
*
|
|
30
|
+
* Matching is case-insensitive. A match requires at least one word from the
|
|
31
|
+
* task prompt appearing in the name or description.
|
|
32
|
+
*/
|
|
33
|
+
export declare function matchMemoryCatalog(teammatesDir: string, teammate: string, taskPrompt: string, maxTokens?: number): Promise<SearchResult[]>;
|
|
34
|
+
export {};
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Memory frontmatter scanning for Pass 1 recall queries.
|
|
3
|
+
*
|
|
4
|
+
* Reads the teammate's memory file catalog (name + description from frontmatter)
|
|
5
|
+
* and does fast text matching against the task prompt. This is a lightweight,
|
|
6
|
+
* no-embedding relevance signal — "here's a menu of what I might know about."
|
|
7
|
+
*/
|
|
8
|
+
import * as fs from "node:fs/promises";
|
|
9
|
+
import * as path from "node:path";
|
|
10
|
+
/**
|
|
11
|
+
* Parse YAML-ish frontmatter from a markdown file's content.
|
|
12
|
+
* Returns name and description fields, or null if no frontmatter found.
|
|
13
|
+
*/
|
|
14
|
+
function parseFrontmatter(content) {
|
|
15
|
+
const match = content.match(/^---\s*\n([\s\S]*?)\n---/);
|
|
16
|
+
if (!match)
|
|
17
|
+
return null;
|
|
18
|
+
const fm = match[1];
|
|
19
|
+
const nameMatch = fm.match(/^name:\s*(.+)$/m);
|
|
20
|
+
const descMatch = fm.match(/^description:\s*(.+)$/m);
|
|
21
|
+
if (!nameMatch)
|
|
22
|
+
return null;
|
|
23
|
+
return {
|
|
24
|
+
name: nameMatch[1].trim(),
|
|
25
|
+
description: descMatch?.[1]?.trim() ?? "",
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Scan a teammate's memory directory and build a catalog of memory entries
|
|
30
|
+
* with their frontmatter metadata.
|
|
31
|
+
*/
|
|
32
|
+
export async function scanMemoryCatalog(teammatesDir, teammate) {
|
|
33
|
+
const memoryDir = path.join(teammatesDir, teammate, "memory");
|
|
34
|
+
const entries = [];
|
|
35
|
+
try {
|
|
36
|
+
const files = await fs.readdir(memoryDir);
|
|
37
|
+
for (const file of files) {
|
|
38
|
+
if (!file.endsWith(".md"))
|
|
39
|
+
continue;
|
|
40
|
+
// Skip daily logs (YYYY-MM-DD.md)
|
|
41
|
+
const stem = path.basename(file, ".md");
|
|
42
|
+
if (/^\d{4}-\d{2}-\d{2}$/.test(stem))
|
|
43
|
+
continue;
|
|
44
|
+
const absolutePath = path.join(memoryDir, file);
|
|
45
|
+
const content = await fs.readFile(absolutePath, "utf-8");
|
|
46
|
+
const fm = parseFrontmatter(content);
|
|
47
|
+
if (!fm)
|
|
48
|
+
continue;
|
|
49
|
+
entries.push({
|
|
50
|
+
uri: `${teammate}/memory/${file}`,
|
|
51
|
+
absolutePath,
|
|
52
|
+
name: fm.name,
|
|
53
|
+
description: fm.description,
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
// No memory/ directory
|
|
59
|
+
}
|
|
60
|
+
return entries;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Match task prompt text against memory catalog entries.
|
|
64
|
+
* Returns memory files whose name or description has significant word overlap
|
|
65
|
+
* with the task prompt. Each match is returned as a SearchResult with the
|
|
66
|
+
* file's full content.
|
|
67
|
+
*
|
|
68
|
+
* Matching is case-insensitive. A match requires at least one word from the
|
|
69
|
+
* task prompt appearing in the name or description.
|
|
70
|
+
*/
|
|
71
|
+
export async function matchMemoryCatalog(teammatesDir, teammate, taskPrompt, maxTokens = 500) {
|
|
72
|
+
const catalog = await scanMemoryCatalog(teammatesDir, teammate);
|
|
73
|
+
if (catalog.length === 0)
|
|
74
|
+
return [];
|
|
75
|
+
// Tokenize the task prompt into lowercase words (3+ chars)
|
|
76
|
+
const promptWords = new Set(taskPrompt
|
|
77
|
+
.toLowerCase()
|
|
78
|
+
.replace(/[^\w\s@/-]/g, " ")
|
|
79
|
+
.split(/\s+/)
|
|
80
|
+
.filter((w) => w.length > 2));
|
|
81
|
+
const results = [];
|
|
82
|
+
for (const entry of catalog) {
|
|
83
|
+
const catalogText = `${entry.name} ${entry.description}`.toLowerCase();
|
|
84
|
+
const catalogWords = catalogText
|
|
85
|
+
.replace(/[^\w\s@/_-]/g, " ")
|
|
86
|
+
.split(/\s+/)
|
|
87
|
+
.filter((w) => w.length > 2);
|
|
88
|
+
// Count overlapping words
|
|
89
|
+
let overlap = 0;
|
|
90
|
+
for (const w of catalogWords) {
|
|
91
|
+
if (promptWords.has(w))
|
|
92
|
+
overlap++;
|
|
93
|
+
}
|
|
94
|
+
// Also check if prompt words appear as substrings in the catalog text
|
|
95
|
+
// (e.g., "goal" matches "project_goals")
|
|
96
|
+
for (const pw of promptWords) {
|
|
97
|
+
if (catalogText.includes(pw) && !catalogWords.includes(pw)) {
|
|
98
|
+
overlap += 0.5;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
if (overlap >= 1) {
|
|
102
|
+
// Read full file content for matched entries
|
|
103
|
+
const content = await fs.readFile(entry.absolutePath, "utf-8");
|
|
104
|
+
// Strip frontmatter from the content
|
|
105
|
+
const body = content.replace(/^---\s*\n[\s\S]*?\n---\s*\n?/, "").trim();
|
|
106
|
+
results.push({
|
|
107
|
+
teammate,
|
|
108
|
+
uri: entry.uri,
|
|
109
|
+
text: body.slice(0, maxTokens * 4), // rough token limit
|
|
110
|
+
score: 0.85 + Math.min(overlap * 0.02, 0.1), // 0.85-0.95 range
|
|
111
|
+
contentType: "typed_memory",
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
// Sort by score descending
|
|
116
|
+
results.sort((a, b) => b.score - a.score);
|
|
117
|
+
return results;
|
|
118
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import * as fs from "node:fs/promises";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
4
|
+
import { matchMemoryCatalog, scanMemoryCatalog } from "./memory-index.js";
|
|
5
|
+
const TEST_DIR = path.join(process.cwd(), ".test-memory-index");
|
|
6
|
+
const TEAMMATE = "testmate";
|
|
7
|
+
beforeAll(async () => {
|
|
8
|
+
const memoryDir = path.join(TEST_DIR, TEAMMATE, "memory");
|
|
9
|
+
await fs.mkdir(memoryDir, { recursive: true });
|
|
10
|
+
// Create typed memory files with frontmatter
|
|
11
|
+
await fs.writeFile(path.join(memoryDir, "project_goals.md"), `---
|
|
12
|
+
name: project_goals
|
|
13
|
+
description: Stack-ranked feature goals for the teammates project
|
|
14
|
+
type: project
|
|
15
|
+
---
|
|
16
|
+
|
|
17
|
+
## Goals
|
|
18
|
+
|
|
19
|
+
1. Recall query architecture
|
|
20
|
+
2. CLI improvements
|
|
21
|
+
`);
|
|
22
|
+
await fs.writeFile(path.join(memoryDir, "feedback_testing.md"), `---
|
|
23
|
+
name: feedback_testing
|
|
24
|
+
description: Integration tests must hit a real database, not mocks
|
|
25
|
+
type: feedback
|
|
26
|
+
---
|
|
27
|
+
|
|
28
|
+
Use real databases in tests. Mocks hide migration bugs.
|
|
29
|
+
`);
|
|
30
|
+
// Create a file without frontmatter (should be skipped)
|
|
31
|
+
await fs.writeFile(path.join(memoryDir, "notes.md"), "Just some notes without frontmatter.\n");
|
|
32
|
+
// Create a daily log (should be skipped)
|
|
33
|
+
await fs.writeFile(path.join(memoryDir, "2026-03-21.md"), "# 2026-03-21\nDaily log content.\n");
|
|
34
|
+
});
|
|
35
|
+
afterAll(async () => {
|
|
36
|
+
await fs.rm(TEST_DIR, { recursive: true, force: true });
|
|
37
|
+
});
|
|
38
|
+
describe("scanMemoryCatalog", () => {
|
|
39
|
+
it("returns entries with frontmatter", async () => {
|
|
40
|
+
const entries = await scanMemoryCatalog(TEST_DIR, TEAMMATE);
|
|
41
|
+
expect(entries.length).toBe(2);
|
|
42
|
+
const names = entries.map((e) => e.name);
|
|
43
|
+
expect(names).toContain("project_goals");
|
|
44
|
+
expect(names).toContain("feedback_testing");
|
|
45
|
+
});
|
|
46
|
+
it("skips files without frontmatter", async () => {
|
|
47
|
+
const entries = await scanMemoryCatalog(TEST_DIR, TEAMMATE);
|
|
48
|
+
const uris = entries.map((e) => e.uri);
|
|
49
|
+
expect(uris).not.toContain(`${TEAMMATE}/memory/notes.md`);
|
|
50
|
+
});
|
|
51
|
+
it("skips daily logs", async () => {
|
|
52
|
+
const entries = await scanMemoryCatalog(TEST_DIR, TEAMMATE);
|
|
53
|
+
const uris = entries.map((e) => e.uri);
|
|
54
|
+
expect(uris).not.toContain(`${TEAMMATE}/memory/2026-03-21.md`);
|
|
55
|
+
});
|
|
56
|
+
it("returns empty array for nonexistent teammate", async () => {
|
|
57
|
+
const entries = await scanMemoryCatalog(TEST_DIR, "nobody");
|
|
58
|
+
expect(entries).toEqual([]);
|
|
59
|
+
});
|
|
60
|
+
});
|
|
61
|
+
describe("matchMemoryCatalog", () => {
|
|
62
|
+
it("matches files whose frontmatter overlaps with the query", async () => {
|
|
63
|
+
const results = await matchMemoryCatalog(TEST_DIR, TEAMMATE, "what are our project goals and features?");
|
|
64
|
+
expect(results.length).toBeGreaterThanOrEqual(1);
|
|
65
|
+
expect(results[0].uri).toContain("project_goals");
|
|
66
|
+
});
|
|
67
|
+
it("matches on description keywords", async () => {
|
|
68
|
+
const results = await matchMemoryCatalog(TEST_DIR, TEAMMATE, "database testing integration");
|
|
69
|
+
expect(results.length).toBeGreaterThanOrEqual(1);
|
|
70
|
+
const uris = results.map((r) => r.uri);
|
|
71
|
+
expect(uris).toContain(`${TEAMMATE}/memory/feedback_testing.md`);
|
|
72
|
+
});
|
|
73
|
+
it("returns empty for unrelated queries", async () => {
|
|
74
|
+
const results = await matchMemoryCatalog(TEST_DIR, TEAMMATE, "quantum physics dark matter");
|
|
75
|
+
expect(results.length).toBe(0);
|
|
76
|
+
});
|
|
77
|
+
it("strips frontmatter from result text", async () => {
|
|
78
|
+
const results = await matchMemoryCatalog(TEST_DIR, TEAMMATE, "project goals features");
|
|
79
|
+
expect(results.length).toBeGreaterThanOrEqual(1);
|
|
80
|
+
expect(results[0].text).not.toContain("---");
|
|
81
|
+
expect(results[0].text).toContain("Goals");
|
|
82
|
+
});
|
|
83
|
+
it("assigns scores in the 0.85-0.95 range", async () => {
|
|
84
|
+
const results = await matchMemoryCatalog(TEST_DIR, TEAMMATE, "project goals features teammates");
|
|
85
|
+
for (const r of results) {
|
|
86
|
+
expect(r.score).toBeGreaterThanOrEqual(0.85);
|
|
87
|
+
expect(r.score).toBeLessThanOrEqual(0.95);
|
|
88
|
+
}
|
|
89
|
+
});
|
|
90
|
+
it("sets contentType to typed_memory", async () => {
|
|
91
|
+
const results = await matchMemoryCatalog(TEST_DIR, TEAMMATE, "project goals");
|
|
92
|
+
for (const r of results) {
|
|
93
|
+
expect(r.contentType).toBe("typed_memory");
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
});
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lightweight query expansion for Pass 1 recall queries.
|
|
3
|
+
*
|
|
4
|
+
* No LLM needed — uses stopword removal and basic text analysis
|
|
5
|
+
* to generate multiple query variations from a task prompt.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Extract meaningful keywords from text by removing stopwords and short tokens.
|
|
9
|
+
* Returns lowercase keywords in order of appearance.
|
|
10
|
+
*/
|
|
11
|
+
export declare function extractKeywords(text: string): string[];
|
|
12
|
+
/**
|
|
13
|
+
* Build multiple query variations from a task prompt and optional conversation context.
|
|
14
|
+
*
|
|
15
|
+
* Returns 1-3 queries:
|
|
16
|
+
* 1. The original task prompt (always)
|
|
17
|
+
* 2. A focused keyword query (if keywords differ meaningfully from the original)
|
|
18
|
+
* 3. A conversation-derived query (if recent conversation context is provided)
|
|
19
|
+
*/
|
|
20
|
+
export declare function buildQueryVariations(taskPrompt: string, conversationContext?: string): string[];
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lightweight query expansion for Pass 1 recall queries.
|
|
3
|
+
*
|
|
4
|
+
* No LLM needed — uses stopword removal and basic text analysis
|
|
5
|
+
* to generate multiple query variations from a task prompt.
|
|
6
|
+
*/
|
|
7
|
+
/** Common English stopwords to filter from queries. */
|
|
8
|
+
const STOPWORDS = new Set([
|
|
9
|
+
"a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
|
|
10
|
+
"of", "with", "by", "from", "is", "are", "was", "were", "be", "been",
|
|
11
|
+
"being", "have", "has", "had", "do", "does", "did", "will", "would",
|
|
12
|
+
"could", "should", "may", "might", "shall", "can", "need", "must",
|
|
13
|
+
"it", "its", "this", "that", "these", "those", "i", "you", "he", "she",
|
|
14
|
+
"we", "they", "me", "him", "her", "us", "them", "my", "your", "his",
|
|
15
|
+
"our", "their", "what", "which", "who", "whom", "where", "when", "how",
|
|
16
|
+
"why", "if", "then", "so", "not", "no", "just", "also", "very", "too",
|
|
17
|
+
"some", "any", "all", "each", "every", "both", "few", "more", "most",
|
|
18
|
+
"other", "into", "over", "after", "before", "between", "through",
|
|
19
|
+
"about", "up", "out", "off", "down", "here", "there", "again", "once",
|
|
20
|
+
"let", "lets", "let's", "get", "got", "go", "going", "make", "made",
|
|
21
|
+
"take", "took", "come", "came", "see", "saw", "know", "knew", "think",
|
|
22
|
+
"thought", "say", "said", "tell", "told", "ask", "asked", "want",
|
|
23
|
+
"wanted", "like", "look", "use", "used", "find", "give", "work",
|
|
24
|
+
]);
|
|
25
|
+
/**
|
|
26
|
+
* Extract meaningful keywords from text by removing stopwords and short tokens.
|
|
27
|
+
* Returns lowercase keywords in order of appearance.
|
|
28
|
+
*/
|
|
29
|
+
export function extractKeywords(text) {
|
|
30
|
+
const words = text
|
|
31
|
+
.toLowerCase()
|
|
32
|
+
.replace(/[^\w\s@/-]/g, " ")
|
|
33
|
+
.split(/\s+/)
|
|
34
|
+
.filter((w) => w.length > 2 && !STOPWORDS.has(w));
|
|
35
|
+
// Deduplicate while preserving order
|
|
36
|
+
const seen = new Set();
|
|
37
|
+
const result = [];
|
|
38
|
+
for (const w of words) {
|
|
39
|
+
if (!seen.has(w)) {
|
|
40
|
+
seen.add(w);
|
|
41
|
+
result.push(w);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return result;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Build multiple query variations from a task prompt and optional conversation context.
|
|
48
|
+
*
|
|
49
|
+
* Returns 1-3 queries:
|
|
50
|
+
* 1. The original task prompt (always)
|
|
51
|
+
* 2. A focused keyword query (if keywords differ meaningfully from the original)
|
|
52
|
+
* 3. A conversation-derived query (if recent conversation context is provided)
|
|
53
|
+
*/
|
|
54
|
+
export function buildQueryVariations(taskPrompt, conversationContext) {
|
|
55
|
+
const queries = [taskPrompt];
|
|
56
|
+
// Query 2: Focused keywords from the task prompt
|
|
57
|
+
const keywords = extractKeywords(taskPrompt);
|
|
58
|
+
if (keywords.length >= 2 && keywords.length <= 20) {
|
|
59
|
+
const keywordQuery = keywords.slice(0, 8).join(" ");
|
|
60
|
+
// Only add if meaningfully different from original
|
|
61
|
+
if (keywordQuery.length < taskPrompt.length * 0.7) {
|
|
62
|
+
queries.push(keywordQuery);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
// Query 3: Recent conversation topic
|
|
66
|
+
if (conversationContext) {
|
|
67
|
+
const recentTopic = extractRecentTopic(conversationContext);
|
|
68
|
+
if (recentTopic) {
|
|
69
|
+
queries.push(recentTopic);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return queries;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Extract the most recent topic/theme from conversation context.
|
|
76
|
+
* Takes the last 1-2 meaningful entries and extracts keywords.
|
|
77
|
+
*/
|
|
78
|
+
function extractRecentTopic(conversationContext) {
|
|
79
|
+
// Split on common conversation entry patterns
|
|
80
|
+
const entries = conversationContext
|
|
81
|
+
.split(/\n\*\*\w+:\*\*\s*/g)
|
|
82
|
+
.filter((e) => e.trim().length > 10);
|
|
83
|
+
if (entries.length === 0)
|
|
84
|
+
return null;
|
|
85
|
+
// Take the last 1-2 entries (most recent conversation)
|
|
86
|
+
const recent = entries.slice(-2).join(" ");
|
|
87
|
+
const keywords = extractKeywords(recent);
|
|
88
|
+
if (keywords.length < 2)
|
|
89
|
+
return null;
|
|
90
|
+
// Build a focused query from the recent conversation keywords
|
|
91
|
+
return keywords.slice(0, 6).join(" ");
|
|
92
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|