@teammates/recall 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +23 -23
- package/package.json +40 -40
- package/src/cli.test.ts +324 -324
- package/src/cli.ts +407 -407
- package/src/embeddings.ts +56 -56
- package/src/index.ts +3 -3
- package/src/indexer.test.ts +262 -262
- package/src/indexer.ts +237 -237
- package/src/search.test.ts +49 -49
- package/src/search.ts +178 -178
- package/tsconfig.json +18 -18
- package/vitest.config.ts +12 -12
package/src/search.ts
CHANGED
|
@@ -1,178 +1,178 @@
|
|
|
1
|
-
import * as fs from "node:fs/promises";
|
|
2
|
-
import * as path from "node:path";
|
|
3
|
-
import { LocalDocumentIndex } from "vectra";
|
|
4
|
-
import { LocalEmbeddings } from "./embeddings.js";
|
|
5
|
-
import { Indexer } from "./indexer.js";
|
|
6
|
-
|
|
7
|
-
export interface SearchOptions {
|
|
8
|
-
/** Path to the .teammates directory */
|
|
9
|
-
teammatesDir: string;
|
|
10
|
-
/** Teammate name to search (searches all if omitted) */
|
|
11
|
-
teammate?: string;
|
|
12
|
-
/** Max results per teammate (default: 5) */
|
|
13
|
-
maxResults?: number;
|
|
14
|
-
/** Max chunks per document (default: 3) */
|
|
15
|
-
maxChunks?: number;
|
|
16
|
-
/** Max tokens per section (default: 500) */
|
|
17
|
-
maxTokens?: number;
|
|
18
|
-
/** Embedding model name */
|
|
19
|
-
model?: string;
|
|
20
|
-
/** Skip auto-sync before searching (default: false) */
|
|
21
|
-
skipSync?: boolean;
|
|
22
|
-
/** Number of recent weekly summaries to always include (default: 2) */
|
|
23
|
-
recencyDepth?: number;
|
|
24
|
-
/** Relevance boost multiplier for typed memories over episodic summaries (default: 1.2) */
|
|
25
|
-
typedMemoryBoost?: number;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
export interface SearchResult {
|
|
29
|
-
teammate: string;
|
|
30
|
-
uri: string;
|
|
31
|
-
text: string;
|
|
32
|
-
score: number;
|
|
33
|
-
/** Content type: "typed_memory", "weekly", "monthly", or "other" */
|
|
34
|
-
contentType?: string;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
/**
|
|
38
|
-
* Classify a URI into a content type for priority scoring.
|
|
39
|
-
*/
|
|
40
|
-
function classifyUri(uri: string): string {
|
|
41
|
-
if (uri.includes("/memory/weekly/")) return "weekly";
|
|
42
|
-
if (uri.includes("/memory/monthly/")) return "monthly";
|
|
43
|
-
// Typed memories are in memory/ but not daily logs (YYYY-MM-DD) and not in subdirs
|
|
44
|
-
const memoryMatch = uri.match(/\/memory\/([^/]+)\.md$/);
|
|
45
|
-
if (memoryMatch) {
|
|
46
|
-
const stem = memoryMatch[1];
|
|
47
|
-
if (/^\d{4}-\d{2}-\d{2}$/.test(stem)) return "daily";
|
|
48
|
-
return "typed_memory";
|
|
49
|
-
}
|
|
50
|
-
return "other";
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
/**
|
|
54
|
-
* Search teammate memories using multi-pass retrieval.
|
|
55
|
-
*
|
|
56
|
-
* Pass 1 (Recency): Always returns the N most recent weekly summaries.
|
|
57
|
-
* Pass 2 (Semantic): Query-driven search across all indexed content.
|
|
58
|
-
* Results are merged, deduped, and typed memories get a relevance boost.
|
|
59
|
-
*/
|
|
60
|
-
export async function search(
|
|
61
|
-
query: string,
|
|
62
|
-
options: SearchOptions,
|
|
63
|
-
): Promise<SearchResult[]> {
|
|
64
|
-
const embeddings = new LocalEmbeddings(options.model);
|
|
65
|
-
const indexer = new Indexer({
|
|
66
|
-
teammatesDir: options.teammatesDir,
|
|
67
|
-
model: options.model,
|
|
68
|
-
});
|
|
69
|
-
const maxResults = options.maxResults ?? 5;
|
|
70
|
-
const maxChunks = options.maxChunks ?? 3;
|
|
71
|
-
const maxTokens = options.maxTokens ?? 500;
|
|
72
|
-
const recencyDepth = options.recencyDepth ?? 2;
|
|
73
|
-
const typedMemoryBoost = options.typedMemoryBoost ?? 1.2;
|
|
74
|
-
|
|
75
|
-
// Auto-sync: upsert any new/changed files before searching
|
|
76
|
-
if (!options.skipSync) {
|
|
77
|
-
if (options.teammate) {
|
|
78
|
-
await indexer.syncTeammate(options.teammate);
|
|
79
|
-
} else {
|
|
80
|
-
await indexer.syncAll();
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
// Determine which teammates to search
|
|
85
|
-
let teammates: string[];
|
|
86
|
-
if (options.teammate) {
|
|
87
|
-
teammates = [options.teammate];
|
|
88
|
-
} else {
|
|
89
|
-
teammates = await indexer.discoverTeammates();
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
const allResults: SearchResult[] = [];
|
|
93
|
-
const seenUris = new Set<string>();
|
|
94
|
-
|
|
95
|
-
// ── Pass 1: Recency (recent weekly summaries, always included) ───
|
|
96
|
-
for (const teammate of teammates) {
|
|
97
|
-
const weeklyDir = path.join(
|
|
98
|
-
options.teammatesDir,
|
|
99
|
-
teammate,
|
|
100
|
-
"memory",
|
|
101
|
-
"weekly",
|
|
102
|
-
);
|
|
103
|
-
try {
|
|
104
|
-
const entries = await fs.readdir(weeklyDir);
|
|
105
|
-
const weeklyFiles = entries
|
|
106
|
-
.filter((e) => e.endsWith(".md"))
|
|
107
|
-
.sort()
|
|
108
|
-
.reverse()
|
|
109
|
-
.slice(0, recencyDepth);
|
|
110
|
-
|
|
111
|
-
for (const file of weeklyFiles) {
|
|
112
|
-
const uri = `${teammate}/memory/weekly/${file}`;
|
|
113
|
-
const text = await fs.readFile(path.join(weeklyDir, file), "utf-8");
|
|
114
|
-
if (text.trim().length === 0) continue;
|
|
115
|
-
seenUris.add(uri);
|
|
116
|
-
allResults.push({
|
|
117
|
-
teammate,
|
|
118
|
-
uri,
|
|
119
|
-
text: text.slice(0, maxTokens * 4), // rough token estimate
|
|
120
|
-
score: 0.9, // high base score for recency results
|
|
121
|
-
contentType: "weekly",
|
|
122
|
-
});
|
|
123
|
-
}
|
|
124
|
-
} catch {
|
|
125
|
-
// No weekly/ directory for this teammate
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
// ── Pass 2: Semantic (query-driven across all indexed content) ───
|
|
130
|
-
for (const teammate of teammates) {
|
|
131
|
-
const indexPath = indexer.indexPath(teammate);
|
|
132
|
-
try {
|
|
133
|
-
await fs.access(indexPath);
|
|
134
|
-
} catch {
|
|
135
|
-
continue;
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
const index = new LocalDocumentIndex({
|
|
139
|
-
folderPath: indexPath,
|
|
140
|
-
embeddings,
|
|
141
|
-
});
|
|
142
|
-
|
|
143
|
-
if (!(await index.isIndexCreated())) continue;
|
|
144
|
-
|
|
145
|
-
const docs = await index.queryDocuments(query, {
|
|
146
|
-
maxDocuments: maxResults,
|
|
147
|
-
maxChunks,
|
|
148
|
-
});
|
|
149
|
-
|
|
150
|
-
for (const doc of docs) {
|
|
151
|
-
if (seenUris.has(doc.uri)) continue; // dedup with recency pass
|
|
152
|
-
seenUris.add(doc.uri);
|
|
153
|
-
|
|
154
|
-
const sections = await doc.renderSections(maxTokens, 1);
|
|
155
|
-
const contentType = classifyUri(doc.uri);
|
|
156
|
-
|
|
157
|
-
for (const section of sections) {
|
|
158
|
-
let score = section.score;
|
|
159
|
-
// Apply type-based priority boost for typed memories
|
|
160
|
-
if (contentType === "typed_memory") {
|
|
161
|
-
score *= typedMemoryBoost;
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
allResults.push({
|
|
165
|
-
teammate,
|
|
166
|
-
uri: doc.uri,
|
|
167
|
-
text: section.text,
|
|
168
|
-
score,
|
|
169
|
-
contentType,
|
|
170
|
-
});
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
// Sort by score descending, return top results
|
|
176
|
-
allResults.sort((a, b) => b.score - a.score);
|
|
177
|
-
return allResults.slice(0, maxResults + recencyDepth); // allow extra slots for recency results
|
|
178
|
-
}
|
|
1
|
+
import * as fs from "node:fs/promises";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import { LocalDocumentIndex } from "vectra";
|
|
4
|
+
import { LocalEmbeddings } from "./embeddings.js";
|
|
5
|
+
import { Indexer } from "./indexer.js";
|
|
6
|
+
|
|
7
|
+
export interface SearchOptions {
|
|
8
|
+
/** Path to the .teammates directory */
|
|
9
|
+
teammatesDir: string;
|
|
10
|
+
/** Teammate name to search (searches all if omitted) */
|
|
11
|
+
teammate?: string;
|
|
12
|
+
/** Max results per teammate (default: 5) */
|
|
13
|
+
maxResults?: number;
|
|
14
|
+
/** Max chunks per document (default: 3) */
|
|
15
|
+
maxChunks?: number;
|
|
16
|
+
/** Max tokens per section (default: 500) */
|
|
17
|
+
maxTokens?: number;
|
|
18
|
+
/** Embedding model name */
|
|
19
|
+
model?: string;
|
|
20
|
+
/** Skip auto-sync before searching (default: false) */
|
|
21
|
+
skipSync?: boolean;
|
|
22
|
+
/** Number of recent weekly summaries to always include (default: 2) */
|
|
23
|
+
recencyDepth?: number;
|
|
24
|
+
/** Relevance boost multiplier for typed memories over episodic summaries (default: 1.2) */
|
|
25
|
+
typedMemoryBoost?: number;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface SearchResult {
|
|
29
|
+
teammate: string;
|
|
30
|
+
uri: string;
|
|
31
|
+
text: string;
|
|
32
|
+
score: number;
|
|
33
|
+
/** Content type: "typed_memory", "weekly", "monthly", or "other" */
|
|
34
|
+
contentType?: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Classify a URI into a content type for priority scoring.
|
|
39
|
+
*/
|
|
40
|
+
function classifyUri(uri: string): string {
|
|
41
|
+
if (uri.includes("/memory/weekly/")) return "weekly";
|
|
42
|
+
if (uri.includes("/memory/monthly/")) return "monthly";
|
|
43
|
+
// Typed memories are in memory/ but not daily logs (YYYY-MM-DD) and not in subdirs
|
|
44
|
+
const memoryMatch = uri.match(/\/memory\/([^/]+)\.md$/);
|
|
45
|
+
if (memoryMatch) {
|
|
46
|
+
const stem = memoryMatch[1];
|
|
47
|
+
if (/^\d{4}-\d{2}-\d{2}$/.test(stem)) return "daily";
|
|
48
|
+
return "typed_memory";
|
|
49
|
+
}
|
|
50
|
+
return "other";
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Search teammate memories using multi-pass retrieval.
|
|
55
|
+
*
|
|
56
|
+
* Pass 1 (Recency): Always returns the N most recent weekly summaries.
|
|
57
|
+
* Pass 2 (Semantic): Query-driven search across all indexed content.
|
|
58
|
+
* Results are merged, deduped, and typed memories get a relevance boost.
|
|
59
|
+
*/
|
|
60
|
+
export async function search(
|
|
61
|
+
query: string,
|
|
62
|
+
options: SearchOptions,
|
|
63
|
+
): Promise<SearchResult[]> {
|
|
64
|
+
const embeddings = new LocalEmbeddings(options.model);
|
|
65
|
+
const indexer = new Indexer({
|
|
66
|
+
teammatesDir: options.teammatesDir,
|
|
67
|
+
model: options.model,
|
|
68
|
+
});
|
|
69
|
+
const maxResults = options.maxResults ?? 5;
|
|
70
|
+
const maxChunks = options.maxChunks ?? 3;
|
|
71
|
+
const maxTokens = options.maxTokens ?? 500;
|
|
72
|
+
const recencyDepth = options.recencyDepth ?? 2;
|
|
73
|
+
const typedMemoryBoost = options.typedMemoryBoost ?? 1.2;
|
|
74
|
+
|
|
75
|
+
// Auto-sync: upsert any new/changed files before searching
|
|
76
|
+
if (!options.skipSync) {
|
|
77
|
+
if (options.teammate) {
|
|
78
|
+
await indexer.syncTeammate(options.teammate);
|
|
79
|
+
} else {
|
|
80
|
+
await indexer.syncAll();
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Determine which teammates to search
|
|
85
|
+
let teammates: string[];
|
|
86
|
+
if (options.teammate) {
|
|
87
|
+
teammates = [options.teammate];
|
|
88
|
+
} else {
|
|
89
|
+
teammates = await indexer.discoverTeammates();
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const allResults: SearchResult[] = [];
|
|
93
|
+
const seenUris = new Set<string>();
|
|
94
|
+
|
|
95
|
+
// ── Pass 1: Recency (recent weekly summaries, always included) ───
|
|
96
|
+
for (const teammate of teammates) {
|
|
97
|
+
const weeklyDir = path.join(
|
|
98
|
+
options.teammatesDir,
|
|
99
|
+
teammate,
|
|
100
|
+
"memory",
|
|
101
|
+
"weekly",
|
|
102
|
+
);
|
|
103
|
+
try {
|
|
104
|
+
const entries = await fs.readdir(weeklyDir);
|
|
105
|
+
const weeklyFiles = entries
|
|
106
|
+
.filter((e) => e.endsWith(".md"))
|
|
107
|
+
.sort()
|
|
108
|
+
.reverse()
|
|
109
|
+
.slice(0, recencyDepth);
|
|
110
|
+
|
|
111
|
+
for (const file of weeklyFiles) {
|
|
112
|
+
const uri = `${teammate}/memory/weekly/${file}`;
|
|
113
|
+
const text = await fs.readFile(path.join(weeklyDir, file), "utf-8");
|
|
114
|
+
if (text.trim().length === 0) continue;
|
|
115
|
+
seenUris.add(uri);
|
|
116
|
+
allResults.push({
|
|
117
|
+
teammate,
|
|
118
|
+
uri,
|
|
119
|
+
text: text.slice(0, maxTokens * 4), // rough token estimate
|
|
120
|
+
score: 0.9, // high base score for recency results
|
|
121
|
+
contentType: "weekly",
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
} catch {
|
|
125
|
+
// No weekly/ directory for this teammate
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// ── Pass 2: Semantic (query-driven across all indexed content) ───
|
|
130
|
+
for (const teammate of teammates) {
|
|
131
|
+
const indexPath = indexer.indexPath(teammate);
|
|
132
|
+
try {
|
|
133
|
+
await fs.access(indexPath);
|
|
134
|
+
} catch {
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const index = new LocalDocumentIndex({
|
|
139
|
+
folderPath: indexPath,
|
|
140
|
+
embeddings,
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
if (!(await index.isIndexCreated())) continue;
|
|
144
|
+
|
|
145
|
+
const docs = await index.queryDocuments(query, {
|
|
146
|
+
maxDocuments: maxResults,
|
|
147
|
+
maxChunks,
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
for (const doc of docs) {
|
|
151
|
+
if (seenUris.has(doc.uri)) continue; // dedup with recency pass
|
|
152
|
+
seenUris.add(doc.uri);
|
|
153
|
+
|
|
154
|
+
const sections = await doc.renderSections(maxTokens, 1);
|
|
155
|
+
const contentType = classifyUri(doc.uri);
|
|
156
|
+
|
|
157
|
+
for (const section of sections) {
|
|
158
|
+
let score = section.score;
|
|
159
|
+
// Apply type-based priority boost for typed memories
|
|
160
|
+
if (contentType === "typed_memory") {
|
|
161
|
+
score *= typedMemoryBoost;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
allResults.push({
|
|
165
|
+
teammate,
|
|
166
|
+
uri: doc.uri,
|
|
167
|
+
text: section.text,
|
|
168
|
+
score,
|
|
169
|
+
contentType,
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// Sort by score descending, return top results
|
|
176
|
+
allResults.sort((a, b) => b.score - a.score);
|
|
177
|
+
return allResults.slice(0, maxResults + recencyDepth); // allow extra slots for recency results
|
|
178
|
+
}
|
package/tsconfig.json
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
{
|
|
2
|
-
"compilerOptions": {
|
|
3
|
-
"target": "ES2022",
|
|
4
|
-
"module": "Node16",
|
|
5
|
-
"moduleResolution": "Node16",
|
|
6
|
-
"lib": ["ES2022"],
|
|
7
|
-
"outDir": "dist",
|
|
8
|
-
"rootDir": "src",
|
|
9
|
-
"strict": true,
|
|
10
|
-
"declaration": true,
|
|
11
|
-
"esModuleInterop": true,
|
|
12
|
-
"skipLibCheck": true,
|
|
13
|
-
"forceConsistentCasingInFileNames": true,
|
|
14
|
-
"resolveJsonModule": true
|
|
15
|
-
},
|
|
16
|
-
"include": ["src"],
|
|
17
|
-
"exclude": ["node_modules", "dist"]
|
|
18
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"compilerOptions": {
|
|
3
|
+
"target": "ES2022",
|
|
4
|
+
"module": "Node16",
|
|
5
|
+
"moduleResolution": "Node16",
|
|
6
|
+
"lib": ["ES2022"],
|
|
7
|
+
"outDir": "dist",
|
|
8
|
+
"rootDir": "src",
|
|
9
|
+
"strict": true,
|
|
10
|
+
"declaration": true,
|
|
11
|
+
"esModuleInterop": true,
|
|
12
|
+
"skipLibCheck": true,
|
|
13
|
+
"forceConsistentCasingInFileNames": true,
|
|
14
|
+
"resolveJsonModule": true
|
|
15
|
+
},
|
|
16
|
+
"include": ["src"],
|
|
17
|
+
"exclude": ["node_modules", "dist"]
|
|
18
|
+
}
|
package/vitest.config.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { defineConfig } from "vitest/config";
|
|
2
|
-
|
|
3
|
-
export default defineConfig({
|
|
4
|
-
test: {
|
|
5
|
-
include: ["src/**/*.test.ts"],
|
|
6
|
-
coverage: {
|
|
7
|
-
provider: "v8",
|
|
8
|
-
reporter: ["text", "json-summary"],
|
|
9
|
-
reportsDirectory: "coverage",
|
|
10
|
-
},
|
|
11
|
-
},
|
|
12
|
-
});
|
|
1
|
+
import { defineConfig } from "vitest/config";
|
|
2
|
+
|
|
3
|
+
export default defineConfig({
|
|
4
|
+
test: {
|
|
5
|
+
include: ["src/**/*.test.ts"],
|
|
6
|
+
coverage: {
|
|
7
|
+
provider: "v8",
|
|
8
|
+
reporter: ["text", "json-summary"],
|
|
9
|
+
reportsDirectory: "coverage",
|
|
10
|
+
},
|
|
11
|
+
},
|
|
12
|
+
});
|