@aeriondyseti/vector-memory-mcp 2.3.0 → 2.4.4-dev.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/server/core/connection.ts +1 -1
- package/server/core/conversation.repository.ts +113 -16
- package/server/core/conversation.service.ts +19 -19
- package/server/core/conversation.ts +7 -5
- package/server/core/embeddings.service.ts +108 -17
- package/server/core/memory.repository.ts +49 -14
- package/server/core/memory.service.ts +47 -42
- package/server/core/memory.ts +40 -1
- package/server/core/migration.service.ts +3 -3
- package/server/core/migrations.ts +60 -20
- package/server/core/parsers/claude-code.parser.ts +3 -3
- package/server/core/parsers/types.ts +1 -1
- package/server/core/sqlite-utils.ts +67 -2
- package/server/index.ts +13 -15
- package/server/transports/http/mcp-transport.ts +5 -5
- package/server/transports/http/server.ts +19 -6
- package/server/transports/mcp/handlers.ts +47 -23
- package/server/transports/mcp/server.ts +5 -5
- package/scripts/lancedb-extract.ts +0 -181
- package/scripts/smoke-test.ts +0 -699
- package/scripts/sync-version.ts +0 -35
- package/scripts/test-runner.ts +0 -76
- package/scripts/warmup.ts +0 -72
|
@@ -1,17 +1,19 @@
|
|
|
1
1
|
import { randomUUID, createHash } from "crypto";
|
|
2
|
-
import type { Memory, SearchIntent, IntentProfile, HybridRow } from "./memory
|
|
3
|
-
import { isDeleted } from "./memory
|
|
4
|
-
import type { SearchResult, SearchOptions } from "./conversation
|
|
5
|
-
import type { MemoryRepository } from "./memory.repository
|
|
6
|
-
import type { EmbeddingsService } from "./embeddings.service
|
|
7
|
-
import type { ConversationHistoryService } from "./conversation.service
|
|
8
|
-
|
|
2
|
+
import type { Memory, SearchIntent, IntentProfile, HybridRow } from "./memory";
|
|
3
|
+
import { isDeleted, computeConfidence } from "./memory";
|
|
4
|
+
import type { SearchResult, SearchOptions } from "./conversation";
|
|
5
|
+
import type { MemoryRepository } from "./memory.repository";
|
|
6
|
+
import type { EmbeddingsService } from "./embeddings.service";
|
|
7
|
+
import type { ConversationHistoryService } from "./conversation.service";
|
|
8
|
+
|
|
9
|
+
// Jitter values halved from original (0.02/0.05/0.15) because RRF_K=10 produces
|
|
10
|
+
// ~6x more score spread than K=60, amplifying jitter's disruption effect.
|
|
9
11
|
const INTENT_PROFILES: Record<SearchIntent, IntentProfile> = {
|
|
10
|
-
continuity: { weights: { relevance: 0.3, recency: 0.5, utility: 0.2 }, jitter: 0.
|
|
11
|
-
fact_check: { weights: { relevance: 0.6, recency: 0.1, utility: 0.3 }, jitter: 0.
|
|
12
|
-
frequent: { weights: { relevance: 0.2, recency: 0.2, utility: 0.6 }, jitter: 0.
|
|
13
|
-
associative: { weights: { relevance: 0.7, recency: 0.1, utility: 0.2 }, jitter: 0.
|
|
14
|
-
explore: { weights: { relevance: 0.4, recency: 0.3, utility: 0.3 }, jitter: 0.
|
|
12
|
+
continuity: { weights: { relevance: 0.3, recency: 0.5, utility: 0.2 }, jitter: 0.01 },
|
|
13
|
+
fact_check: { weights: { relevance: 0.6, recency: 0.1, utility: 0.3 }, jitter: 0.01 },
|
|
14
|
+
frequent: { weights: { relevance: 0.2, recency: 0.2, utility: 0.6 }, jitter: 0.01 },
|
|
15
|
+
associative: { weights: { relevance: 0.7, recency: 0.1, utility: 0.2 }, jitter: 0.025 },
|
|
16
|
+
explore: { weights: { relevance: 0.4, recency: 0.3, utility: 0.3 }, jitter: 0.08 },
|
|
15
17
|
};
|
|
16
18
|
|
|
17
19
|
const sigmoid = (x: number): number => 1 / (1 + Math.exp(-x));
|
|
@@ -87,19 +89,10 @@ export class MemoryService {
|
|
|
87
89
|
async getMultiple(ids: string[]): Promise<Memory[]> {
|
|
88
90
|
if (ids.length === 0) return [];
|
|
89
91
|
const memories = await this.repository.findByIds(ids);
|
|
90
|
-
// Track access in bulk
|
|
91
92
|
const now = new Date();
|
|
92
|
-
const
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
this.repository.upsert({
|
|
96
|
-
...m,
|
|
97
|
-
accessCount: m.accessCount + 1,
|
|
98
|
-
lastAccessed: now,
|
|
99
|
-
})
|
|
100
|
-
)
|
|
101
|
-
);
|
|
102
|
-
return live;
|
|
93
|
+
const liveIds = memories.filter((m) => !isDeleted(m)).map((m) => m.id);
|
|
94
|
+
this.repository.bulkUpdateAccess(liveIds, now);
|
|
95
|
+
return memories.filter((m) => !isDeleted(m));
|
|
103
96
|
}
|
|
104
97
|
|
|
105
98
|
async delete(id: string): Promise<boolean> {
|
|
@@ -186,10 +179,10 @@ export class MemoryService {
|
|
|
186
179
|
async search(
|
|
187
180
|
query: string,
|
|
188
181
|
intent: SearchIntent,
|
|
189
|
-
limit: number = 10,
|
|
190
|
-
includeDeleted: boolean = false,
|
|
191
182
|
options?: SearchOptions
|
|
192
183
|
): Promise<SearchResult[]> {
|
|
184
|
+
const limit = options?.limit ?? 10;
|
|
185
|
+
const includeDeleted = options?.includeDeleted ?? false;
|
|
193
186
|
const queryEmbedding = await this.embeddings.embed(query);
|
|
194
187
|
const profile = INTENT_PROFILES[intent];
|
|
195
188
|
const now = new Date();
|
|
@@ -223,6 +216,7 @@ export class MemoryService {
|
|
|
223
216
|
updatedAt: candidate.updatedAt,
|
|
224
217
|
source: "memory" as const,
|
|
225
218
|
score: this.computeMemoryScore(candidate, profile, now),
|
|
219
|
+
confidence: computeConfidence(candidate.signals),
|
|
226
220
|
supersededBy: candidate.supersededBy,
|
|
227
221
|
usefulness: candidate.usefulness,
|
|
228
222
|
accessCount: candidate.accessCount,
|
|
@@ -249,6 +243,7 @@ export class MemoryService {
|
|
|
249
243
|
updatedAt: row.createdAt,
|
|
250
244
|
source: "conversation_history" as const,
|
|
251
245
|
score: row.rrfScore * historyWeight,
|
|
246
|
+
confidence: computeConfidence(row.signals),
|
|
252
247
|
supersededBy: null,
|
|
253
248
|
sessionId: (row.metadata?.session_id as string) ?? "",
|
|
254
249
|
role: (row.metadata?.role as string) ?? "unknown",
|
|
@@ -272,19 +267,7 @@ export class MemoryService {
|
|
|
272
267
|
|
|
273
268
|
async trackAccess(ids: string[]): Promise<void> {
|
|
274
269
|
if (ids.length === 0) return;
|
|
275
|
-
|
|
276
|
-
const now = new Date();
|
|
277
|
-
await Promise.all(
|
|
278
|
-
memories
|
|
279
|
-
.filter((m) => !isDeleted(m))
|
|
280
|
-
.map((m) =>
|
|
281
|
-
this.repository.upsert({
|
|
282
|
-
...m,
|
|
283
|
-
accessCount: m.accessCount + 1,
|
|
284
|
-
lastAccessed: now,
|
|
285
|
-
})
|
|
286
|
-
)
|
|
287
|
-
);
|
|
270
|
+
this.repository.bulkUpdateAccess(ids, new Date());
|
|
288
271
|
}
|
|
289
272
|
|
|
290
273
|
private static readonly UUID_ZERO =
|
|
@@ -292,8 +275,16 @@ export class MemoryService {
|
|
|
292
275
|
|
|
293
276
|
private static waypointId(project?: string): string {
|
|
294
277
|
if (!project?.length) return MemoryService.UUID_ZERO;
|
|
295
|
-
const
|
|
296
|
-
|
|
278
|
+
const normalized = project.trim().toLowerCase();
|
|
279
|
+
const hex = createHash("sha256").update(`waypoint:${normalized}`).digest("hex");
|
|
280
|
+
return `wp:${hex.slice(0, 32)}`;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/** Legacy UUID-formatted waypoint ID for migration fallback reads. */
|
|
284
|
+
private static legacyWaypointId(project?: string): string | null {
|
|
285
|
+
if (!project?.length) return null; // UUID_ZERO is still current for no-project
|
|
286
|
+
const normalized = project.trim().toLowerCase();
|
|
287
|
+
const hex = createHash("sha256").update(`waypoint:${normalized}`).digest("hex");
|
|
297
288
|
return [
|
|
298
289
|
hex.slice(0, 8),
|
|
299
290
|
hex.slice(8, 12),
|
|
@@ -386,6 +377,20 @@ ${list(args.memory_ids)}`;
|
|
|
386
377
|
}
|
|
387
378
|
|
|
388
379
|
async getLatestWaypoint(project?: string): Promise<Memory | null> {
|
|
389
|
-
|
|
380
|
+
const waypoint = await this.get(MemoryService.waypointId(project));
|
|
381
|
+
if (waypoint) return waypoint;
|
|
382
|
+
|
|
383
|
+
// Fallback: try legacy UUID-formatted waypoint ID and migrate on read
|
|
384
|
+
const legacyId = MemoryService.legacyWaypointId(project);
|
|
385
|
+
if (!legacyId) return null;
|
|
386
|
+
|
|
387
|
+
const legacy = await this.repository.findById(legacyId);
|
|
388
|
+
if (!legacy) return null;
|
|
389
|
+
|
|
390
|
+
// Migrate: write under new ID, delete old
|
|
391
|
+
const newId = MemoryService.waypointId(project);
|
|
392
|
+
await this.repository.upsert({ ...legacy, id: newId });
|
|
393
|
+
await this.repository.markDeleted(legacyId);
|
|
394
|
+
return { ...legacy, id: newId };
|
|
390
395
|
}
|
|
391
396
|
}
|
package/server/core/memory.ts
CHANGED
|
@@ -38,7 +38,46 @@ export interface IntentProfile {
|
|
|
38
38
|
jitter: number;
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
+
/** Signals preserved from the hybrid search pipeline for confidence scoring. */
|
|
42
|
+
export interface SearchSignals {
|
|
43
|
+
cosineSimilarity: number | null;
|
|
44
|
+
ftsMatch: boolean;
|
|
45
|
+
knnRank: number | null;
|
|
46
|
+
ftsRank: number | null;
|
|
47
|
+
}
|
|
48
|
+
|
|
41
49
|
/** Augments any entity type with an RRF score from hybrid search. */
|
|
42
|
-
export type WithRrfScore<T> = T & { rrfScore: number };
|
|
50
|
+
export type WithRrfScore<T> = T & { rrfScore: number; signals: SearchSignals };
|
|
43
51
|
|
|
44
52
|
export type HybridRow = WithRrfScore<Memory>;
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Compute absolute confidence (0-1) from search signals.
|
|
56
|
+
*
|
|
57
|
+
* Based primarily on cosine similarity (the strongest absolute signal)
|
|
58
|
+
* mapped through a sigmoid with an agreement bonus for dual-path matches.
|
|
59
|
+
* The midpoint and steepness are calibrated for all-MiniLM-L6-v2 embeddings.
|
|
60
|
+
*/
|
|
61
|
+
// Calibrated against all-MiniLM-L6-v2: noise ceiling ~0.25, weak-relevant floor ~0.30
|
|
62
|
+
const CONFIDENCE_STEEPNESS = 14;
|
|
63
|
+
const CONFIDENCE_MIDPOINT = 0.35;
|
|
64
|
+
const CONFIDENCE_AGREEMENT_BONUS = 0.08;
|
|
65
|
+
|
|
66
|
+
export function computeConfidence(signals: SearchSignals): number {
|
|
67
|
+
const sim = signals.cosineSimilarity;
|
|
68
|
+
|
|
69
|
+
if (sim === null) {
|
|
70
|
+
// FTS-only result — keyword match but no semantic confirmation
|
|
71
|
+
return signals.ftsMatch ? 0.40 : 0.0;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Shifted sigmoid: maps cosine similarity to interpretable confidence
|
|
75
|
+
let confidence = 1 / (1 + Math.exp(-CONFIDENCE_STEEPNESS * (sim - CONFIDENCE_MIDPOINT)));
|
|
76
|
+
|
|
77
|
+
// Dual-path agreement bonus: found by both KNN and FTS
|
|
78
|
+
if (signals.ftsMatch) {
|
|
79
|
+
confidence = Math.min(1.0, confidence + CONFIDENCE_AGREEMENT_BONUS);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
return confidence;
|
|
83
|
+
}
|
|
@@ -3,9 +3,9 @@ import { createHash } from "crypto";
|
|
|
3
3
|
import { existsSync, statSync, readdirSync } from "fs";
|
|
4
4
|
import { resolve, dirname } from "path";
|
|
5
5
|
import { fileURLToPath } from "url";
|
|
6
|
-
import { serializeVector } from "./sqlite-utils
|
|
7
|
-
import type { MemoryRepository } from "./memory.repository
|
|
8
|
-
import type { EmbeddingsService } from "./embeddings.service
|
|
6
|
+
import { serializeVector } from "./sqlite-utils";
|
|
7
|
+
import type { MemoryRepository } from "./memory.repository";
|
|
8
|
+
import type { EmbeddingsService } from "./embeddings.service";
|
|
9
9
|
|
|
10
10
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
11
11
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
|
-
import type { EmbeddingsService } from "./embeddings.service
|
|
3
|
-
import { serializeVector } from "./sqlite-utils
|
|
2
|
+
import type { EmbeddingsService } from "./embeddings.service";
|
|
3
|
+
import { serializeVector } from "./sqlite-utils";
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Pre-migration step: remove vec0 virtual table entries from sqlite_master
|
|
@@ -127,8 +127,25 @@ export async function backfillVectors(
|
|
|
127
127
|
db: Database,
|
|
128
128
|
embeddings: EmbeddingsService,
|
|
129
129
|
): Promise<void> {
|
|
130
|
+
// Quick gap check: if no rows are missing vectors, skip the expensive backfill
|
|
131
|
+
const hasMemories = db.prepare("SELECT 1 FROM memories LIMIT 1").get();
|
|
132
|
+
const hasConvos = db.prepare("SELECT 1 FROM conversation_history LIMIT 1").get();
|
|
133
|
+
|
|
134
|
+
if (!hasMemories && !hasConvos) return;
|
|
135
|
+
|
|
136
|
+
const memoryGap = hasMemories && db.prepare(
|
|
137
|
+
`SELECT 1 FROM memories m LEFT JOIN memories_vec v ON m.id = v.id
|
|
138
|
+
WHERE v.id IS NULL OR length(v.vector) = 0 LIMIT 1`,
|
|
139
|
+
).get();
|
|
140
|
+
|
|
141
|
+
const convoGap = hasConvos && db.prepare(
|
|
142
|
+
`SELECT 1 FROM conversation_history c LEFT JOIN conversation_history_vec v ON c.id = v.id
|
|
143
|
+
WHERE v.id IS NULL OR length(v.vector) = 0 LIMIT 1`,
|
|
144
|
+
).get();
|
|
145
|
+
|
|
146
|
+
if (!memoryGap && !convoGap) return;
|
|
147
|
+
|
|
130
148
|
// ── Memories ──────────────────────────────────────────────────────
|
|
131
|
-
// Catch both missing rows (v.id IS NULL) and corrupt 0-byte BLOBs
|
|
132
149
|
const missingMemories = db
|
|
133
150
|
.prepare(
|
|
134
151
|
`SELECT m.id, m.content, json_extract(m.metadata, '$.type') AS type
|
|
@@ -151,14 +168,27 @@ export async function backfillVectors(
|
|
|
151
168
|
new Array(embeddings.dimension).fill(0),
|
|
152
169
|
);
|
|
153
170
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
row.type === "waypoint"
|
|
158
|
-
? zeroVector
|
|
159
|
-
: serializeVector(await embeddings.embed(row.content));
|
|
171
|
+
// Separate waypoints from content that needs embedding
|
|
172
|
+
const toEmbed = missingMemories.filter((r) => r.type !== "waypoint");
|
|
173
|
+
const waypoints = missingMemories.filter((r) => r.type === "waypoint");
|
|
160
174
|
|
|
161
|
-
|
|
175
|
+
// Batch embed all non-waypoint content
|
|
176
|
+
const vectors = toEmbed.length > 0
|
|
177
|
+
? await embeddings.embedBatch(toEmbed.map((r) => r.content))
|
|
178
|
+
: [];
|
|
179
|
+
|
|
180
|
+
db.exec("BEGIN");
|
|
181
|
+
try {
|
|
182
|
+
for (const row of waypoints) {
|
|
183
|
+
insertVec.run(row.id, zeroVector);
|
|
184
|
+
}
|
|
185
|
+
for (let i = 0; i < toEmbed.length; i++) {
|
|
186
|
+
insertVec.run(toEmbed[i].id, serializeVector(vectors[i]));
|
|
187
|
+
}
|
|
188
|
+
db.exec("COMMIT");
|
|
189
|
+
} catch (e) {
|
|
190
|
+
db.exec("ROLLBACK");
|
|
191
|
+
throw e;
|
|
162
192
|
}
|
|
163
193
|
|
|
164
194
|
console.error(
|
|
@@ -185,17 +215,27 @@ export async function backfillVectors(
|
|
|
185
215
|
"INSERT OR REPLACE INTO conversation_history_vec (id, vector) VALUES (?, ?)",
|
|
186
216
|
);
|
|
187
217
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
218
|
+
// Batch embed in chunks of 32
|
|
219
|
+
const BATCH_SIZE = 32;
|
|
220
|
+
db.exec("BEGIN");
|
|
221
|
+
try {
|
|
222
|
+
for (let i = 0; i < missingConvos.length; i += BATCH_SIZE) {
|
|
223
|
+
const batch = missingConvos.slice(i, i + BATCH_SIZE);
|
|
224
|
+
const vecs = await embeddings.embedBatch(batch.map((r) => r.content));
|
|
225
|
+
for (let j = 0; j < batch.length; j++) {
|
|
226
|
+
insertConvoVec.run(batch[j].id, serializeVector(vecs[j]));
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
if ((i + BATCH_SIZE) % 100 < BATCH_SIZE) {
|
|
230
|
+
console.error(
|
|
231
|
+
`[vector-memory-mcp] ...${Math.min(i + BATCH_SIZE, missingConvos.length)}/${missingConvos.length} conversation chunks`,
|
|
232
|
+
);
|
|
233
|
+
}
|
|
198
234
|
}
|
|
235
|
+
db.exec("COMMIT");
|
|
236
|
+
} catch (e) {
|
|
237
|
+
db.exec("ROLLBACK");
|
|
238
|
+
throw e;
|
|
199
239
|
}
|
|
200
240
|
|
|
201
241
|
console.error(
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { readFile, readdir, stat } from "fs/promises";
|
|
2
2
|
import { basename, dirname, join } from "path";
|
|
3
|
-
import type { ParsedMessage, SessionFileInfo } from "../conversation
|
|
4
|
-
import type { SessionLogParser } from "./types
|
|
3
|
+
import type { ParsedMessage, SessionFileInfo } from "../conversation";
|
|
4
|
+
import type { SessionLogParser } from "./types";
|
|
5
5
|
|
|
6
6
|
// UUID pattern for session IDs
|
|
7
7
|
const UUID_PATTERN =
|
|
@@ -45,7 +45,7 @@ export class ClaudeCodeSessionParser implements SessionLogParser {
|
|
|
45
45
|
const fileName = basename(filePath, ".jsonl");
|
|
46
46
|
const parentDir = basename(dirname(filePath));
|
|
47
47
|
// Check if this is inside a subagents directory
|
|
48
|
-
const isSubagentFile =
|
|
48
|
+
const isSubagentFile = /[/\\]subagents[/\\]/.test(filePath);
|
|
49
49
|
|
|
50
50
|
// For subagent files, project dir is 3 levels up: <project>/<session>/subagents/<file>
|
|
51
51
|
// For main files, project dir is direct parent
|
|
@@ -1,7 +1,29 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
2
|
|
|
3
|
-
/** RRF constant
|
|
4
|
-
export const RRF_K =
|
|
3
|
+
/** RRF constant — lower K gives sharper top-rank discrimination in the 1/(K+rank) formula */
|
|
4
|
+
export const RRF_K = 10;
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Maximum parameters per SQLite query to stay within SQLITE_MAX_VARIABLE_NUMBER.
|
|
8
|
+
*/
|
|
9
|
+
export const SQLITE_BATCH_SIZE = 100;
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Execute a query in batches when the number of parameters exceeds SQLITE_BATCH_SIZE.
|
|
13
|
+
* Splits the ids array and concatenates results.
|
|
14
|
+
*/
|
|
15
|
+
export function batchedQuery<T>(
|
|
16
|
+
db: Database,
|
|
17
|
+
ids: string[],
|
|
18
|
+
queryFn: (batch: string[]) => T[]
|
|
19
|
+
): T[] {
|
|
20
|
+
if (ids.length <= SQLITE_BATCH_SIZE) return queryFn(ids);
|
|
21
|
+
const results: T[] = [];
|
|
22
|
+
for (let i = 0; i < ids.length; i += SQLITE_BATCH_SIZE) {
|
|
23
|
+
results.push(...queryFn(ids.slice(i, i + SQLITE_BATCH_SIZE)));
|
|
24
|
+
}
|
|
25
|
+
return results;
|
|
26
|
+
}
|
|
5
27
|
|
|
6
28
|
/**
|
|
7
29
|
* Serialize a number[] embedding to raw float32 bytes for BLOB storage.
|
|
@@ -94,6 +116,49 @@ export function hybridRRF(
|
|
|
94
116
|
return scores;
|
|
95
117
|
}
|
|
96
118
|
|
|
119
|
+
import type { SearchSignals } from "./memory";
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Compute hybrid RRF scores while preserving per-result search signals
|
|
123
|
+
* (cosine similarity, FTS match, rank positions) for confidence scoring.
|
|
124
|
+
*/
|
|
125
|
+
export function hybridRRFWithSignals(
|
|
126
|
+
vectorResults: Array<{ id: string; distance: number }>,
|
|
127
|
+
ftsResults: Array<{ id: string }>,
|
|
128
|
+
k: number = RRF_K
|
|
129
|
+
): Map<string, SearchSignals & { rrfScore: number }> {
|
|
130
|
+
const knnMap = new Map<string, { similarity: number; rank: number }>();
|
|
131
|
+
vectorResults.forEach((r, i) => {
|
|
132
|
+
knnMap.set(r.id, { similarity: 1 - r.distance, rank: i + 1 });
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
const ftsMap = new Map<string, number>();
|
|
136
|
+
ftsResults.forEach((r, i) => {
|
|
137
|
+
ftsMap.set(r.id, i + 1);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
const allIds = new Set([...knnMap.keys(), ...ftsMap.keys()]);
|
|
141
|
+
const results = new Map<string, SearchSignals & { rrfScore: number }>();
|
|
142
|
+
|
|
143
|
+
for (const id of allIds) {
|
|
144
|
+
const knn = knnMap.get(id);
|
|
145
|
+
const ftsRank = ftsMap.get(id) ?? null;
|
|
146
|
+
let rrfScore = 0;
|
|
147
|
+
if (knn) rrfScore += 1 / (k + knn.rank);
|
|
148
|
+
if (ftsRank !== null) rrfScore += 1 / (k + ftsRank);
|
|
149
|
+
|
|
150
|
+
results.set(id, {
|
|
151
|
+
rrfScore,
|
|
152
|
+
cosineSimilarity: knn?.similarity ?? null,
|
|
153
|
+
ftsMatch: ftsRank !== null,
|
|
154
|
+
knnRank: knn?.rank ?? null,
|
|
155
|
+
ftsRank,
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
return results;
|
|
160
|
+
}
|
|
161
|
+
|
|
97
162
|
/**
|
|
98
163
|
* Sort ids by RRF score descending and return top N.
|
|
99
164
|
*/
|
package/server/index.ts
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
|
|
3
|
-
import { loadConfig, parseCliArgs } from "./config/index
|
|
4
|
-
import { connectToDatabase } from "./core/connection
|
|
5
|
-
import { backfillVectors } from "./core/migrations
|
|
6
|
-
import { MemoryRepository } from "./core/memory.repository
|
|
7
|
-
import { ConversationRepository } from "./core/conversation.repository
|
|
8
|
-
import { EmbeddingsService } from "./core/embeddings.service
|
|
9
|
-
import { MemoryService } from "./core/memory.service
|
|
10
|
-
import { ConversationHistoryService } from "./core/conversation.service
|
|
11
|
-
import { startServer } from "./transports/mcp/server
|
|
12
|
-
import { startHttpServer } from "./transports/http/server
|
|
3
|
+
import { loadConfig, parseCliArgs } from "./config/index";
|
|
4
|
+
import { connectToDatabase } from "./core/connection";
|
|
5
|
+
import { backfillVectors } from "./core/migrations";
|
|
6
|
+
import { MemoryRepository } from "./core/memory.repository";
|
|
7
|
+
import { ConversationRepository } from "./core/conversation.repository";
|
|
8
|
+
import { EmbeddingsService } from "./core/embeddings.service";
|
|
9
|
+
import { MemoryService } from "./core/memory.service";
|
|
10
|
+
import { ConversationHistoryService } from "./core/conversation.service";
|
|
11
|
+
import { startServer } from "./transports/mcp/server";
|
|
12
|
+
import { startHttpServer } from "./transports/http/server";
|
|
13
13
|
|
|
14
14
|
async function main(): Promise<void> {
|
|
15
15
|
const args = process.argv.slice(2);
|
|
@@ -25,17 +25,15 @@ async function main(): Promise<void> {
|
|
|
25
25
|
const overrides = parseCliArgs(args);
|
|
26
26
|
const config = loadConfig(overrides);
|
|
27
27
|
|
|
28
|
-
// Initialize database
|
|
28
|
+
// Initialize database and backfill any missing vectors before services start
|
|
29
29
|
const db = connectToDatabase(config.dbPath);
|
|
30
|
+
const embeddings = new EmbeddingsService(config.embeddingModel, config.embeddingDimension);
|
|
31
|
+
await backfillVectors(db, embeddings);
|
|
30
32
|
|
|
31
33
|
// Initialize layers
|
|
32
34
|
const repository = new MemoryRepository(db);
|
|
33
|
-
const embeddings = new EmbeddingsService(config.embeddingModel, config.embeddingDimension);
|
|
34
35
|
const memoryService = new MemoryService(repository, embeddings);
|
|
35
36
|
|
|
36
|
-
// Backfill any missing vectors (e.g. after vec0-to-BLOB migration)
|
|
37
|
-
await backfillVectors(db, embeddings);
|
|
38
|
-
|
|
39
37
|
if (config.pluginMode) {
|
|
40
38
|
console.error("[vector-memory-mcp] Running in plugin mode");
|
|
41
39
|
}
|
|
@@ -21,11 +21,11 @@ import {
|
|
|
21
21
|
} from "@modelcontextprotocol/sdk/types.js";
|
|
22
22
|
import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
|
|
23
23
|
|
|
24
|
-
import { tools } from "../mcp/tools
|
|
25
|
-
import { handleToolCall } from "../mcp/handlers
|
|
26
|
-
import { SERVER_INSTRUCTIONS } from "../mcp/server
|
|
27
|
-
import { VERSION } from "../../config/index
|
|
28
|
-
import type { MemoryService } from "../../core/memory.service
|
|
24
|
+
import { tools } from "../mcp/tools";
|
|
25
|
+
import { handleToolCall } from "../mcp/handlers";
|
|
26
|
+
import { SERVER_INSTRUCTIONS } from "../mcp/server";
|
|
27
|
+
import { VERSION } from "../../config/index";
|
|
28
|
+
import type { MemoryService } from "../../core/memory.service";
|
|
29
29
|
|
|
30
30
|
interface Session {
|
|
31
31
|
server: Server;
|
|
@@ -3,11 +3,11 @@ import { cors } from "hono/cors";
|
|
|
3
3
|
import { createServer } from "net";
|
|
4
4
|
import { writeFileSync, mkdirSync, unlinkSync } from "fs";
|
|
5
5
|
import { join } from "path";
|
|
6
|
-
import type { MemoryService } from "../../core/memory.service
|
|
7
|
-
import type { Config } from "../../config/index
|
|
8
|
-
import { isDeleted } from "../../core/memory
|
|
9
|
-
import { createMcpRoutes } from "./mcp-transport
|
|
10
|
-
import type { Memory, SearchIntent } from "../../core/memory
|
|
6
|
+
import type { MemoryService } from "../../core/memory.service";
|
|
7
|
+
import type { Config } from "../../config/index";
|
|
8
|
+
import { isDeleted } from "../../core/memory";
|
|
9
|
+
import { createMcpRoutes } from "./mcp-transport";
|
|
10
|
+
import type { Memory, SearchIntent } from "../../core/memory";
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
/**
|
|
@@ -111,10 +111,22 @@ export function createHttpApp(memoryService: MemoryService, config: Config): Hon
|
|
|
111
111
|
embeddingDimension: config.embeddingDimension,
|
|
112
112
|
historyEnabled: config.conversationHistory.enabled,
|
|
113
113
|
pluginMode: config.pluginMode,
|
|
114
|
+
embeddingReady: memoryService.getEmbeddings().isReady,
|
|
114
115
|
},
|
|
115
116
|
});
|
|
116
117
|
});
|
|
117
118
|
|
|
119
|
+
// Warmup endpoint — triggers ONNX model load if not already cached
|
|
120
|
+
app.post("/warmup", async (c) => {
|
|
121
|
+
const embeddings = memoryService.getEmbeddings();
|
|
122
|
+
if (embeddings.isReady) {
|
|
123
|
+
return c.json({ status: "already_warm" });
|
|
124
|
+
}
|
|
125
|
+
const start = Date.now();
|
|
126
|
+
await embeddings.warmup();
|
|
127
|
+
return c.json({ status: "warmed", elapsed: Date.now() - start });
|
|
128
|
+
});
|
|
129
|
+
|
|
118
130
|
// Search endpoint
|
|
119
131
|
app.post("/search", async (c) => {
|
|
120
132
|
try {
|
|
@@ -127,7 +139,7 @@ export function createHttpApp(memoryService: MemoryService, config: Config): Hon
|
|
|
127
139
|
return c.json({ error: "Missing or invalid 'query' field" }, 400);
|
|
128
140
|
}
|
|
129
141
|
|
|
130
|
-
const results = await memoryService.search(query, intent, limit);
|
|
142
|
+
const results = await memoryService.search(query, intent, { limit });
|
|
131
143
|
|
|
132
144
|
return c.json({
|
|
133
145
|
results: results.map((r) => ({
|
|
@@ -135,6 +147,7 @@ export function createHttpApp(memoryService: MemoryService, config: Config): Hon
|
|
|
135
147
|
content: r.content,
|
|
136
148
|
metadata: r.metadata,
|
|
137
149
|
source: r.source,
|
|
150
|
+
confidence: r.confidence,
|
|
138
151
|
createdAt: r.createdAt.toISOString(),
|
|
139
152
|
})),
|
|
140
153
|
count: results.length,
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
|
|
2
|
-
import type { MemoryService } from "../../core/memory.service
|
|
3
|
-
import type { ConversationHistoryService } from "../../core/conversation.service
|
|
4
|
-
import type { SearchIntent } from "../../core/memory
|
|
5
|
-
import type { HistoryFilters, SearchResult } from "../../core/conversation
|
|
6
|
-
import { DEBUG } from "../../config/index
|
|
2
|
+
import type { MemoryService } from "../../core/memory.service";
|
|
3
|
+
import type { ConversationHistoryService } from "../../core/conversation.service";
|
|
4
|
+
import type { SearchIntent } from "../../core/memory";
|
|
5
|
+
import type { HistoryFilters, SearchResult } from "../../core/conversation";
|
|
6
|
+
import { DEBUG } from "../../config/index";
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* Safely coerce a tool argument to an array. Handles the case where the MCP
|
|
@@ -51,6 +51,14 @@ function parseDate(value: unknown, fieldName: string): Date | undefined {
|
|
|
51
51
|
return date;
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
+
function requireString(args: Record<string, unknown> | undefined, field: string): string {
|
|
55
|
+
const value = args?.[field];
|
|
56
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
57
|
+
throw new Error(`${field} is required`);
|
|
58
|
+
}
|
|
59
|
+
return value;
|
|
60
|
+
}
|
|
61
|
+
|
|
54
62
|
export async function handleStoreMemories(
|
|
55
63
|
args: Record<string, unknown> | undefined,
|
|
56
64
|
service: MemoryService
|
|
@@ -189,7 +197,9 @@ export async function handleSearchMemories(
|
|
|
189
197
|
return errorResult(errorText(e));
|
|
190
198
|
}
|
|
191
199
|
|
|
192
|
-
const results = await service.search(query, intent,
|
|
200
|
+
const results = await service.search(query, intent, {
|
|
201
|
+
limit,
|
|
202
|
+
includeDeleted,
|
|
193
203
|
includeHistory,
|
|
194
204
|
historyOnly,
|
|
195
205
|
historyFilters,
|
|
@@ -202,19 +212,7 @@ export async function handleSearchMemories(
|
|
|
202
212
|
};
|
|
203
213
|
}
|
|
204
214
|
|
|
205
|
-
const formatted = results.map((r
|
|
206
|
-
let result = `[${r.source}] ID: ${r.id}\nContent: ${r.content}`;
|
|
207
|
-
if (r.metadata && Object.keys(r.metadata).length > 0) {
|
|
208
|
-
result += `\nMetadata: ${JSON.stringify(r.metadata)}`;
|
|
209
|
-
}
|
|
210
|
-
if (r.source === "memory" && includeDeleted && r.supersededBy) {
|
|
211
|
-
result += `\n[DELETED]`;
|
|
212
|
-
}
|
|
213
|
-
if (r.source === "conversation_history" && r.sessionId) {
|
|
214
|
-
result += `\nSession: ${r.sessionId}`;
|
|
215
|
-
}
|
|
216
|
-
return result;
|
|
217
|
-
});
|
|
215
|
+
const formatted = results.map((r) => formatSearchResult(r, includeDeleted));
|
|
218
216
|
|
|
219
217
|
return {
|
|
220
218
|
content: [{ type: "text", text: formatted.join("\n\n---\n\n") }],
|
|
@@ -241,6 +239,20 @@ function formatMemoryDetail(
|
|
|
241
239
|
return result;
|
|
242
240
|
}
|
|
243
241
|
|
|
242
|
+
function formatSearchResult(r: SearchResult, includeDeleted: boolean): string {
|
|
243
|
+
let result = `[${r.source}] ID: ${r.id}\nConfidence: ${r.confidence.toFixed(2)}\nContent: ${r.content}`;
|
|
244
|
+
if (r.metadata && Object.keys(r.metadata).length > 0) {
|
|
245
|
+
result += `\nMetadata: ${JSON.stringify(r.metadata)}`;
|
|
246
|
+
}
|
|
247
|
+
if (r.source === "memory" && includeDeleted && r.supersededBy) {
|
|
248
|
+
result += `\n[DELETED]`;
|
|
249
|
+
}
|
|
250
|
+
if (r.source === "conversation_history" && r.sessionId) {
|
|
251
|
+
result += `\nSession: ${r.sessionId}`;
|
|
252
|
+
}
|
|
253
|
+
return result;
|
|
254
|
+
}
|
|
255
|
+
|
|
244
256
|
export async function handleGetMemories(
|
|
245
257
|
args: Record<string, unknown> | undefined,
|
|
246
258
|
service: MemoryService
|
|
@@ -267,8 +279,11 @@ export async function handleReportMemoryUsefulness(
|
|
|
267
279
|
args: Record<string, unknown> | undefined,
|
|
268
280
|
service: MemoryService
|
|
269
281
|
): Promise<CallToolResult> {
|
|
270
|
-
const memoryId = args
|
|
271
|
-
const useful = args?.useful
|
|
282
|
+
const memoryId = requireString(args, "memory_id");
|
|
283
|
+
const useful = args?.useful;
|
|
284
|
+
if (typeof useful !== "boolean") {
|
|
285
|
+
return errorResult("useful is required and must be a boolean");
|
|
286
|
+
}
|
|
272
287
|
|
|
273
288
|
const memory = await service.vote(memoryId, useful ? 1 : -1);
|
|
274
289
|
|
|
@@ -290,10 +305,19 @@ export async function handleSetWaypoint(
|
|
|
290
305
|
args: Record<string, unknown> | undefined,
|
|
291
306
|
service: MemoryService
|
|
292
307
|
): Promise<CallToolResult> {
|
|
308
|
+
let project: string;
|
|
309
|
+
let summary: string;
|
|
310
|
+
try {
|
|
311
|
+
project = requireString(args, "project");
|
|
312
|
+
summary = requireString(args, "summary");
|
|
313
|
+
} catch (e) {
|
|
314
|
+
return errorResult(errorText(e));
|
|
315
|
+
}
|
|
316
|
+
|
|
293
317
|
const memory = await service.setWaypoint({
|
|
294
|
-
project
|
|
318
|
+
project,
|
|
295
319
|
branch: args?.branch as string | undefined,
|
|
296
|
-
summary
|
|
320
|
+
summary,
|
|
297
321
|
completed: (args?.completed as string[] | undefined) ?? [],
|
|
298
322
|
in_progress_blocked: (args?.in_progress_blocked as string[] | undefined) ?? [],
|
|
299
323
|
key_decisions: (args?.key_decisions as string[] | undefined) ?? [],
|