@aeriondyseti/vector-memory-mcp 2.4.0 → 2.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/server/core/connection.ts +1 -1
- package/server/core/conversation.repository.ts +91 -2
- package/server/core/conversation.service.ts +19 -19
- package/server/core/conversation.ts +2 -5
- package/server/core/embeddings.service.ts +6 -0
- package/server/core/memory.repository.ts +35 -9
- package/server/core/memory.service.ts +37 -36
- package/server/core/migration.service.ts +3 -3
- package/server/core/migrations.ts +17 -35
- package/server/core/parsers/claude-code.parser.ts +3 -3
- package/server/core/parsers/types.ts +1 -1
- package/server/core/sqlite-utils.ts +22 -0
- package/server/index.ts +10 -10
- package/server/transports/http/mcp-transport.ts +5 -5
- package/server/transports/http/server.ts +6 -6
- package/server/transports/mcp/handlers.ts +47 -23
- package/server/transports/mcp/server.ts +5 -5
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Database } from "bun:sqlite";
|
|
2
2
|
import { existsSync, mkdirSync } from "fs";
|
|
3
3
|
import { dirname } from "path";
|
|
4
|
-
import { removeVec0Tables, runMigrations } from "./migrations
|
|
4
|
+
import { removeVec0Tables, runMigrations } from "./migrations";
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Open (or create) a SQLite database at the given path
|
|
@@ -2,7 +2,7 @@ import type { Database } from "bun:sqlite";
|
|
|
2
2
|
import type {
|
|
3
3
|
ConversationHybridRow,
|
|
4
4
|
HistoryFilters,
|
|
5
|
-
} from "./conversation
|
|
5
|
+
} from "./conversation";
|
|
6
6
|
import {
|
|
7
7
|
serializeVector,
|
|
8
8
|
safeParseJsonObject,
|
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
hybridRRF,
|
|
11
11
|
topByRRF,
|
|
12
12
|
knnSearch,
|
|
13
|
-
} from "./sqlite-utils
|
|
13
|
+
} from "./sqlite-utils";
|
|
14
14
|
|
|
15
15
|
export class ConversationRepository {
|
|
16
16
|
constructor(private db: Database) {}
|
|
@@ -105,6 +105,95 @@ export class ConversationRepository {
|
|
|
105
105
|
tx();
|
|
106
106
|
}
|
|
107
107
|
|
|
108
|
+
async replaceSession(
|
|
109
|
+
sessionId: string,
|
|
110
|
+
rows: Array<{
|
|
111
|
+
id: string;
|
|
112
|
+
vector: number[];
|
|
113
|
+
content: string;
|
|
114
|
+
metadata: string;
|
|
115
|
+
created_at: number;
|
|
116
|
+
session_id: string;
|
|
117
|
+
role: string;
|
|
118
|
+
message_index_start: number;
|
|
119
|
+
message_index_end: number;
|
|
120
|
+
project: string;
|
|
121
|
+
}>
|
|
122
|
+
): Promise<void> {
|
|
123
|
+
const insertMain = this.db.prepare(
|
|
124
|
+
`INSERT OR REPLACE INTO conversation_history
|
|
125
|
+
(id, content, metadata, created_at, session_id, role, message_index_start, message_index_end, project)
|
|
126
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
127
|
+
);
|
|
128
|
+
const deleteVec = this.db.prepare(
|
|
129
|
+
`DELETE FROM conversation_history_vec WHERE id = ?`
|
|
130
|
+
);
|
|
131
|
+
const insertVec = this.db.prepare(
|
|
132
|
+
`INSERT INTO conversation_history_vec (id, vector) VALUES (?, ?)`
|
|
133
|
+
);
|
|
134
|
+
const deleteFts = this.db.prepare(
|
|
135
|
+
`DELETE FROM conversation_history_fts WHERE id = ?`
|
|
136
|
+
);
|
|
137
|
+
const insertFts = this.db.prepare(
|
|
138
|
+
`INSERT INTO conversation_history_fts (id, content) VALUES (?, ?)`
|
|
139
|
+
);
|
|
140
|
+
|
|
141
|
+
const tx = this.db.transaction(() => {
|
|
142
|
+
// Delete old chunks first
|
|
143
|
+
const idRows = this.db
|
|
144
|
+
.prepare(`SELECT id FROM conversation_history WHERE session_id = ?`)
|
|
145
|
+
.all(sessionId) as Array<{ id: string }>;
|
|
146
|
+
|
|
147
|
+
if (idRows.length > 0) {
|
|
148
|
+
const ids = idRows.map((r) => r.id);
|
|
149
|
+
const placeholders = ids.map(() => "?").join(", ");
|
|
150
|
+
this.db
|
|
151
|
+
.prepare(
|
|
152
|
+
`DELETE FROM conversation_history_vec WHERE id IN (${placeholders})`
|
|
153
|
+
)
|
|
154
|
+
.run(...ids);
|
|
155
|
+
this.db
|
|
156
|
+
.prepare(
|
|
157
|
+
`DELETE FROM conversation_history_fts WHERE id IN (${placeholders})`
|
|
158
|
+
)
|
|
159
|
+
.run(...ids);
|
|
160
|
+
this.db
|
|
161
|
+
.prepare(`DELETE FROM conversation_history WHERE session_id = ?`)
|
|
162
|
+
.run(sessionId);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// Insert new chunks
|
|
166
|
+
for (const row of rows) {
|
|
167
|
+
insertMain.run(
|
|
168
|
+
row.id,
|
|
169
|
+
row.content,
|
|
170
|
+
row.metadata,
|
|
171
|
+
row.created_at,
|
|
172
|
+
row.session_id,
|
|
173
|
+
row.role,
|
|
174
|
+
row.message_index_start,
|
|
175
|
+
row.message_index_end,
|
|
176
|
+
row.project
|
|
177
|
+
);
|
|
178
|
+
deleteVec.run(row.id);
|
|
179
|
+
insertVec.run(row.id, serializeVector(row.vector));
|
|
180
|
+
deleteFts.run(row.id);
|
|
181
|
+
insertFts.run(row.id, row.content);
|
|
182
|
+
}
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
tx();
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Hybrid search combining vector KNN and FTS5, fused with Reciprocal Rank Fusion.
|
|
190
|
+
*
|
|
191
|
+
* NOTE: Filters (session, role, project, date) are applied AFTER candidate selection
|
|
192
|
+
* and RRF scoring, not pushed into the KNN/FTS queries. This is an intentional
|
|
193
|
+
* performance tradeoff — KNN is brute-force JS-side (no SQL pre-filter possible),
|
|
194
|
+
* and filtering post-RRF avoids duplicating filter logic across both retrieval paths.
|
|
195
|
+
* The consequence is that filtered queries may return fewer than `limit` results.
|
|
196
|
+
*/
|
|
108
197
|
async findHybrid(
|
|
109
198
|
embedding: number[],
|
|
110
199
|
query: string,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createHash } from "crypto";
|
|
2
2
|
import { readFile, writeFile, mkdir } from "fs/promises";
|
|
3
3
|
import { dirname, join } from "path";
|
|
4
|
-
import type { ConversationRepository } from "./conversation.repository
|
|
4
|
+
import type { ConversationRepository } from "./conversation.repository";
|
|
5
5
|
import type {
|
|
6
6
|
ConversationChunk,
|
|
7
7
|
ConversationHybridRow,
|
|
@@ -10,12 +10,12 @@ import type {
|
|
|
10
10
|
ParsedMessage,
|
|
11
11
|
SessionFileInfo,
|
|
12
12
|
SessionIndexDetail,
|
|
13
|
-
} from "./conversation
|
|
14
|
-
import type { ConversationHistoryConfig } from "../config/index
|
|
15
|
-
import { resolveSessionLogPath } from "../config/index
|
|
16
|
-
import type { EmbeddingsService } from "./embeddings.service
|
|
17
|
-
import type { SessionLogParser } from "./parsers/types
|
|
18
|
-
import { ClaudeCodeSessionParser } from "./parsers/claude-code.parser
|
|
13
|
+
} from "./conversation";
|
|
14
|
+
import type { ConversationHistoryConfig } from "../config/index";
|
|
15
|
+
import { resolveSessionLogPath } from "../config/index";
|
|
16
|
+
import type { EmbeddingsService } from "./embeddings.service";
|
|
17
|
+
import type { SessionLogParser } from "./parsers/types";
|
|
18
|
+
import { ClaudeCodeSessionParser } from "./parsers/claude-code.parser";
|
|
19
19
|
|
|
20
20
|
/**
|
|
21
21
|
* Generate a deterministic chunk ID from session ID and message indices.
|
|
@@ -78,12 +78,7 @@ export function chunkMessages(
|
|
|
78
78
|
messageIndexEnd: lastMsg.messageIndex,
|
|
79
79
|
project: firstMsg.project,
|
|
80
80
|
metadata: {
|
|
81
|
-
session_id: firstMsg.sessionId,
|
|
82
81
|
timestamp: firstMsg.timestamp.toISOString(),
|
|
83
|
-
role,
|
|
84
|
-
message_index_start: firstMsg.messageIndex,
|
|
85
|
-
message_index_end: lastMsg.messageIndex,
|
|
86
|
-
project: firstMsg.project,
|
|
87
82
|
git_branch: firstMsg.gitBranch,
|
|
88
83
|
is_subagent: firstMsg.isSubagent,
|
|
89
84
|
agent_id: firstMsg.agentId,
|
|
@@ -273,20 +268,24 @@ export class ConversationHistoryService {
|
|
|
273
268
|
this.config.chunkOverlap
|
|
274
269
|
);
|
|
275
270
|
|
|
276
|
-
//
|
|
277
|
-
await this.repository.deleteBySessionId(file.sessionId);
|
|
278
|
-
|
|
279
|
-
// Embed all chunks
|
|
271
|
+
// Embed all chunks FIRST (pure computation, no DB side effects)
|
|
280
272
|
const embeddings = await this.embeddings.embedBatch(
|
|
281
273
|
chunks.map((c) => c.content)
|
|
282
274
|
);
|
|
283
275
|
|
|
284
|
-
//
|
|
276
|
+
// Build rows
|
|
285
277
|
const rows = chunks.map((chunk, i) => ({
|
|
286
278
|
id: chunk.id,
|
|
287
279
|
vector: embeddings[i],
|
|
288
280
|
content: chunk.content,
|
|
289
|
-
metadata: JSON.stringify(
|
|
281
|
+
metadata: JSON.stringify({
|
|
282
|
+
...chunk.metadata,
|
|
283
|
+
session_id: chunk.sessionId,
|
|
284
|
+
role: chunk.role,
|
|
285
|
+
message_index_start: chunk.messageIndexStart,
|
|
286
|
+
message_index_end: chunk.messageIndexEnd,
|
|
287
|
+
project: chunk.project,
|
|
288
|
+
}),
|
|
290
289
|
created_at: chunk.timestamp.getTime(),
|
|
291
290
|
session_id: chunk.sessionId,
|
|
292
291
|
role: chunk.role,
|
|
@@ -295,7 +294,8 @@ export class ConversationHistoryService {
|
|
|
295
294
|
project: chunk.project,
|
|
296
295
|
}));
|
|
297
296
|
|
|
298
|
-
|
|
297
|
+
// Atomically replace old chunks with new ones
|
|
298
|
+
await this.repository.replaceSession(file.sessionId, rows);
|
|
299
299
|
|
|
300
300
|
// Update index state
|
|
301
301
|
const session: IndexedSession = {
|
|
@@ -14,12 +14,7 @@ export interface ParsedMessage {
|
|
|
14
14
|
|
|
15
15
|
/** Metadata stored per conversation chunk in the database */
|
|
16
16
|
export interface ConversationChunkMetadata {
|
|
17
|
-
session_id: string;
|
|
18
17
|
timestamp: string;
|
|
19
|
-
role: string;
|
|
20
|
-
message_index_start: number;
|
|
21
|
-
message_index_end: number;
|
|
22
|
-
project: string;
|
|
23
18
|
git_branch?: string;
|
|
24
19
|
is_subagent: boolean;
|
|
25
20
|
agent_id?: string;
|
|
@@ -114,6 +109,8 @@ export interface HistoryFilters {
|
|
|
114
109
|
|
|
115
110
|
/** Options for the integrated search across both sources */
|
|
116
111
|
export interface SearchOptions {
|
|
112
|
+
limit?: number;
|
|
113
|
+
includeDeleted?: boolean;
|
|
117
114
|
includeHistory?: boolean;
|
|
118
115
|
historyOnly?: boolean;
|
|
119
116
|
historyWeight?: number;
|
|
@@ -108,6 +108,12 @@ export class EmbeddingsService {
|
|
|
108
108
|
|
|
109
109
|
private meanPool(data: Float32Array, mask: number[], seqLen: number): number[] {
|
|
110
110
|
const dim = this._dimension;
|
|
111
|
+
const expectedLen = seqLen * dim;
|
|
112
|
+
if (data.length < expectedLen) {
|
|
113
|
+
throw new Error(
|
|
114
|
+
`ONNX output size ${data.length} < expected ${expectedLen} (seqLen=${seqLen}, dim=${dim}). Model/dimension mismatch?`,
|
|
115
|
+
);
|
|
116
|
+
}
|
|
111
117
|
const pooled = new Array(dim).fill(0);
|
|
112
118
|
let maskSum = 0;
|
|
113
119
|
for (let t = 0; t < seqLen; t++) {
|
|
@@ -7,12 +7,14 @@ import {
|
|
|
7
7
|
hybridRRF,
|
|
8
8
|
topByRRF,
|
|
9
9
|
knnSearch,
|
|
10
|
-
|
|
10
|
+
batchedQuery,
|
|
11
|
+
SQLITE_BATCH_SIZE,
|
|
12
|
+
} from "./sqlite-utils";
|
|
11
13
|
import {
|
|
12
14
|
type Memory,
|
|
13
15
|
type HybridRow,
|
|
14
16
|
DELETED_TOMBSTONE,
|
|
15
|
-
} from "./memory
|
|
17
|
+
} from "./memory";
|
|
16
18
|
|
|
17
19
|
export class MemoryRepository {
|
|
18
20
|
constructor(private db: Database) {}
|
|
@@ -144,14 +146,16 @@ export class MemoryRepository {
|
|
|
144
146
|
async findByIds(ids: string[]): Promise<Memory[]> {
|
|
145
147
|
if (ids.length === 0) return [];
|
|
146
148
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
149
|
+
return batchedQuery(this.db, ids, (batch) => {
|
|
150
|
+
const placeholders = batch.map(() => "?").join(", ");
|
|
151
|
+
const rows = this.db
|
|
152
|
+
.prepare(`SELECT * FROM memories WHERE id IN (${placeholders})`)
|
|
153
|
+
.all(...batch) as Array<Record<string, unknown>>;
|
|
151
154
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
+
return rows.map((row) => {
|
|
156
|
+
const embedding = this.getEmbedding(row.id as string);
|
|
157
|
+
return this.rowToMemory(row, embedding);
|
|
158
|
+
});
|
|
155
159
|
});
|
|
156
160
|
}
|
|
157
161
|
|
|
@@ -165,6 +169,28 @@ export class MemoryRepository {
|
|
|
165
169
|
return result.changes > 0;
|
|
166
170
|
}
|
|
167
171
|
|
|
172
|
+
/**
|
|
173
|
+
* Increment access_count and update last_accessed for multiple memories in batch.
|
|
174
|
+
* Uses batched IN clauses to stay within SQLite parameter limits.
|
|
175
|
+
*/
|
|
176
|
+
bulkUpdateAccess(ids: string[], now: Date): void {
|
|
177
|
+
if (ids.length === 0) return;
|
|
178
|
+
const ts = now.getTime();
|
|
179
|
+
|
|
180
|
+
const runBatch = (batch: string[]) => {
|
|
181
|
+
const placeholders = batch.map(() => "?").join(", ");
|
|
182
|
+
this.db
|
|
183
|
+
.prepare(
|
|
184
|
+
`UPDATE memories SET access_count = access_count + 1, last_accessed = ? WHERE id IN (${placeholders})`
|
|
185
|
+
)
|
|
186
|
+
.run(ts, ...batch);
|
|
187
|
+
};
|
|
188
|
+
|
|
189
|
+
for (let i = 0; i < ids.length; i += SQLITE_BATCH_SIZE) {
|
|
190
|
+
runBatch(ids.slice(i, i + SQLITE_BATCH_SIZE));
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
168
194
|
/**
|
|
169
195
|
* Hybrid search combining vector KNN and FTS5, fused with Reciprocal Rank Fusion.
|
|
170
196
|
*/
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { randomUUID, createHash } from "crypto";
|
|
2
|
-
import type { Memory, SearchIntent, IntentProfile, HybridRow } from "./memory
|
|
3
|
-
import { isDeleted } from "./memory
|
|
4
|
-
import type { SearchResult, SearchOptions } from "./conversation
|
|
5
|
-
import type { MemoryRepository } from "./memory.repository
|
|
6
|
-
import type { EmbeddingsService } from "./embeddings.service
|
|
7
|
-
import type { ConversationHistoryService } from "./conversation.service
|
|
2
|
+
import type { Memory, SearchIntent, IntentProfile, HybridRow } from "./memory";
|
|
3
|
+
import { isDeleted } from "./memory";
|
|
4
|
+
import type { SearchResult, SearchOptions } from "./conversation";
|
|
5
|
+
import type { MemoryRepository } from "./memory.repository";
|
|
6
|
+
import type { EmbeddingsService } from "./embeddings.service";
|
|
7
|
+
import type { ConversationHistoryService } from "./conversation.service";
|
|
8
8
|
|
|
9
9
|
const INTENT_PROFILES: Record<SearchIntent, IntentProfile> = {
|
|
10
10
|
continuity: { weights: { relevance: 0.3, recency: 0.5, utility: 0.2 }, jitter: 0.02 },
|
|
@@ -87,19 +87,10 @@ export class MemoryService {
|
|
|
87
87
|
async getMultiple(ids: string[]): Promise<Memory[]> {
|
|
88
88
|
if (ids.length === 0) return [];
|
|
89
89
|
const memories = await this.repository.findByIds(ids);
|
|
90
|
-
// Track access in bulk
|
|
91
90
|
const now = new Date();
|
|
92
|
-
const
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
this.repository.upsert({
|
|
96
|
-
...m,
|
|
97
|
-
accessCount: m.accessCount + 1,
|
|
98
|
-
lastAccessed: now,
|
|
99
|
-
})
|
|
100
|
-
)
|
|
101
|
-
);
|
|
102
|
-
return live;
|
|
91
|
+
const liveIds = memories.filter((m) => !isDeleted(m)).map((m) => m.id);
|
|
92
|
+
this.repository.bulkUpdateAccess(liveIds, now);
|
|
93
|
+
return memories.filter((m) => !isDeleted(m));
|
|
103
94
|
}
|
|
104
95
|
|
|
105
96
|
async delete(id: string): Promise<boolean> {
|
|
@@ -186,10 +177,10 @@ export class MemoryService {
|
|
|
186
177
|
async search(
|
|
187
178
|
query: string,
|
|
188
179
|
intent: SearchIntent,
|
|
189
|
-
limit: number = 10,
|
|
190
|
-
includeDeleted: boolean = false,
|
|
191
180
|
options?: SearchOptions
|
|
192
181
|
): Promise<SearchResult[]> {
|
|
182
|
+
const limit = options?.limit ?? 10;
|
|
183
|
+
const includeDeleted = options?.includeDeleted ?? false;
|
|
193
184
|
const queryEmbedding = await this.embeddings.embed(query);
|
|
194
185
|
const profile = INTENT_PROFILES[intent];
|
|
195
186
|
const now = new Date();
|
|
@@ -272,19 +263,7 @@ export class MemoryService {
|
|
|
272
263
|
|
|
273
264
|
async trackAccess(ids: string[]): Promise<void> {
|
|
274
265
|
if (ids.length === 0) return;
|
|
275
|
-
|
|
276
|
-
const now = new Date();
|
|
277
|
-
await Promise.all(
|
|
278
|
-
memories
|
|
279
|
-
.filter((m) => !isDeleted(m))
|
|
280
|
-
.map((m) =>
|
|
281
|
-
this.repository.upsert({
|
|
282
|
-
...m,
|
|
283
|
-
accessCount: m.accessCount + 1,
|
|
284
|
-
lastAccessed: now,
|
|
285
|
-
})
|
|
286
|
-
)
|
|
287
|
-
);
|
|
266
|
+
this.repository.bulkUpdateAccess(ids, new Date());
|
|
288
267
|
}
|
|
289
268
|
|
|
290
269
|
private static readonly UUID_ZERO =
|
|
@@ -292,8 +271,16 @@ export class MemoryService {
|
|
|
292
271
|
|
|
293
272
|
private static waypointId(project?: string): string {
|
|
294
273
|
if (!project?.length) return MemoryService.UUID_ZERO;
|
|
295
|
-
const
|
|
296
|
-
|
|
274
|
+
const normalized = project.trim().toLowerCase();
|
|
275
|
+
const hex = createHash("sha256").update(`waypoint:${normalized}`).digest("hex");
|
|
276
|
+
return `wp:${hex.slice(0, 32)}`;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** Legacy UUID-formatted waypoint ID for migration fallback reads. */
|
|
280
|
+
private static legacyWaypointId(project?: string): string | null {
|
|
281
|
+
if (!project?.length) return null; // UUID_ZERO is still current for no-project
|
|
282
|
+
const normalized = project.trim().toLowerCase();
|
|
283
|
+
const hex = createHash("sha256").update(`waypoint:${normalized}`).digest("hex");
|
|
297
284
|
return [
|
|
298
285
|
hex.slice(0, 8),
|
|
299
286
|
hex.slice(8, 12),
|
|
@@ -386,6 +373,20 @@ ${list(args.memory_ids)}`;
|
|
|
386
373
|
}
|
|
387
374
|
|
|
388
375
|
async getLatestWaypoint(project?: string): Promise<Memory | null> {
|
|
389
|
-
|
|
376
|
+
const waypoint = await this.get(MemoryService.waypointId(project));
|
|
377
|
+
if (waypoint) return waypoint;
|
|
378
|
+
|
|
379
|
+
// Fallback: try legacy UUID-formatted waypoint ID and migrate on read
|
|
380
|
+
const legacyId = MemoryService.legacyWaypointId(project);
|
|
381
|
+
if (!legacyId) return null;
|
|
382
|
+
|
|
383
|
+
const legacy = await this.repository.findById(legacyId);
|
|
384
|
+
if (!legacy) return null;
|
|
385
|
+
|
|
386
|
+
// Migrate: write under new ID, delete old
|
|
387
|
+
const newId = MemoryService.waypointId(project);
|
|
388
|
+
await this.repository.upsert({ ...legacy, id: newId });
|
|
389
|
+
await this.repository.markDeleted(legacyId);
|
|
390
|
+
return { ...legacy, id: newId };
|
|
390
391
|
}
|
|
391
392
|
}
|
|
@@ -3,9 +3,9 @@ import { createHash } from "crypto";
|
|
|
3
3
|
import { existsSync, statSync, readdirSync } from "fs";
|
|
4
4
|
import { resolve, dirname } from "path";
|
|
5
5
|
import { fileURLToPath } from "url";
|
|
6
|
-
import { serializeVector } from "./sqlite-utils
|
|
7
|
-
import type { MemoryRepository } from "./memory.repository
|
|
8
|
-
import type { EmbeddingsService } from "./embeddings.service
|
|
6
|
+
import { serializeVector } from "./sqlite-utils";
|
|
7
|
+
import type { MemoryRepository } from "./memory.repository";
|
|
8
|
+
import type { EmbeddingsService } from "./embeddings.service";
|
|
9
9
|
|
|
10
10
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
11
11
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
|
-
import type { EmbeddingsService } from "./embeddings.service
|
|
3
|
-
import { serializeVector } from "./sqlite-utils
|
|
2
|
+
import type { EmbeddingsService } from "./embeddings.service";
|
|
3
|
+
import { serializeVector } from "./sqlite-utils";
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Pre-migration step: remove vec0 virtual table entries from sqlite_master
|
|
@@ -127,41 +127,23 @@ export async function backfillVectors(
|
|
|
127
127
|
db: Database,
|
|
128
128
|
embeddings: EmbeddingsService,
|
|
129
129
|
): Promise<void> {
|
|
130
|
-
//
|
|
131
|
-
const
|
|
132
|
-
|
|
133
|
-
.get();
|
|
134
|
-
const memoriesExist = db.prepare("SELECT 1 FROM memories LIMIT 1").get();
|
|
135
|
-
const convosExist = db.prepare("SELECT 1 FROM conversation_history LIMIT 1").get();
|
|
130
|
+
// Quick gap check: if no rows are missing vectors, skip the expensive backfill
|
|
131
|
+
const hasMemories = db.prepare("SELECT 1 FROM memories LIMIT 1").get();
|
|
132
|
+
const hasConvos = db.prepare("SELECT 1 FROM conversation_history LIMIT 1").get();
|
|
136
133
|
|
|
137
|
-
|
|
138
|
-
// Only run the expensive LEFT JOIN when there's reason to suspect gaps.
|
|
139
|
-
const convoSentinel = db
|
|
140
|
-
.prepare("SELECT 1 FROM conversation_history_vec LIMIT 1")
|
|
141
|
-
.get();
|
|
142
|
-
const mayNeedMemoryBackfill = memoriesExist && !sentinel;
|
|
143
|
-
const mayNeedConvoBackfill = convosExist && !convoSentinel;
|
|
134
|
+
if (!hasMemories && !hasConvos) return;
|
|
144
135
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
).get();
|
|
157
|
-
if (!convoGap) return;
|
|
158
|
-
} else if (!gap && !convosExist) {
|
|
159
|
-
return;
|
|
160
|
-
}
|
|
161
|
-
} else {
|
|
162
|
-
return; // No data at all
|
|
163
|
-
}
|
|
164
|
-
}
|
|
136
|
+
const memoryGap = hasMemories && db.prepare(
|
|
137
|
+
`SELECT 1 FROM memories m LEFT JOIN memories_vec v ON m.id = v.id
|
|
138
|
+
WHERE v.id IS NULL OR length(v.vector) = 0 LIMIT 1`,
|
|
139
|
+
).get();
|
|
140
|
+
|
|
141
|
+
const convoGap = hasConvos && db.prepare(
|
|
142
|
+
`SELECT 1 FROM conversation_history c LEFT JOIN conversation_history_vec v ON c.id = v.id
|
|
143
|
+
WHERE v.id IS NULL OR length(v.vector) = 0 LIMIT 1`,
|
|
144
|
+
).get();
|
|
145
|
+
|
|
146
|
+
if (!memoryGap && !convoGap) return;
|
|
165
147
|
|
|
166
148
|
// ── Memories ──────────────────────────────────────────────────────
|
|
167
149
|
const missingMemories = db
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { readFile, readdir, stat } from "fs/promises";
|
|
2
2
|
import { basename, dirname, join } from "path";
|
|
3
|
-
import type { ParsedMessage, SessionFileInfo } from "../conversation
|
|
4
|
-
import type { SessionLogParser } from "./types
|
|
3
|
+
import type { ParsedMessage, SessionFileInfo } from "../conversation";
|
|
4
|
+
import type { SessionLogParser } from "./types";
|
|
5
5
|
|
|
6
6
|
// UUID pattern for session IDs
|
|
7
7
|
const UUID_PATTERN =
|
|
@@ -45,7 +45,7 @@ export class ClaudeCodeSessionParser implements SessionLogParser {
|
|
|
45
45
|
const fileName = basename(filePath, ".jsonl");
|
|
46
46
|
const parentDir = basename(dirname(filePath));
|
|
47
47
|
// Check if this is inside a subagents directory
|
|
48
|
-
const isSubagentFile =
|
|
48
|
+
const isSubagentFile = /[/\\]subagents[/\\]/.test(filePath);
|
|
49
49
|
|
|
50
50
|
// For subagent files, project dir is 3 levels up: <project>/<session>/subagents/<file>
|
|
51
51
|
// For main files, project dir is direct parent
|
|
@@ -3,6 +3,28 @@ import type { Database } from "bun:sqlite";
|
|
|
3
3
|
/** RRF constant matching the previous LanceDB reranker default */
|
|
4
4
|
export const RRF_K = 60;
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* Maximum parameters per SQLite query to stay within SQLITE_MAX_VARIABLE_NUMBER.
|
|
8
|
+
*/
|
|
9
|
+
export const SQLITE_BATCH_SIZE = 100;
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Execute a query in batches when the number of parameters exceeds SQLITE_BATCH_SIZE.
|
|
13
|
+
* Splits the ids array and concatenates results.
|
|
14
|
+
*/
|
|
15
|
+
export function batchedQuery<T>(
|
|
16
|
+
db: Database,
|
|
17
|
+
ids: string[],
|
|
18
|
+
queryFn: (batch: string[]) => T[]
|
|
19
|
+
): T[] {
|
|
20
|
+
if (ids.length <= SQLITE_BATCH_SIZE) return queryFn(ids);
|
|
21
|
+
const results: T[] = [];
|
|
22
|
+
for (let i = 0; i < ids.length; i += SQLITE_BATCH_SIZE) {
|
|
23
|
+
results.push(...queryFn(ids.slice(i, i + SQLITE_BATCH_SIZE)));
|
|
24
|
+
}
|
|
25
|
+
return results;
|
|
26
|
+
}
|
|
27
|
+
|
|
6
28
|
/**
|
|
7
29
|
* Serialize a number[] embedding to raw float32 bytes for BLOB storage.
|
|
8
30
|
*/
|
package/server/index.ts
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
|
|
3
|
-
import { loadConfig, parseCliArgs } from "./config/index
|
|
4
|
-
import { connectToDatabase } from "./core/connection
|
|
5
|
-
import { backfillVectors } from "./core/migrations
|
|
6
|
-
import { MemoryRepository } from "./core/memory.repository
|
|
7
|
-
import { ConversationRepository } from "./core/conversation.repository
|
|
8
|
-
import { EmbeddingsService } from "./core/embeddings.service
|
|
9
|
-
import { MemoryService } from "./core/memory.service
|
|
10
|
-
import { ConversationHistoryService } from "./core/conversation.service
|
|
11
|
-
import { startServer } from "./transports/mcp/server
|
|
12
|
-
import { startHttpServer } from "./transports/http/server
|
|
3
|
+
import { loadConfig, parseCliArgs } from "./config/index";
|
|
4
|
+
import { connectToDatabase } from "./core/connection";
|
|
5
|
+
import { backfillVectors } from "./core/migrations";
|
|
6
|
+
import { MemoryRepository } from "./core/memory.repository";
|
|
7
|
+
import { ConversationRepository } from "./core/conversation.repository";
|
|
8
|
+
import { EmbeddingsService } from "./core/embeddings.service";
|
|
9
|
+
import { MemoryService } from "./core/memory.service";
|
|
10
|
+
import { ConversationHistoryService } from "./core/conversation.service";
|
|
11
|
+
import { startServer } from "./transports/mcp/server";
|
|
12
|
+
import { startHttpServer } from "./transports/http/server";
|
|
13
13
|
|
|
14
14
|
async function main(): Promise<void> {
|
|
15
15
|
const args = process.argv.slice(2);
|
|
@@ -21,11 +21,11 @@ import {
|
|
|
21
21
|
} from "@modelcontextprotocol/sdk/types.js";
|
|
22
22
|
import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
|
|
23
23
|
|
|
24
|
-
import { tools } from "../mcp/tools
|
|
25
|
-
import { handleToolCall } from "../mcp/handlers
|
|
26
|
-
import { SERVER_INSTRUCTIONS } from "../mcp/server
|
|
27
|
-
import { VERSION } from "../../config/index
|
|
28
|
-
import type { MemoryService } from "../../core/memory.service
|
|
24
|
+
import { tools } from "../mcp/tools";
|
|
25
|
+
import { handleToolCall } from "../mcp/handlers";
|
|
26
|
+
import { SERVER_INSTRUCTIONS } from "../mcp/server";
|
|
27
|
+
import { VERSION } from "../../config/index";
|
|
28
|
+
import type { MemoryService } from "../../core/memory.service";
|
|
29
29
|
|
|
30
30
|
interface Session {
|
|
31
31
|
server: Server;
|
|
@@ -3,11 +3,11 @@ import { cors } from "hono/cors";
|
|
|
3
3
|
import { createServer } from "net";
|
|
4
4
|
import { writeFileSync, mkdirSync, unlinkSync } from "fs";
|
|
5
5
|
import { join } from "path";
|
|
6
|
-
import type { MemoryService } from "../../core/memory.service
|
|
7
|
-
import type { Config } from "../../config/index
|
|
8
|
-
import { isDeleted } from "../../core/memory
|
|
9
|
-
import { createMcpRoutes } from "./mcp-transport
|
|
10
|
-
import type { Memory, SearchIntent } from "../../core/memory
|
|
6
|
+
import type { MemoryService } from "../../core/memory.service";
|
|
7
|
+
import type { Config } from "../../config/index";
|
|
8
|
+
import { isDeleted } from "../../core/memory";
|
|
9
|
+
import { createMcpRoutes } from "./mcp-transport";
|
|
10
|
+
import type { Memory, SearchIntent } from "../../core/memory";
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
/**
|
|
@@ -139,7 +139,7 @@ export function createHttpApp(memoryService: MemoryService, config: Config): Hon
|
|
|
139
139
|
return c.json({ error: "Missing or invalid 'query' field" }, 400);
|
|
140
140
|
}
|
|
141
141
|
|
|
142
|
-
const results = await memoryService.search(query, intent, limit);
|
|
142
|
+
const results = await memoryService.search(query, intent, { limit });
|
|
143
143
|
|
|
144
144
|
return c.json({
|
|
145
145
|
results: results.map((r) => ({
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
|
|
2
|
-
import type { MemoryService } from "../../core/memory.service
|
|
3
|
-
import type { ConversationHistoryService } from "../../core/conversation.service
|
|
4
|
-
import type { SearchIntent } from "../../core/memory
|
|
5
|
-
import type { HistoryFilters, SearchResult } from "../../core/conversation
|
|
6
|
-
import { DEBUG } from "../../config/index
|
|
2
|
+
import type { MemoryService } from "../../core/memory.service";
|
|
3
|
+
import type { ConversationHistoryService } from "../../core/conversation.service";
|
|
4
|
+
import type { SearchIntent } from "../../core/memory";
|
|
5
|
+
import type { HistoryFilters, SearchResult } from "../../core/conversation";
|
|
6
|
+
import { DEBUG } from "../../config/index";
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
9
|
* Safely coerce a tool argument to an array. Handles the case where the MCP
|
|
@@ -51,6 +51,14 @@ function parseDate(value: unknown, fieldName: string): Date | undefined {
|
|
|
51
51
|
return date;
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
+
function requireString(args: Record<string, unknown> | undefined, field: string): string {
|
|
55
|
+
const value = args?.[field];
|
|
56
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
57
|
+
throw new Error(`${field} is required`);
|
|
58
|
+
}
|
|
59
|
+
return value;
|
|
60
|
+
}
|
|
61
|
+
|
|
54
62
|
export async function handleStoreMemories(
|
|
55
63
|
args: Record<string, unknown> | undefined,
|
|
56
64
|
service: MemoryService
|
|
@@ -189,7 +197,9 @@ export async function handleSearchMemories(
|
|
|
189
197
|
return errorResult(errorText(e));
|
|
190
198
|
}
|
|
191
199
|
|
|
192
|
-
const results = await service.search(query, intent,
|
|
200
|
+
const results = await service.search(query, intent, {
|
|
201
|
+
limit,
|
|
202
|
+
includeDeleted,
|
|
193
203
|
includeHistory,
|
|
194
204
|
historyOnly,
|
|
195
205
|
historyFilters,
|
|
@@ -202,19 +212,7 @@ export async function handleSearchMemories(
|
|
|
202
212
|
};
|
|
203
213
|
}
|
|
204
214
|
|
|
205
|
-
const formatted = results.map((r
|
|
206
|
-
let result = `[${r.source}] ID: ${r.id}\nContent: ${r.content}`;
|
|
207
|
-
if (r.metadata && Object.keys(r.metadata).length > 0) {
|
|
208
|
-
result += `\nMetadata: ${JSON.stringify(r.metadata)}`;
|
|
209
|
-
}
|
|
210
|
-
if (r.source === "memory" && includeDeleted && r.supersededBy) {
|
|
211
|
-
result += `\n[DELETED]`;
|
|
212
|
-
}
|
|
213
|
-
if (r.source === "conversation_history" && r.sessionId) {
|
|
214
|
-
result += `\nSession: ${r.sessionId}`;
|
|
215
|
-
}
|
|
216
|
-
return result;
|
|
217
|
-
});
|
|
215
|
+
const formatted = results.map((r) => formatSearchResult(r, includeDeleted));
|
|
218
216
|
|
|
219
217
|
return {
|
|
220
218
|
content: [{ type: "text", text: formatted.join("\n\n---\n\n") }],
|
|
@@ -241,6 +239,20 @@ function formatMemoryDetail(
|
|
|
241
239
|
return result;
|
|
242
240
|
}
|
|
243
241
|
|
|
242
|
+
function formatSearchResult(r: SearchResult, includeDeleted: boolean): string {
|
|
243
|
+
let result = `[${r.source}] ID: ${r.id}\nContent: ${r.content}`;
|
|
244
|
+
if (r.metadata && Object.keys(r.metadata).length > 0) {
|
|
245
|
+
result += `\nMetadata: ${JSON.stringify(r.metadata)}`;
|
|
246
|
+
}
|
|
247
|
+
if (r.source === "memory" && includeDeleted && r.supersededBy) {
|
|
248
|
+
result += `\n[DELETED]`;
|
|
249
|
+
}
|
|
250
|
+
if (r.source === "conversation_history" && r.sessionId) {
|
|
251
|
+
result += `\nSession: ${r.sessionId}`;
|
|
252
|
+
}
|
|
253
|
+
return result;
|
|
254
|
+
}
|
|
255
|
+
|
|
244
256
|
export async function handleGetMemories(
|
|
245
257
|
args: Record<string, unknown> | undefined,
|
|
246
258
|
service: MemoryService
|
|
@@ -267,8 +279,11 @@ export async function handleReportMemoryUsefulness(
|
|
|
267
279
|
args: Record<string, unknown> | undefined,
|
|
268
280
|
service: MemoryService
|
|
269
281
|
): Promise<CallToolResult> {
|
|
270
|
-
const memoryId = args
|
|
271
|
-
const useful = args?.useful
|
|
282
|
+
const memoryId = requireString(args, "memory_id");
|
|
283
|
+
const useful = args?.useful;
|
|
284
|
+
if (typeof useful !== "boolean") {
|
|
285
|
+
return errorResult("useful is required and must be a boolean");
|
|
286
|
+
}
|
|
272
287
|
|
|
273
288
|
const memory = await service.vote(memoryId, useful ? 1 : -1);
|
|
274
289
|
|
|
@@ -290,10 +305,19 @@ export async function handleSetWaypoint(
|
|
|
290
305
|
args: Record<string, unknown> | undefined,
|
|
291
306
|
service: MemoryService
|
|
292
307
|
): Promise<CallToolResult> {
|
|
308
|
+
let project: string;
|
|
309
|
+
let summary: string;
|
|
310
|
+
try {
|
|
311
|
+
project = requireString(args, "project");
|
|
312
|
+
summary = requireString(args, "summary");
|
|
313
|
+
} catch (e) {
|
|
314
|
+
return errorResult(errorText(e));
|
|
315
|
+
}
|
|
316
|
+
|
|
293
317
|
const memory = await service.setWaypoint({
|
|
294
|
-
project
|
|
318
|
+
project,
|
|
295
319
|
branch: args?.branch as string | undefined,
|
|
296
|
-
summary
|
|
320
|
+
summary,
|
|
297
321
|
completed: (args?.completed as string[] | undefined) ?? [],
|
|
298
322
|
in_progress_blocked: (args?.in_progress_blocked as string[] | undefined) ?? [],
|
|
299
323
|
key_decisions: (args?.key_decisions as string[] | undefined) ?? [],
|
|
@@ -6,12 +6,12 @@ import {
|
|
|
6
6
|
ListResourcesRequestSchema,
|
|
7
7
|
ReadResourceRequestSchema,
|
|
8
8
|
} from "@modelcontextprotocol/sdk/types.js";
|
|
9
|
-
import { resources, readResource } from "./resources
|
|
9
|
+
import { resources, readResource } from "./resources";
|
|
10
10
|
|
|
11
|
-
import { tools } from "./tools
|
|
12
|
-
import { handleToolCall } from "./handlers
|
|
13
|
-
import type { MemoryService } from "../../core/memory.service
|
|
14
|
-
import { VERSION } from "../../config/index
|
|
11
|
+
import { tools } from "./tools";
|
|
12
|
+
import { handleToolCall } from "./handlers";
|
|
13
|
+
import type { MemoryService } from "../../core/memory.service";
|
|
14
|
+
import { VERSION } from "../../config/index";
|
|
15
15
|
|
|
16
16
|
export const SERVER_INSTRUCTIONS = `This server is the user's canonical memory system. It provides persistent, semantic vector memory that survives across conversations and sessions.
|
|
17
17
|
|