@aeriondyseti/vector-memory-mcp 1.0.2-dev.0 → 1.1.0-dev.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +4 -4
- package/dist/src/config/index.d.ts +10 -0
- package/dist/src/config/index.d.ts.map +1 -1
- package/dist/src/config/index.js +13 -0
- package/dist/src/config/index.js.map +1 -1
- package/dist/src/db/conversation-history.repository.d.ts +24 -0
- package/dist/src/db/conversation-history.repository.d.ts.map +1 -0
- package/dist/src/db/conversation-history.repository.js +184 -0
- package/dist/src/db/conversation-history.repository.js.map +1 -0
- package/dist/src/db/conversation-history.schema.d.ts +10 -0
- package/dist/src/db/conversation-history.schema.d.ts.map +1 -0
- package/dist/src/db/conversation-history.schema.js +31 -0
- package/dist/src/db/conversation-history.schema.js.map +1 -0
- package/dist/src/db/lancedb-utils.d.ts +35 -0
- package/dist/src/db/lancedb-utils.d.ts.map +1 -0
- package/dist/src/db/lancedb-utils.js +77 -0
- package/dist/src/db/lancedb-utils.js.map +1 -0
- package/dist/src/db/memory.repository.d.ts +2 -12
- package/dist/src/db/memory.repository.d.ts.map +1 -1
- package/dist/src/db/memory.repository.js +13 -56
- package/dist/src/db/memory.repository.js.map +1 -1
- package/dist/src/db/schema.d.ts +4 -1
- package/dist/src/db/schema.d.ts.map +1 -1
- package/dist/src/db/schema.js +8 -4
- package/dist/src/db/schema.js.map +1 -1
- package/dist/src/index.js +8 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/mcp/handlers.d.ts +3 -0
- package/dist/src/mcp/handlers.d.ts.map +1 -1
- package/dist/src/mcp/handlers.js +149 -17
- package/dist/src/mcp/handlers.js.map +1 -1
- package/dist/src/mcp/tools.d.ts +3 -0
- package/dist/src/mcp/tools.d.ts.map +1 -1
- package/dist/src/mcp/tools.js +54 -3
- package/dist/src/mcp/tools.js.map +1 -1
- package/dist/src/services/conversation-history.service.d.ts +64 -0
- package/dist/src/services/conversation-history.service.d.ts.map +1 -0
- package/dist/src/services/conversation-history.service.js +244 -0
- package/dist/src/services/conversation-history.service.js.map +1 -0
- package/dist/src/services/memory.service.d.ts +24 -0
- package/dist/src/services/memory.service.d.ts.map +1 -1
- package/dist/src/services/memory.service.js +58 -0
- package/dist/src/services/memory.service.js.map +1 -1
- package/dist/src/services/session-parser.d.ts +59 -0
- package/dist/src/services/session-parser.d.ts.map +1 -0
- package/dist/src/services/session-parser.js +147 -0
- package/dist/src/services/session-parser.js.map +1 -0
- package/dist/src/types/conversation-history.d.ts +74 -0
- package/dist/src/types/conversation-history.d.ts.map +1 -0
- package/dist/src/types/conversation-history.js +2 -0
- package/dist/src/types/conversation-history.js.map +1 -0
- package/dist/src/types/memory.d.ts +4 -2
- package/dist/src/types/memory.d.ts.map +1 -1
- package/package.json +4 -4
- package/src/config/index.ts +23 -0
- package/src/db/conversation-history.repository.ts +255 -0
- package/src/db/conversation-history.schema.ts +40 -0
- package/src/db/lancedb-utils.ts +97 -0
- package/src/db/memory.repository.ts +18 -67
- package/src/db/schema.ts +17 -21
- package/src/index.ts +16 -0
- package/src/mcp/handlers.ts +178 -22
- package/src/mcp/tools.ts +66 -3
- package/src/services/conversation-history.service.ts +320 -0
- package/src/services/memory.service.ts +74 -0
- package/src/services/session-parser.ts +232 -0
- package/src/types/conversation-history.ts +82 -0
- package/src/types/memory.ts +4 -3
package/dist/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aeriondyseti/vector-memory-mcp",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0-dev.2",
|
|
4
4
|
"description": "A zero-configuration RAG memory server for MCP clients",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/src/index.js",
|
|
@@ -17,13 +17,13 @@
|
|
|
17
17
|
],
|
|
18
18
|
"repository": {
|
|
19
19
|
"type": "git",
|
|
20
|
-
"url": "
|
|
20
|
+
"url": "https://github.com/aeriondyseti/vector-memory-mcp"
|
|
21
21
|
},
|
|
22
22
|
"author": "AerionDyseti",
|
|
23
23
|
"bugs": {
|
|
24
|
-
"url": "https://github.com/
|
|
24
|
+
"url": "https://github.com/aeriondyseti/vector-memory-mcp/issues"
|
|
25
25
|
},
|
|
26
|
-
"homepage": "https://github.com/
|
|
26
|
+
"homepage": "https://github.com/aeriondyseti/vector-memory-mcp#readme",
|
|
27
27
|
"scripts": {
|
|
28
28
|
"start": "node dist/src/index.js",
|
|
29
29
|
"start:bun": "bun run src/index.ts",
|
|
@@ -8,12 +8,22 @@ export interface Config {
|
|
|
8
8
|
httpHost: string;
|
|
9
9
|
enableHttp: boolean;
|
|
10
10
|
transportMode: TransportMode;
|
|
11
|
+
conversationHistory: {
|
|
12
|
+
enabled: boolean;
|
|
13
|
+
sessionPath: string | null;
|
|
14
|
+
historyWeight: number;
|
|
15
|
+
};
|
|
11
16
|
}
|
|
12
17
|
export interface ConfigOverrides {
|
|
13
18
|
dbPath?: string;
|
|
14
19
|
httpPort?: number;
|
|
15
20
|
enableHttp?: boolean;
|
|
16
21
|
transportMode?: TransportMode;
|
|
22
|
+
conversationHistory?: {
|
|
23
|
+
enabled?: boolean;
|
|
24
|
+
sessionPath?: string;
|
|
25
|
+
historyWeight?: number;
|
|
26
|
+
};
|
|
17
27
|
}
|
|
18
28
|
export declare function loadConfig(overrides?: ConfigOverrides): Config;
|
|
19
29
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/config/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,QAAsB,CAAC;AAE3C,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;AAEtD,MAAM,WAAW,MAAM;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,OAAO,CAAC;IACpB,aAAa,EAAE,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/config/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,QAAsB,CAAC;AAE3C,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;AAEtD,MAAM,WAAW,MAAM;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,OAAO,CAAC;IACpB,aAAa,EAAE,aAAa,CAAC;IAC7B,mBAAmB,EAAE;QACnB,OAAO,EAAE,OAAO,CAAC;QACjB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;CACH;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,mBAAmB,CAAC,EAAE;QACpB,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,aAAa,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;CACH;AAaD,wBAAgB,UAAU,CAAC,SAAS,GAAE,eAAoB,GAAG,MAAM,CAmBlE;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,eAAe,CA2B5D;AAGD,eAAO,MAAM,MAAM,QAAe,CAAC"}
|
package/dist/src/config/index.js
CHANGED
|
@@ -23,6 +23,11 @@ export function loadConfig(overrides = {}) {
|
|
|
23
23
|
httpHost: DEFAULT_HTTP_HOST,
|
|
24
24
|
enableHttp,
|
|
25
25
|
transportMode,
|
|
26
|
+
conversationHistory: {
|
|
27
|
+
enabled: overrides.conversationHistory?.enabled ?? false,
|
|
28
|
+
sessionPath: overrides.conversationHistory?.sessionPath ?? null,
|
|
29
|
+
historyWeight: overrides.conversationHistory?.historyWeight ?? 0.5,
|
|
30
|
+
},
|
|
26
31
|
};
|
|
27
32
|
}
|
|
28
33
|
/**
|
|
@@ -33,6 +38,8 @@ export function parseCliArgs(argv) {
|
|
|
33
38
|
"--db-file": String,
|
|
34
39
|
"--port": Number,
|
|
35
40
|
"--no-http": Boolean,
|
|
41
|
+
"--conversation-history": Boolean,
|
|
42
|
+
"--session-path": String,
|
|
36
43
|
// Aliases
|
|
37
44
|
"-d": "--db-file",
|
|
38
45
|
"-p": "--port",
|
|
@@ -41,6 +48,12 @@ export function parseCliArgs(argv) {
|
|
|
41
48
|
dbPath: args["--db-file"],
|
|
42
49
|
httpPort: args["--port"],
|
|
43
50
|
enableHttp: args["--no-http"] ? false : undefined,
|
|
51
|
+
conversationHistory: args["--conversation-history"] || args["--session-path"]
|
|
52
|
+
? {
|
|
53
|
+
enabled: args["--conversation-history"],
|
|
54
|
+
sessionPath: args["--session-path"],
|
|
55
|
+
}
|
|
56
|
+
: undefined,
|
|
44
57
|
};
|
|
45
58
|
}
|
|
46
59
|
// Default config for imports that don't use CLI args
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/config/index.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AACxC,OAAO,WAAW,MAAM,oBAAoB,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,CAAC;AAEnE,MAAM,CAAC,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/config/index.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AACxC,OAAO,WAAW,MAAM,oBAAoB,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,CAAC;AAEnE,MAAM,CAAC,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC;AA+B3C,yDAAyD;AACzD,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,gBAAgB,EAAE,aAAa,CAAC,CAAC;AAC7E,MAAM,uBAAuB,GAAG,yBAAyB,CAAC;AAC1D,MAAM,2BAA2B,GAAG,GAAG,CAAC;AACxC,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAC/B,MAAM,iBAAiB,GAAG,WAAW,CAAC;AAEtC,SAAS,WAAW,CAAC,IAAY;IAC/B,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,YAA6B,EAAE;IACxD,MAAM,aAAa,GAAG,SAAS,CAAC,aAAa,IAAI,OAAO,CAAC;IACzD,yEAAyE;IACzE,MAAM,UAAU,GAAG,SAAS,CAAC,UAAU,IAAI,IAAI,CAAC;IAEhD,OAAO;QACL,MAAM,EAAE,WAAW,CAAC,SAAS,CAAC,MAAM,IAAI,eAAe,CAAC;QACxD,cAAc,EAAE,uBAAuB;QACvC,kBAAkB,EAAE,2BAA2B;QAC/C,QAAQ,EAAE,SAAS,CAAC,QAAQ,IAAI,iBAAiB;QACjD,QAAQ,EAAE,iBAAiB;QAC3B,UAAU;QACV,aAAa;QACb,mBAAmB,EAAE;YACnB,OAAO,EAAE,SAAS,CAAC,mBAAmB,EAAE,OAAO,IAAI,KAAK;YACxD,WAAW,EAAE,SAAS,CAAC,mBAAmB,EAAE,WAAW,IAAI,IAAI;YAC/D,aAAa,EAAE,SAAS,CAAC,mBAAmB,EAAE,aAAa,IAAI,GAAG;SACnE;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,IAAc;IACzC,MAAM,IAAI,GAAG,GAAG,CACd;QACE,WAAW,EAAE,MAAM;QACnB,QAAQ,EAAE,MAAM;QAChB,WAAW,EAAE,OAAO;QACpB,wBAAwB,EAAE,OAAO;QACjC,gBAAgB,EAAE,MAAM;QAExB,UAAU;QACV,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,QAAQ;KACf,EACD,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,CAC3B,CAAC;IAEF,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC;QACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC;QACxB,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;QACjD,mBAAmB,EAAE,IAAI,CAAC,wBAAwB,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC;YAC3E,CAAC,CAAC;gBACE,OAAO,EAAE,IAAI,CAAC,wBAAwB,CAAC;gBACvC,WAAW,EAAE,IAAI,CAAC,gBAAgB,CAAC;aACpC;YACH,CAAC,CAAC,SAAS;KACd,CAAC;AACJ,CAAC;AAED,qDAAqD;AACrD,MAAM,CAAC,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import * as lancedb from "@lancedb/lancedb";
|
|
2
|
+
import type { ConversationHistoryEntry, ConversationHistoryHybridRow, IndexedSession, IndexedSessionSummary } from "../types/conversation-history.js";
|
|
3
|
+
export declare class ConversationHistoryRepository {
|
|
4
|
+
private db;
|
|
5
|
+
private tablePromise;
|
|
6
|
+
private sessionsTablePromise;
|
|
7
|
+
private ensureFtsIndex;
|
|
8
|
+
private getReranker;
|
|
9
|
+
constructor(db: lancedb.Connection);
|
|
10
|
+
private getTable;
|
|
11
|
+
private getSessionsTable;
|
|
12
|
+
private rowToEntry;
|
|
13
|
+
private rowToSessionSummary;
|
|
14
|
+
private rowToSession;
|
|
15
|
+
insert(entries: ConversationHistoryEntry[]): Promise<void>;
|
|
16
|
+
findHybrid(embedding: number[], query: string, limit: number): Promise<ConversationHistoryHybridRow[]>;
|
|
17
|
+
findBySessionId(sessionId: string): Promise<ConversationHistoryEntry[]>;
|
|
18
|
+
deleteBySessionId(sessionId: string): Promise<number>;
|
|
19
|
+
getIndexedSession(sessionId: string): Promise<IndexedSession | null>;
|
|
20
|
+
upsertIndexedSession(session: IndexedSession): Promise<void>;
|
|
21
|
+
listIndexedSessions(): Promise<IndexedSessionSummary[]>;
|
|
22
|
+
deleteIndexedSession(sessionId: string): Promise<boolean>;
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=conversation-history.repository.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"conversation-history.repository.d.ts","sourceRoot":"","sources":["../../../src/db/conversation-history.repository.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAe5C,OAAO,KAAK,EACV,wBAAwB,EACxB,4BAA4B,EAC5B,cAAc,EACd,qBAAqB,EAEtB,MAAM,kCAAkC,CAAC;AAE1C,qBAAa,6BAA6B;IAW5B,OAAO,CAAC,EAAE;IATtB,OAAO,CAAC,YAAY,CAA+B;IACnD,OAAO,CAAC,oBAAoB,CAA+B;IAG3D,OAAO,CAAC,cAAc,CAAsB;IAG5C,OAAO,CAAC,WAAW,CAAyB;gBAExB,EAAE,EAAE,OAAO,CAAC,UAAU;IAI1C,OAAO,CAAC,QAAQ;IAchB,OAAO,CAAC,gBAAgB;IAcxB,OAAO,CAAC,UAAU;IAclB,OAAO,CAAC,mBAAmB;IAe3B,OAAO,CAAC,YAAY;IAUd,MAAM,CAAC,OAAO,EAAE,wBAAwB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAmB1D,UAAU,CACd,SAAS,EAAE,MAAM,EAAE,EACnB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,4BAA4B,EAAE,CAAC;IAuBpC,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,wBAAwB,EAAE,CAAC;IAYvE,iBAAiB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAoBrD,iBAAiB,CACrB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAe3B,oBAAoB,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IA8B5D,mBAAmB,IAAI,OAAO,CAAC,qBAAqB,EAAE,CAAC;IASvD,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;CAehE"}
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import { CONVERSATION_HISTORY_TABLE, INDEXED_SESSIONS_TABLE, conversationHistorySchema, indexedSessionsSchema, } from "./conversation-history.schema.js";
|
|
2
|
+
import { arrowVectorToArray, getOrCreateTable, createFtsMutex, createRerankerMutex, escapeLanceDbString, } from "./lancedb-utils.js";
|
|
3
|
+
export class ConversationHistoryRepository {
|
|
4
|
+
db;
|
|
5
|
+
// Cached table handles — initialized once, retained for instance lifetime
|
|
6
|
+
tablePromise = null;
|
|
7
|
+
sessionsTablePromise = null;
|
|
8
|
+
// FTS index mutex — once created, the promise is never cleared (index persists in LanceDB)
|
|
9
|
+
ensureFtsIndex;
|
|
10
|
+
// Cached reranker — k=60 is constant, no need to recreate per search
|
|
11
|
+
getReranker = createRerankerMutex();
|
|
12
|
+
constructor(db) {
|
|
13
|
+
this.db = db;
|
|
14
|
+
this.ensureFtsIndex = createFtsMutex(() => this.getTable());
|
|
15
|
+
}
|
|
16
|
+
getTable() {
|
|
17
|
+
if (!this.tablePromise) {
|
|
18
|
+
this.tablePromise = getOrCreateTable(this.db, CONVERSATION_HISTORY_TABLE, conversationHistorySchema).catch((e) => {
|
|
19
|
+
this.tablePromise = null;
|
|
20
|
+
throw e;
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
return this.tablePromise;
|
|
24
|
+
}
|
|
25
|
+
getSessionsTable() {
|
|
26
|
+
if (!this.sessionsTablePromise) {
|
|
27
|
+
this.sessionsTablePromise = getOrCreateTable(this.db, INDEXED_SESSIONS_TABLE, indexedSessionsSchema).catch((e) => {
|
|
28
|
+
this.sessionsTablePromise = null;
|
|
29
|
+
throw e;
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
return this.sessionsTablePromise;
|
|
33
|
+
}
|
|
34
|
+
rowToEntry(row) {
|
|
35
|
+
return {
|
|
36
|
+
id: row.id,
|
|
37
|
+
content: row.content,
|
|
38
|
+
embedding: arrowVectorToArray(row.vector),
|
|
39
|
+
sessionId: row.session_id,
|
|
40
|
+
role: row.role,
|
|
41
|
+
messageIndex: row.message_index,
|
|
42
|
+
timestamp: new Date(row.timestamp),
|
|
43
|
+
metadata: JSON.parse(row.metadata),
|
|
44
|
+
createdAt: new Date(row.created_at),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
rowToSessionSummary(row) {
|
|
48
|
+
return {
|
|
49
|
+
sessionId: row.session_id,
|
|
50
|
+
messageCount: row.message_count,
|
|
51
|
+
firstMessageAt: new Date(row.first_message_at),
|
|
52
|
+
lastMessageAt: new Date(row.last_message_at),
|
|
53
|
+
indexedAt: new Date(row.indexed_at),
|
|
54
|
+
// Use null check (not truthiness) — empty string is a valid value distinct from null
|
|
55
|
+
...(row.project != null ? { project: row.project } : {}),
|
|
56
|
+
...(row.git_branch != null ? { gitBranch: row.git_branch } : {}),
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
rowToSession(row) {
|
|
60
|
+
return {
|
|
61
|
+
...this.rowToSessionSummary(row),
|
|
62
|
+
filePath: row.file_path,
|
|
63
|
+
fileSize: row.file_size,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
// --- Conversation History Operations ---
|
|
67
|
+
async insert(entries) {
|
|
68
|
+
if (entries.length === 0)
|
|
69
|
+
return;
|
|
70
|
+
const table = await this.getTable();
|
|
71
|
+
await table.add(entries.map((entry) => ({
|
|
72
|
+
id: entry.id,
|
|
73
|
+
vector: entry.embedding,
|
|
74
|
+
content: entry.content,
|
|
75
|
+
session_id: entry.sessionId,
|
|
76
|
+
role: entry.role,
|
|
77
|
+
message_index: entry.messageIndex,
|
|
78
|
+
timestamp: entry.timestamp.getTime(),
|
|
79
|
+
metadata: JSON.stringify(entry.metadata),
|
|
80
|
+
created_at: entry.createdAt.getTime(),
|
|
81
|
+
})));
|
|
82
|
+
}
|
|
83
|
+
async findHybrid(embedding, query, limit) {
|
|
84
|
+
await this.ensureFtsIndex();
|
|
85
|
+
const table = await this.getTable();
|
|
86
|
+
const reranker = await this.getReranker();
|
|
87
|
+
const results = await table
|
|
88
|
+
.query()
|
|
89
|
+
.nearestTo(embedding)
|
|
90
|
+
.fullTextSearch(query)
|
|
91
|
+
.rerank(reranker)
|
|
92
|
+
.limit(limit)
|
|
93
|
+
.toArray();
|
|
94
|
+
return results.map((row) => {
|
|
95
|
+
const entry = this.rowToEntry(row);
|
|
96
|
+
return {
|
|
97
|
+
...entry,
|
|
98
|
+
rrfScore: row._relevance_score ?? 0,
|
|
99
|
+
};
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
async findBySessionId(sessionId) {
|
|
103
|
+
const table = await this.getTable();
|
|
104
|
+
const results = await table
|
|
105
|
+
.query()
|
|
106
|
+
.where(`session_id = '${escapeLanceDbString(sessionId)}'`)
|
|
107
|
+
.toArray();
|
|
108
|
+
return results.map((row) => this.rowToEntry(row));
|
|
109
|
+
}
|
|
110
|
+
async deleteBySessionId(sessionId) {
|
|
111
|
+
const table = await this.getTable();
|
|
112
|
+
// Select only id — avoids deserializing embedding vectors just for a count
|
|
113
|
+
const existing = await table
|
|
114
|
+
.query()
|
|
115
|
+
.where(`session_id = '${escapeLanceDbString(sessionId)}'`)
|
|
116
|
+
.select(["id"])
|
|
117
|
+
.toArray();
|
|
118
|
+
const count = existing.length;
|
|
119
|
+
if (count > 0) {
|
|
120
|
+
await table.delete(`session_id = '${escapeLanceDbString(sessionId)}'`);
|
|
121
|
+
}
|
|
122
|
+
return count;
|
|
123
|
+
}
|
|
124
|
+
// --- Indexed Sessions Tracking ---
|
|
125
|
+
async getIndexedSession(sessionId) {
|
|
126
|
+
const table = await this.getSessionsTable();
|
|
127
|
+
const results = await table
|
|
128
|
+
.query()
|
|
129
|
+
.where(`session_id = '${escapeLanceDbString(sessionId)}'`)
|
|
130
|
+
.limit(1)
|
|
131
|
+
.toArray();
|
|
132
|
+
if (results.length === 0) {
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
return this.rowToSession(results[0]);
|
|
136
|
+
}
|
|
137
|
+
async upsertIndexedSession(session) {
|
|
138
|
+
const table = await this.getSessionsTable();
|
|
139
|
+
const existing = await table
|
|
140
|
+
.query()
|
|
141
|
+
.where(`session_id = '${escapeLanceDbString(session.sessionId)}'`)
|
|
142
|
+
.limit(1)
|
|
143
|
+
.toArray();
|
|
144
|
+
const row = {
|
|
145
|
+
session_id: session.sessionId,
|
|
146
|
+
file_path: session.filePath,
|
|
147
|
+
file_size: session.fileSize,
|
|
148
|
+
message_count: session.messageCount,
|
|
149
|
+
first_message_at: session.firstMessageAt.getTime(),
|
|
150
|
+
last_message_at: session.lastMessageAt.getTime(),
|
|
151
|
+
indexed_at: session.indexedAt.getTime(),
|
|
152
|
+
project: session.project ?? null,
|
|
153
|
+
git_branch: session.gitBranch ?? null,
|
|
154
|
+
};
|
|
155
|
+
if (existing.length === 0) {
|
|
156
|
+
await table.add([row]);
|
|
157
|
+
}
|
|
158
|
+
else {
|
|
159
|
+
await table.update({
|
|
160
|
+
where: `session_id = '${escapeLanceDbString(session.sessionId)}'`,
|
|
161
|
+
values: row,
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
async listIndexedSessions() {
|
|
166
|
+
const table = await this.getSessionsTable();
|
|
167
|
+
const results = await table.query().toArray();
|
|
168
|
+
return results.map((row) => this.rowToSessionSummary(row));
|
|
169
|
+
}
|
|
170
|
+
async deleteIndexedSession(sessionId) {
|
|
171
|
+
const table = await this.getSessionsTable();
|
|
172
|
+
const existing = await table
|
|
173
|
+
.query()
|
|
174
|
+
.where(`session_id = '${escapeLanceDbString(sessionId)}'`)
|
|
175
|
+
.limit(1)
|
|
176
|
+
.toArray();
|
|
177
|
+
if (existing.length === 0) {
|
|
178
|
+
return false;
|
|
179
|
+
}
|
|
180
|
+
await table.delete(`session_id = '${escapeLanceDbString(sessionId)}'`);
|
|
181
|
+
return true;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
//# sourceMappingURL=conversation-history.repository.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"conversation-history.repository.js","sourceRoot":"","sources":["../../../src/db/conversation-history.repository.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,0BAA0B,EAC1B,sBAAsB,EACtB,yBAAyB,EACzB,qBAAqB,GACtB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EACL,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,oBAAoB,CAAC;AAS5B,MAAM,OAAO,6BAA6B;IAWpB;IAVpB,0EAA0E;IAClE,YAAY,GAA0B,IAAI,CAAC;IAC3C,oBAAoB,GAA0B,IAAI,CAAC;IAE3D,2FAA2F;IACnF,cAAc,CAAsB;IAE5C,qEAAqE;IAC7D,WAAW,GAAG,mBAAmB,EAAE,CAAC;IAE5C,YAAoB,EAAsB;QAAtB,OAAE,GAAF,EAAE,CAAoB;QACxC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC9D,CAAC;IAEO,QAAQ;QACd,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,IAAI,CAAC,YAAY,GAAG,gBAAgB,CAClC,IAAI,CAAC,EAAE,EACP,0BAA0B,EAC1B,yBAAyB,CAC1B,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACZ,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;gBACzB,MAAM,CAAC,CAAC;YACV,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAEO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC/B,IAAI,CAAC,oBAAoB,GAAG,gBAAgB,CAC1C,IAAI,CAAC,EAAE,EACP,sBAAsB,EACtB,qBAAqB,CACtB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACZ,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;gBACjC,MAAM,CAAC,CAAC;YACV,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACnC,CAAC;IAEO,UAAU,CAAC,GAA4B;QAC7C,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAY;YACpB,OAAO,EAAE,GAAG,CAAC,OAAiB;YAC9B,SAAS,EAAE,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC;YACzC,SAAS,EAAE,GAAG,CAAC,UAAoB;YACnC,IAAI,EAAE,GAAG,CAAC,IAAmB;YAC7B,YAAY,EAAE,GAAG,CAAC,aAAuB;YACzC,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,SAAmB,CAAC;YAC5C,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAkB,CAAC;YAC5C,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,UAAoB,CAAC;SAC9C,CAAC;IACJ,CAAC;IAEO,mBAAmB,CACzB,GAA4B;QAE5B,OAAO;YACL,SAAS,EAAE,GAAG,CAAC,UAAoB;YACnC,YAAY,EAAE,GAAG,CAAC,aAAuB;YACzC,cAAc,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,gBAA0B,CAAC;YACxD,aAAa,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,eAAyB,CAAC;YACtD,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,UAAoB,CAAC;YAC7C,qFAAqF;YACrF,GAAG,CAAC,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,OAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClE,GAAG,CAAC,GAAG,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,UAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC3E,CAAC;IACJ,CAAC;IAEO,YAAY,CAAC,GAA4B;QAC/C,OAAO;YACL,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC;YAChC,QAAQ,EAAE,GAAG,CAAC,SAAmB;YACjC,QAAQ,EAAE,GAAG,CAAC,SAAmB;SAClC,CAAC;IACJ,CAAC;IAED,0CAA0C;IAE1C,KAAK,CAAC,MAAM,CAAC,OAAmC;QAC9C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEjC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,KAAK,CAAC,GAAG,CACb,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACtB,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,MAAM,EAAE,KAAK,CAAC,SAAS;YACvB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,UAAU,EAAE,KAAK,CAAC,SAAS;YAC3B,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,aAAa,EAAE,KAAK,CAAC,YAAY;YACjC,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE;YACpC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC;YACxC,UAAU,EAAE,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE;SACtC,CAAC,CAAC,CACJ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CACd,SAAmB,EACnB,KAAa,EACb,KAAa;QAEb,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAE1C,MAAM,OAAO,GAAG,MAAM,KAAK;aACxB,KAAK,EAAE;aACP,SAAS,CAAC,SAAS,CAAC;aACpB,cAAc,CAAC,KAAK,CAAC;aACrB,MAAM,CAAC,QAAQ,CAAC;aAChB,KAAK,CAAC,KAAK,CAAC;aACZ,OAAO,EAAE,CAAC;QAEb,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACzB,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,GAA8B,CAAC,CAAC;YAC9D,OAAO;gBACL,GAAG,KAAK;gBACR,QAAQ,EAAG,GAAG,CAAC,gBAA2B,IAAI,CAAC;aAChD,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,SAAiB;QACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,MAAM,KAAK;aACxB,KAAK,EAAE;aACP,KAAK,CAAC,iBAAiB,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC;aACzD,OAAO,EAAE,CAAC;QAEb,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACzB,IAAI,CAAC,UAAU,CAAC,GAA8B,CAAC,CAChD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,SAAiB;QACvC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEpC,2EAA2E;QAC3E,MAAM,QAAQ,GAAG,MAAM,KAAK;aACzB,KAAK,EAAE;aACP,KAAK,CAAC,iBAAiB,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC;aACzD,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC;aACd,OAAO,EAAE,CAAC;QACb,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC;QAE9B,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,MAAM,KAAK,CAAC,MAAM,CAAC,iBAAiB,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACzE,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,oCAAoC;IAEpC,KAAK,CAAC,iBAAiB,CACrB,SAAiB;QAEjB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,MAAM,KAAK;aACxB,KAAK,EAAE;aACP,KAAK,CAAC,iBAAiB,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC;aACzD,KAAK,CAAC,CAAC,CAAC;aACR,OAAO,EAAE,CAAC;QAEb,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAA4B,CAAC,CAAC;IAClE,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,OAAuB;QAChD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,KAAK;aACzB,KAAK,EAAE;aACP,KAAK,CAAC,iBAAiB,mBAAmB,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC;aACjE,KAAK,CAAC,CAAC,CAAC;aACR,OAAO,EAAE,CAAC;QAEb,MAAM,GAAG,GAAG;YACV,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,SAAS,EAAE,OAAO,CAAC,QAAQ;YAC3B,SAAS,EAAE,OAAO,CAAC,QAAQ;YAC3B,aAAa,EAAE,OAAO,CAAC,YAAY;YACnC,gBAAgB,EAAE,OAAO,CAAC,cAAc,CAAC,OAAO,EAAE;YAClD,eAAe,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE;YAChD,UAAU,EAAE,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE;YACvC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI;YAChC,UAAU,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI;SACtC,CAAC;QAEF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,CAAC,MAAM,CAAC;gBACjB,KAAK,EAAE,iBAAiB,mBAAmB,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG;gBACjE,MAAM,EAAE,GAAG;aACZ,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,CAAC;QAE9C,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACzB,IAAI,CAAC,mBAAmB,CAAC,GAA8B,CAAC,CACzD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,SAAiB;QAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,KAAK;aACzB,KAAK,EAAE;aACP,KAAK,CAAC,iBAAiB,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC;aACzD,KAAK,CAAC,CAAC,CAAC;aACR,OAAO,EAAE,CAAC;QAEb,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,KAAK,CAAC,MAAM,CAAC,iBAAiB,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACvE,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { Schema } from "apache-arrow";
|
|
2
|
+
export declare const CONVERSATION_HISTORY_TABLE = "conversation_history";
|
|
3
|
+
/**
|
|
4
|
+
* Tracks which sessions have been indexed and their file sizes,
|
|
5
|
+
* enabling idempotent incremental indexing.
|
|
6
|
+
*/
|
|
7
|
+
export declare const INDEXED_SESSIONS_TABLE = "indexed_sessions";
|
|
8
|
+
export declare const conversationHistorySchema: Schema<any>;
|
|
9
|
+
export declare const indexedSessionsSchema: Schema<any>;
|
|
10
|
+
//# sourceMappingURL=conversation-history.schema.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"conversation-history.schema.d.ts","sourceRoot":"","sources":["../../../src/db/conversation-history.schema.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EAKP,MAAM,cAAc,CAAC;AAGtB,eAAO,MAAM,0BAA0B,yBAAyB,CAAC;AAEjE;;;GAGG;AACH,eAAO,MAAM,sBAAsB,qBAAqB,CAAC;AAEzD,eAAO,MAAM,yBAAyB,aAUpC,CAAC;AAEH,eAAO,MAAM,qBAAqB,aAUhC,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { Schema, Field, Utf8, Int32, Float64, } from "apache-arrow";
|
|
2
|
+
import { vectorField, timestampField } from "./schema.js";
|
|
3
|
+
export const CONVERSATION_HISTORY_TABLE = "conversation_history";
|
|
4
|
+
/**
|
|
5
|
+
* Tracks which sessions have been indexed and their file sizes,
|
|
6
|
+
* enabling idempotent incremental indexing.
|
|
7
|
+
*/
|
|
8
|
+
export const INDEXED_SESSIONS_TABLE = "indexed_sessions";
|
|
9
|
+
export const conversationHistorySchema = new Schema([
|
|
10
|
+
new Field("id", new Utf8(), false),
|
|
11
|
+
vectorField(),
|
|
12
|
+
new Field("content", new Utf8(), false),
|
|
13
|
+
new Field("session_id", new Utf8(), false),
|
|
14
|
+
new Field("role", new Utf8(), false), // "user" | "assistant"
|
|
15
|
+
new Field("message_index", new Int32(), false),
|
|
16
|
+
timestampField("timestamp"),
|
|
17
|
+
new Field("metadata", new Utf8(), false), // JSON string
|
|
18
|
+
timestampField("created_at"),
|
|
19
|
+
]);
|
|
20
|
+
export const indexedSessionsSchema = new Schema([
|
|
21
|
+
new Field("session_id", new Utf8(), false),
|
|
22
|
+
new Field("file_path", new Utf8(), false),
|
|
23
|
+
new Field("file_size", new Float64(), false), // Float64 avoids Int32 overflow and BigInt handling
|
|
24
|
+
new Field("message_count", new Int32(), false),
|
|
25
|
+
timestampField("first_message_at"),
|
|
26
|
+
timestampField("last_message_at"),
|
|
27
|
+
timestampField("indexed_at"),
|
|
28
|
+
new Field("project", new Utf8(), true), // Nullable
|
|
29
|
+
new Field("git_branch", new Utf8(), true), // Nullable
|
|
30
|
+
]);
|
|
31
|
+
//# sourceMappingURL=conversation-history.schema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"conversation-history.schema.js","sourceRoot":"","sources":["../../../src/db/conversation-history.schema.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EACN,KAAK,EACL,IAAI,EACJ,KAAK,EACL,OAAO,GACR,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE1D,MAAM,CAAC,MAAM,0BAA0B,GAAG,sBAAsB,CAAC;AAEjE;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,kBAAkB,CAAC;AAEzD,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,MAAM,CAAC;IAClD,IAAI,KAAK,CAAC,IAAI,EAAE,IAAI,IAAI,EAAE,EAAE,KAAK,CAAC;IAClC,WAAW,EAAE;IACb,IAAI,KAAK,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE,KAAK,CAAC;IACvC,IAAI,KAAK,CAAC,YAAY,EAAE,IAAI,IAAI,EAAE,EAAE,KAAK,CAAC;IAC1C,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,EAAE,KAAK,CAAC,EAAE,uBAAuB;IAC7D,IAAI,KAAK,CAAC,eAAe,EAAE,IAAI,KAAK,EAAE,EAAE,KAAK,CAAC;IAC9C,cAAc,CAAC,WAAW,CAAC;IAC3B,IAAI,KAAK,CAAC,UAAU,EAAE,IAAI,IAAI,EAAE,EAAE,KAAK,CAAC,EAAE,cAAc;IACxD,cAAc,CAAC,YAAY,CAAC;CAC7B,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,MAAM,CAAC;IAC9C,IAAI,KAAK,CAAC,YAAY,EAAE,IAAI,IAAI,EAAE,EAAE,KAAK,CAAC;IAC1C,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,IAAI,EAAE,EAAE,KAAK,CAAC;IACzC,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,OAAO,EAAE,EAAE,KAAK,CAAC,EAAE,oDAAoD;IAClG,IAAI,KAAK,CAAC,eAAe,EAAE,IAAI,KAAK,EAAE,EAAE,KAAK,CAAC;IAC9C,cAAc,CAAC,kBAAkB,CAAC;IAClC,cAAc,CAAC,iBAAiB,CAAC;IACjC,cAAc,CAAC,YAAY,CAAC;IAC5B,IAAI,KAAK,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE,IAAI,CAAC,EAAE,WAAW;IACnD,IAAI,KAAK,CAAC,YAAY,EAAE,IAAI,IAAI,EAAE,EAAE,IAAI,CAAC,EAAE,WAAW;CACvD,CAAC,CAAC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import * as lancedb from "@lancedb/lancedb";
|
|
2
|
+
import { rerankers, type Table } from "@lancedb/lancedb";
|
|
3
|
+
import type { Schema } from "apache-arrow";
|
|
4
|
+
/**
|
|
5
|
+
* Escapes a string for use in LanceDB/DataFusion SQL WHERE clauses.
|
|
6
|
+
* Doubles single quotes to prevent SQL injection (standard SQL escaping).
|
|
7
|
+
*/
|
|
8
|
+
export declare function escapeLanceDbString(value: string): string;
|
|
9
|
+
/**
|
|
10
|
+
* Converts LanceDB's Arrow Vector type to a plain number[].
|
|
11
|
+
* LanceDB returns an Arrow Vector object which is iterable but not an array.
|
|
12
|
+
*/
|
|
13
|
+
export declare function arrowVectorToArray(value: unknown): number[];
|
|
14
|
+
/**
|
|
15
|
+
* Opens an existing table or creates it with the given schema.
|
|
16
|
+
* Does NOT cache — callers should cache the returned Table if desired.
|
|
17
|
+
*/
|
|
18
|
+
export declare function getOrCreateTable(db: lancedb.Connection, name: string, schema: Schema): Promise<Table>;
|
|
19
|
+
/**
|
|
20
|
+
* Creates a mutex-guarded function that ensures an FTS index exists on a table's content column.
|
|
21
|
+
*
|
|
22
|
+
* Once the FTS index is confirmed/created, the promise is retained for the lifetime of the
|
|
23
|
+
* caller — the index persists in LanceDB, so re-checking is unnecessary. On error, the
|
|
24
|
+
* mutex resets so the next call can retry.
|
|
25
|
+
*
|
|
26
|
+
* The key design constraint: the promise must be captured synchronously (before any await)
|
|
27
|
+
* to prevent concurrent callers from racing past the guard.
|
|
28
|
+
*/
|
|
29
|
+
export declare function createFtsMutex(getTable: () => Promise<Table>): () => Promise<void>;
|
|
30
|
+
/**
|
|
31
|
+
* Creates a promise-mutex for RRFReranker instantiation.
|
|
32
|
+
* Same pattern as createFtsMutex: create once, cache forever, reset on error.
|
|
33
|
+
*/
|
|
34
|
+
export declare function createRerankerMutex(k?: number): () => Promise<rerankers.RRFReranker>;
|
|
35
|
+
//# sourceMappingURL=lancedb-utils.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lancedb-utils.d.ts","sourceRoot":"","sources":["../../../src/db/lancedb-utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAS,SAAS,EAAE,KAAK,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAChE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAE3C;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,EAAE,CAI3D;AAED;;;GAGG;AACH,wBAAsB,gBAAgB,CACpC,EAAE,EAAE,OAAO,CAAC,UAAU,EACtB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,KAAK,CAAC,CAMhB;AAED;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,MAAM,OAAO,CAAC,KAAK,CAAC,GAC7B,MAAM,OAAO,CAAC,IAAI,CAAC,CA0BrB;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,CAAC,GAAE,MAAW,GACb,MAAM,OAAO,CAAC,SAAS,CAAC,WAAW,CAAC,CAYtC"}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { Index, rerankers } from "@lancedb/lancedb";
|
|
2
|
+
/**
|
|
3
|
+
* Escapes a string for use in LanceDB/DataFusion SQL WHERE clauses.
|
|
4
|
+
* Doubles single quotes to prevent SQL injection (standard SQL escaping).
|
|
5
|
+
*/
|
|
6
|
+
export function escapeLanceDbString(value) {
|
|
7
|
+
return value.replace(/'/g, "''");
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Converts LanceDB's Arrow Vector type to a plain number[].
|
|
11
|
+
* LanceDB returns an Arrow Vector object which is iterable but not an array.
|
|
12
|
+
*/
|
|
13
|
+
export function arrowVectorToArray(value) {
|
|
14
|
+
return Array.isArray(value)
|
|
15
|
+
? value
|
|
16
|
+
: Array.from(value);
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Opens an existing table or creates it with the given schema.
|
|
20
|
+
* Does NOT cache — callers should cache the returned Table if desired.
|
|
21
|
+
*/
|
|
22
|
+
export async function getOrCreateTable(db, name, schema) {
|
|
23
|
+
const names = await db.tableNames();
|
|
24
|
+
if (names.includes(name)) {
|
|
25
|
+
return await db.openTable(name);
|
|
26
|
+
}
|
|
27
|
+
return await db.createTable(name, [], { schema });
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Creates a mutex-guarded function that ensures an FTS index exists on a table's content column.
|
|
31
|
+
*
|
|
32
|
+
* Once the FTS index is confirmed/created, the promise is retained for the lifetime of the
|
|
33
|
+
* caller — the index persists in LanceDB, so re-checking is unnecessary. On error, the
|
|
34
|
+
* mutex resets so the next call can retry.
|
|
35
|
+
*
|
|
36
|
+
* The key design constraint: the promise must be captured synchronously (before any await)
|
|
37
|
+
* to prevent concurrent callers from racing past the guard.
|
|
38
|
+
*/
|
|
39
|
+
export function createFtsMutex(getTable) {
|
|
40
|
+
let promise = null;
|
|
41
|
+
return () => {
|
|
42
|
+
if (promise)
|
|
43
|
+
return promise;
|
|
44
|
+
promise = (async () => {
|
|
45
|
+
const table = await getTable();
|
|
46
|
+
const indices = await table.listIndices();
|
|
47
|
+
const hasFtsIndex = indices.some((idx) => idx.columns.includes("content") && idx.indexType === "FTS");
|
|
48
|
+
if (!hasFtsIndex) {
|
|
49
|
+
await table.createIndex("content", {
|
|
50
|
+
config: Index.fts(),
|
|
51
|
+
});
|
|
52
|
+
await table.waitForIndex(["content_idx"], 30);
|
|
53
|
+
}
|
|
54
|
+
})().catch((error) => {
|
|
55
|
+
promise = null;
|
|
56
|
+
throw error;
|
|
57
|
+
});
|
|
58
|
+
return promise;
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Creates a promise-mutex for RRFReranker instantiation.
|
|
63
|
+
* Same pattern as createFtsMutex: create once, cache forever, reset on error.
|
|
64
|
+
*/
|
|
65
|
+
export function createRerankerMutex(k = 60) {
|
|
66
|
+
let promise = null;
|
|
67
|
+
return () => {
|
|
68
|
+
if (!promise) {
|
|
69
|
+
promise = rerankers.RRFReranker.create(k).catch((e) => {
|
|
70
|
+
promise = null;
|
|
71
|
+
throw e;
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
return promise;
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
//# sourceMappingURL=lancedb-utils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lancedb-utils.js","sourceRoot":"","sources":["../../../src/db/lancedb-utils.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAc,MAAM,kBAAkB,CAAC;AAGhE;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAa;IAC/C,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;AACnC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAc;IAC/C,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACzB,CAAC,CAAC,KAAK;QACP,CAAC,CAAE,KAAK,CAAC,IAAI,CAAC,KAAyB,CAAc,CAAC;AAC1D,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,EAAsB,EACtB,IAAY,EACZ,MAAc;IAEd,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;IACpC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,OAAO,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,MAAM,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;AACpD,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,cAAc,CAC5B,QAA8B;IAE9B,IAAI,OAAO,GAAyB,IAAI,CAAC;IAEzC,OAAO,GAAG,EAAE;QACV,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;QAE5B,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,KAAK,GAAG,MAAM,QAAQ,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;YAC1C,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAC9B,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,SAAS,KAAK,KAAK,CACpE,CAAC;YAEF,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE;oBACjC,MAAM,EAAE,KAAK,CAAC,GAAG,EAAE;iBACpB,CAAC,CAAC;gBACH,MAAM,KAAK,CAAC,YAAY,CAAC,CAAC,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC;YAChD,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACnB,OAAO,GAAG,IAAI,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CACjC,IAAY,EAAE;IAEd,IAAI,OAAO,GAA0C,IAAI,CAAC;IAE1D,OAAO,GAAG,EAAE;QACV,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,SAAS,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACpD,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM,CAAC,CAAC;YACV,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -2,8 +2,9 @@ import * as lancedb from "@lancedb/lancedb";
|
|
|
2
2
|
import { type Memory, type HybridRow } from "../types/memory.js";
|
|
3
3
|
export declare class MemoryRepository {
|
|
4
4
|
private db;
|
|
5
|
-
private ftsIndexPromise;
|
|
6
5
|
private migrationPromise;
|
|
6
|
+
private ensureFtsIndex;
|
|
7
|
+
private getReranker;
|
|
7
8
|
constructor(db: lancedb.Connection);
|
|
8
9
|
private getTable;
|
|
9
10
|
/**
|
|
@@ -16,17 +17,6 @@ export declare class MemoryRepository {
|
|
|
16
17
|
* This handles databases created before the hybrid memory system was introduced.
|
|
17
18
|
*/
|
|
18
19
|
private migrateSchemaIfNeeded;
|
|
19
|
-
/**
|
|
20
|
-
* Ensures the FTS index exists on the content column.
|
|
21
|
-
* Uses a mutex pattern to prevent concurrent index creation.
|
|
22
|
-
* The key insight: we must capture the promise BEFORE any await.
|
|
23
|
-
*/
|
|
24
|
-
private ensureFtsIndex;
|
|
25
|
-
/**
|
|
26
|
-
* Creates the FTS index if it doesn't already exist.
|
|
27
|
-
* Gets its own table reference to ensure consistent index state.
|
|
28
|
-
*/
|
|
29
|
-
private createFtsIndexIfNeeded;
|
|
30
20
|
/**
|
|
31
21
|
* Converts a raw LanceDB row to a Memory object.
|
|
32
22
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory.repository.d.ts","sourceRoot":"","sources":["../../../src/db/memory.repository.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"memory.repository.d.ts","sourceRoot":"","sources":["../../../src/db/memory.repository.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAI5C,OAAO,EACL,KAAK,MAAM,EACX,KAAK,SAAS,EAEf,MAAM,oBAAoB,CAAC;AAE5B,qBAAa,gBAAgB;IAUf,OAAO,CAAC,EAAE;IARtB,OAAO,CAAC,gBAAgB,CAA8B;IAGtD,OAAO,CAAC,cAAc,CAAsB;IAG5C,OAAO,CAAC,WAAW,CAAyB;gBAExB,EAAE,EAAE,OAAO,CAAC,UAAU;YAI5B,QAAQ;IAWtB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAavB;;;OAGG;YACW,qBAAqB;IAqBnC;;OAEG;IACH,OAAO,CAAC,WAAW;IAiBb,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBrC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBrC,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAW5C,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAqB/C;;;;;;;;OAQG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;CAsB1F"}
|