@aeriondyseti/vector-memory-mcp 1.1.0-dev.5 → 2.0.0-rc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -4
- package/package.json +12 -18
- package/scripts/migrate-from-lancedb.ts +56 -0
- package/scripts/smoke-test.ts +699 -0
- package/scripts/test-runner.ts +11 -1
- package/src/db/connection.ts +18 -4
- package/src/db/conversation.repository.ts +164 -78
- package/src/db/memory.repository.ts +180 -168
- package/src/db/migrations.ts +70 -0
- package/src/db/sqlite-utils.ts +78 -0
- package/src/http/server.ts +40 -35
- package/src/index.ts +33 -3
- package/src/mcp/server.ts +2 -1
- package/src/migration.ts +254 -0
- package/dist/package.json +0 -71
- package/dist/scripts/test-runner.d.ts +0 -9
- package/dist/scripts/test-runner.d.ts.map +0 -1
- package/dist/scripts/test-runner.js +0 -61
- package/dist/scripts/test-runner.js.map +0 -1
- package/dist/scripts/warmup.d.ts +0 -8
- package/dist/scripts/warmup.d.ts.map +0 -1
- package/dist/scripts/warmup.js +0 -61
- package/dist/scripts/warmup.js.map +0 -1
- package/dist/src/config/index.d.ts +0 -41
- package/dist/src/config/index.d.ts.map +0 -1
- package/dist/src/config/index.js +0 -75
- package/dist/src/config/index.js.map +0 -1
- package/dist/src/db/connection.d.ts +0 -3
- package/dist/src/db/connection.d.ts.map +0 -1
- package/dist/src/db/connection.js +0 -10
- package/dist/src/db/connection.js.map +0 -1
- package/dist/src/db/conversation.repository.d.ts +0 -26
- package/dist/src/db/conversation.repository.d.ts.map +0 -1
- package/dist/src/db/conversation.repository.js +0 -72
- package/dist/src/db/conversation.repository.js.map +0 -1
- package/dist/src/db/conversation.schema.d.ts +0 -4
- package/dist/src/db/conversation.schema.d.ts.map +0 -1
- package/dist/src/db/conversation.schema.js +0 -15
- package/dist/src/db/conversation.schema.js.map +0 -1
- package/dist/src/db/lancedb-utils.d.ts +0 -45
- package/dist/src/db/lancedb-utils.d.ts.map +0 -1
- package/dist/src/db/lancedb-utils.js +0 -106
- package/dist/src/db/lancedb-utils.js.map +0 -1
- package/dist/src/db/memory.repository.d.ts +0 -40
- package/dist/src/db/memory.repository.d.ts.map +0 -1
- package/dist/src/db/memory.repository.js +0 -183
- package/dist/src/db/memory.repository.js.map +0 -1
- package/dist/src/db/schema.d.ts +0 -7
- package/dist/src/db/schema.d.ts.map +0 -1
- package/dist/src/db/schema.js +0 -19
- package/dist/src/db/schema.js.map +0 -1
- package/dist/src/http/mcp-transport.d.ts +0 -19
- package/dist/src/http/mcp-transport.d.ts.map +0 -1
- package/dist/src/http/mcp-transport.js +0 -191
- package/dist/src/http/mcp-transport.js.map +0 -1
- package/dist/src/http/server.d.ts +0 -13
- package/dist/src/http/server.d.ts.map +0 -1
- package/dist/src/http/server.js +0 -224
- package/dist/src/http/server.js.map +0 -1
- package/dist/src/index.d.ts +0 -3
- package/dist/src/index.d.ts.map +0 -1
- package/dist/src/index.js +0 -68
- package/dist/src/index.js.map +0 -1
- package/dist/src/mcp/handlers.d.ts +0 -15
- package/dist/src/mcp/handlers.d.ts.map +0 -1
- package/dist/src/mcp/handlers.js +0 -313
- package/dist/src/mcp/handlers.js.map +0 -1
- package/dist/src/mcp/server.d.ts +0 -5
- package/dist/src/mcp/server.d.ts.map +0 -1
- package/dist/src/mcp/server.js +0 -22
- package/dist/src/mcp/server.js.map +0 -1
- package/dist/src/mcp/tools.d.ts +0 -13
- package/dist/src/mcp/tools.d.ts.map +0 -1
- package/dist/src/mcp/tools.js +0 -352
- package/dist/src/mcp/tools.js.map +0 -1
- package/dist/src/services/conversation.service.d.ts +0 -38
- package/dist/src/services/conversation.service.d.ts.map +0 -1
- package/dist/src/services/conversation.service.js +0 -252
- package/dist/src/services/conversation.service.js.map +0 -1
- package/dist/src/services/embeddings.service.d.ts +0 -12
- package/dist/src/services/embeddings.service.d.ts.map +0 -1
- package/dist/src/services/embeddings.service.js +0 -37
- package/dist/src/services/embeddings.service.js.map +0 -1
- package/dist/src/services/memory.service.d.ts +0 -40
- package/dist/src/services/memory.service.d.ts.map +0 -1
- package/dist/src/services/memory.service.js +0 -258
- package/dist/src/services/memory.service.js.map +0 -1
- package/dist/src/services/parsers/claude-code.parser.d.ts +0 -8
- package/dist/src/services/parsers/claude-code.parser.d.ts.map +0 -1
- package/dist/src/services/parsers/claude-code.parser.js +0 -191
- package/dist/src/services/parsers/claude-code.parser.js.map +0 -1
- package/dist/src/services/parsers/types.d.ts +0 -9
- package/dist/src/services/parsers/types.d.ts.map +0 -1
- package/dist/src/services/parsers/types.js +0 -2
- package/dist/src/services/parsers/types.js.map +0 -1
- package/dist/src/types/conversation.d.ts +0 -99
- package/dist/src/types/conversation.d.ts.map +0 -1
- package/dist/src/types/conversation.js +0 -2
- package/dist/src/types/conversation.js.map +0 -1
- package/dist/src/types/memory.d.ts +0 -30
- package/dist/src/types/memory.d.ts.map +0 -1
- package/dist/src/types/memory.js +0 -18
- package/dist/src/types/memory.js.map +0 -1
- package/src/db/conversation.schema.ts +0 -33
- package/src/db/lancedb-utils.ts +0 -125
- package/src/db/schema.ts +0 -38
|
@@ -1,99 +0,0 @@
|
|
|
1
|
-
/** A single parsed message from a session log */
|
|
2
|
-
export interface ParsedMessage {
|
|
3
|
-
uuid: string;
|
|
4
|
-
role: "user" | "assistant";
|
|
5
|
-
content: string;
|
|
6
|
-
timestamp: Date;
|
|
7
|
-
messageIndex: number;
|
|
8
|
-
sessionId: string;
|
|
9
|
-
project: string;
|
|
10
|
-
gitBranch?: string;
|
|
11
|
-
isSubagent: boolean;
|
|
12
|
-
agentId?: string;
|
|
13
|
-
}
|
|
14
|
-
/** Metadata stored per conversation chunk in the database */
|
|
15
|
-
export interface ConversationChunkMetadata {
|
|
16
|
-
session_id: string;
|
|
17
|
-
timestamp: string;
|
|
18
|
-
role: string;
|
|
19
|
-
message_index_start: number;
|
|
20
|
-
message_index_end: number;
|
|
21
|
-
project: string;
|
|
22
|
-
git_branch?: string;
|
|
23
|
-
is_subagent: boolean;
|
|
24
|
-
agent_id?: string;
|
|
25
|
-
}
|
|
26
|
-
/** A chunk of conversation ready for indexing */
|
|
27
|
-
export interface ConversationChunk {
|
|
28
|
-
id: string;
|
|
29
|
-
content: string;
|
|
30
|
-
sessionId: string;
|
|
31
|
-
timestamp: Date;
|
|
32
|
-
endTimestamp: Date;
|
|
33
|
-
role: string;
|
|
34
|
-
messageIndexStart: number;
|
|
35
|
-
messageIndexEnd: number;
|
|
36
|
-
project: string;
|
|
37
|
-
metadata: ConversationChunkMetadata;
|
|
38
|
-
}
|
|
39
|
-
/** Tracking record for an indexed session */
|
|
40
|
-
export interface IndexedSession {
|
|
41
|
-
sessionId: string;
|
|
42
|
-
filePath: string;
|
|
43
|
-
project: string;
|
|
44
|
-
lastModified: number;
|
|
45
|
-
chunkCount: number;
|
|
46
|
-
messageCount: number;
|
|
47
|
-
indexedAt: Date;
|
|
48
|
-
firstMessageAt: Date;
|
|
49
|
-
lastMessageAt: Date;
|
|
50
|
-
}
|
|
51
|
-
/** Raw row from conversation_history table with RRF score */
|
|
52
|
-
export interface ConversationHybridRow {
|
|
53
|
-
id: string;
|
|
54
|
-
content: string;
|
|
55
|
-
metadata: Record<string, unknown>;
|
|
56
|
-
createdAt: Date;
|
|
57
|
-
rrfScore: number;
|
|
58
|
-
}
|
|
59
|
-
/** Unified search result with source provenance */
|
|
60
|
-
export interface SearchResult {
|
|
61
|
-
id: string;
|
|
62
|
-
content: string;
|
|
63
|
-
metadata: Record<string, unknown>;
|
|
64
|
-
createdAt: Date;
|
|
65
|
-
updatedAt: Date;
|
|
66
|
-
source: "memory" | "conversation_history";
|
|
67
|
-
score: number;
|
|
68
|
-
supersededBy?: string | null;
|
|
69
|
-
usefulness?: number;
|
|
70
|
-
accessCount?: number;
|
|
71
|
-
lastAccessed?: Date | null;
|
|
72
|
-
sessionId?: string;
|
|
73
|
-
role?: string;
|
|
74
|
-
messageIndexStart?: number;
|
|
75
|
-
messageIndexEnd?: number;
|
|
76
|
-
}
|
|
77
|
-
/** Session file info returned by the parser's file discovery */
|
|
78
|
-
export interface SessionFileInfo {
|
|
79
|
-
filePath: string;
|
|
80
|
-
sessionId: string;
|
|
81
|
-
project: string;
|
|
82
|
-
lastModified: Date;
|
|
83
|
-
}
|
|
84
|
-
/** Search filter options for conversation history */
|
|
85
|
-
export interface HistoryFilters {
|
|
86
|
-
sessionId?: string;
|
|
87
|
-
role?: string;
|
|
88
|
-
project?: string;
|
|
89
|
-
after?: Date;
|
|
90
|
-
before?: Date;
|
|
91
|
-
}
|
|
92
|
-
/** Options for the integrated search across both sources */
|
|
93
|
-
export interface SearchOptions {
|
|
94
|
-
includeHistory?: boolean;
|
|
95
|
-
historyOnly?: boolean;
|
|
96
|
-
historyWeight?: number;
|
|
97
|
-
historyFilters?: HistoryFilters;
|
|
98
|
-
}
|
|
99
|
-
//# sourceMappingURL=conversation.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"conversation.d.ts","sourceRoot":"","sources":["../../../src/types/conversation.ts"],"names":[],"mappings":"AAAA,iDAAiD;AACjD,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,6DAA6D;AAC7D,MAAM,WAAW,yBAAyB;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,mBAAmB,EAAE,MAAM,CAAC;IAC5B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,iDAAiD;AACjD,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,IAAI,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,yBAAyB,CAAC;CACrC;AAED,6CAA6C;AAC7C,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,IAAI,CAAC;IAChB,cAAc,EAAE,IAAI,CAAC;IACrB,aAAa,EAAE,IAAI,CAAC;CACrB;AAED,6DAA6D;AAC7D,MAAM,WAAW,qBAAqB;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,SAAS,EAAE,IAAI,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,mDAAmD;AACnD,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;IAChB,MAAM,EAAE,QAAQ,GAAG,sBAAsB,CAAC;IAC1C,KAAK,EAAE,MAAM,CAAC;IAEd,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC;IAE3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,gEAAgE;AAChE,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,IAAI,CAAC;CACpB;AAED,qDAAqD;AACrD,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,IAAI,CAAC;IACb,MAAM,CAAC,EAAE,IAAI,CAAC;CACf;AAED,4DAA4D;AAC5D,MAAM,WAAW,aAAa;IAC5B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,cAAc,CAAC;CACjC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"conversation.js","sourceRoot":"","sources":["../../../src/types/conversation.ts"],"names":[],"mappings":""}
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
export declare const DELETED_TOMBSTONE = "DELETED";
|
|
2
|
-
export interface Memory {
|
|
3
|
-
id: string;
|
|
4
|
-
content: string;
|
|
5
|
-
embedding: number[];
|
|
6
|
-
metadata: Record<string, unknown>;
|
|
7
|
-
createdAt: Date;
|
|
8
|
-
updatedAt: Date;
|
|
9
|
-
supersededBy: string | null;
|
|
10
|
-
usefulness: number;
|
|
11
|
-
accessCount: number;
|
|
12
|
-
lastAccessed: Date | null;
|
|
13
|
-
}
|
|
14
|
-
export declare function isDeleted(memory: Memory): boolean;
|
|
15
|
-
export declare function memoryToDict(memory: Memory): Record<string, unknown>;
|
|
16
|
-
export type SearchIntent = 'continuity' | 'fact_check' | 'frequent' | 'associative' | 'explore';
|
|
17
|
-
export interface IntentProfile {
|
|
18
|
-
weights: {
|
|
19
|
-
relevance: number;
|
|
20
|
-
recency: number;
|
|
21
|
-
utility: number;
|
|
22
|
-
};
|
|
23
|
-
jitter: number;
|
|
24
|
-
}
|
|
25
|
-
/** Augments any entity type with an RRF score from hybrid search. */
|
|
26
|
-
export type WithRrfScore<T> = T & {
|
|
27
|
-
rrfScore: number;
|
|
28
|
-
};
|
|
29
|
-
export type HybridRow = WithRrfScore<Memory>;
|
|
30
|
-
//# sourceMappingURL=memory.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/types/memory.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,iBAAiB,YAAY,CAAC;AAE3C,MAAM,WAAW,MAAM;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAC;CAC3B;AAED,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEjD;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAYpE;AAED,MAAM,MAAM,YAAY,GAAG,YAAY,GAAG,YAAY,GAAG,UAAU,GAAG,aAAa,GAAG,SAAS,CAAC;AAEhG,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IACjE,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,qEAAqE;AACrE,MAAM,MAAM,YAAY,CAAC,CAAC,IAAI,CAAC,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvD,MAAM,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC"}
|
package/dist/src/types/memory.js
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
export const DELETED_TOMBSTONE = "DELETED";
|
|
2
|
-
export function isDeleted(memory) {
|
|
3
|
-
return memory.supersededBy === DELETED_TOMBSTONE;
|
|
4
|
-
}
|
|
5
|
-
export function memoryToDict(memory) {
|
|
6
|
-
return {
|
|
7
|
-
id: memory.id,
|
|
8
|
-
content: memory.content,
|
|
9
|
-
metadata: memory.metadata,
|
|
10
|
-
createdAt: memory.createdAt.toISOString(),
|
|
11
|
-
updatedAt: memory.updatedAt.toISOString(),
|
|
12
|
-
supersededBy: memory.supersededBy,
|
|
13
|
-
usefulness: memory.usefulness,
|
|
14
|
-
accessCount: memory.accessCount,
|
|
15
|
-
lastAccessed: memory.lastAccessed?.toISOString() ?? null,
|
|
16
|
-
};
|
|
17
|
-
}
|
|
18
|
-
//# sourceMappingURL=memory.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"memory.js","sourceRoot":"","sources":["../../../src/types/memory.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,iBAAiB,GAAG,SAAS,CAAC;AAe3C,MAAM,UAAU,SAAS,CAAC,MAAc;IACtC,OAAO,MAAM,CAAC,YAAY,KAAK,iBAAiB,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAc;IACzC,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,EAAE;QACb,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,WAAW,EAAE;QACzC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,WAAW,EAAE;QACzC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,WAAW,EAAE,IAAI,IAAI;KACzD,CAAC;AACJ,CAAC"}
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
Schema,
|
|
3
|
-
Field,
|
|
4
|
-
FixedSizeList,
|
|
5
|
-
Float32,
|
|
6
|
-
Utf8,
|
|
7
|
-
Timestamp,
|
|
8
|
-
TimeUnit,
|
|
9
|
-
Int32,
|
|
10
|
-
} from "apache-arrow";
|
|
11
|
-
|
|
12
|
-
export const CONVERSATION_TABLE_NAME = "conversation_history";
|
|
13
|
-
|
|
14
|
-
export const conversationSchema = new Schema([
|
|
15
|
-
new Field("id", new Utf8(), false),
|
|
16
|
-
new Field(
|
|
17
|
-
"vector",
|
|
18
|
-
new FixedSizeList(384, new Field("item", new Float32(), false)),
|
|
19
|
-
false
|
|
20
|
-
),
|
|
21
|
-
new Field("content", new Utf8(), false),
|
|
22
|
-
new Field("metadata", new Utf8(), false), // JSON string
|
|
23
|
-
new Field(
|
|
24
|
-
"created_at",
|
|
25
|
-
new Timestamp(TimeUnit.MILLISECOND),
|
|
26
|
-
false
|
|
27
|
-
),
|
|
28
|
-
new Field("session_id", new Utf8(), false),
|
|
29
|
-
new Field("role", new Utf8(), false),
|
|
30
|
-
new Field("message_index_start", new Int32(), false),
|
|
31
|
-
new Field("message_index_end", new Int32(), false),
|
|
32
|
-
new Field("project", new Utf8(), false),
|
|
33
|
-
]);
|
package/src/db/lancedb-utils.ts
DELETED
|
@@ -1,125 +0,0 @@
|
|
|
1
|
-
import * as lancedb from "@lancedb/lancedb";
|
|
2
|
-
import { Index, rerankers, type Table } from "@lancedb/lancedb";
|
|
3
|
-
import type { Schema } from "apache-arrow";
|
|
4
|
-
|
|
5
|
-
/**
|
|
6
|
-
* Escape a string value for safe interpolation into LanceDB/DataFusion SQL WHERE clauses.
|
|
7
|
-
*
|
|
8
|
-
* DataFusion uses ANSI SQL string literal rules:
|
|
9
|
-
* - String literals are delimited by single quotes
|
|
10
|
-
* - Single quotes within strings are escaped by doubling: ' -> ''
|
|
11
|
-
* - Backslashes are NOT escape characters (treated literally)
|
|
12
|
-
*/
|
|
13
|
-
export function escapeSql(value: string): string {
|
|
14
|
-
return value.replace(/'/g, "''");
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
/** Default k parameter for Reciprocal Rank Fusion reranking. */
|
|
18
|
-
export const RRF_K = 60;
|
|
19
|
-
|
|
20
|
-
/**
|
|
21
|
-
* Converts LanceDB's Arrow Vector type to a plain number[].
|
|
22
|
-
* LanceDB returns an Arrow Vector object which is iterable but not an array.
|
|
23
|
-
*/
|
|
24
|
-
export function arrowVectorToArray(value: unknown): number[] {
|
|
25
|
-
return Array.isArray(value)
|
|
26
|
-
? value
|
|
27
|
-
: (Array.from(value as Iterable<number>) as number[]);
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Safely parse a JSON string into an object, returning an empty object on failure.
|
|
32
|
-
*/
|
|
33
|
-
export function safeParseJsonObject(raw: string): Record<string, unknown> {
|
|
34
|
-
try {
|
|
35
|
-
return JSON.parse(raw);
|
|
36
|
-
} catch {
|
|
37
|
-
return {};
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
/**
|
|
42
|
-
* Opens an existing table or creates it with the given schema.
|
|
43
|
-
* Does NOT cache — callers should cache the returned Table if desired.
|
|
44
|
-
*/
|
|
45
|
-
export async function getOrCreateTable(
|
|
46
|
-
db: lancedb.Connection,
|
|
47
|
-
name: string,
|
|
48
|
-
schema: Schema
|
|
49
|
-
): Promise<Table> {
|
|
50
|
-
try {
|
|
51
|
-
return await db.openTable(name);
|
|
52
|
-
} catch (err: unknown) {
|
|
53
|
-
// Only proceed to create if the table was not found
|
|
54
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
55
|
-
if (!message.includes("was not found") && !message.includes("does not exist")) {
|
|
56
|
-
throw err;
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
try {
|
|
60
|
-
return await db.createTable(name, [], { schema });
|
|
61
|
-
} catch {
|
|
62
|
-
// Another caller may have created it concurrently
|
|
63
|
-
return await db.openTable(name);
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
/**
|
|
68
|
-
* Creates a mutex-guarded function that ensures an FTS index exists on a table's content column.
|
|
69
|
-
*
|
|
70
|
-
* Once the FTS index is confirmed/created, the promise is retained for the lifetime of the
|
|
71
|
-
* caller — the index persists in LanceDB, so re-checking is unnecessary. On error, the
|
|
72
|
-
* mutex resets so the next call can retry.
|
|
73
|
-
*
|
|
74
|
-
* The key design constraint: the promise must be captured synchronously (before any await)
|
|
75
|
-
* to prevent concurrent callers from racing past the guard.
|
|
76
|
-
*/
|
|
77
|
-
export function createFtsMutex(
|
|
78
|
-
getTable: () => Promise<Table>
|
|
79
|
-
): () => Promise<void> {
|
|
80
|
-
let promise: Promise<void> | null = null;
|
|
81
|
-
|
|
82
|
-
return () => {
|
|
83
|
-
if (promise) return promise;
|
|
84
|
-
|
|
85
|
-
promise = (async () => {
|
|
86
|
-
const table = await getTable();
|
|
87
|
-
const indices = await table.listIndices();
|
|
88
|
-
const hasFtsIndex = indices.some(
|
|
89
|
-
(idx) => idx.columns.includes("content") && idx.indexType === "FTS"
|
|
90
|
-
);
|
|
91
|
-
|
|
92
|
-
if (!hasFtsIndex) {
|
|
93
|
-
await table.createIndex("content", {
|
|
94
|
-
config: Index.fts(),
|
|
95
|
-
});
|
|
96
|
-
await table.waitForIndex(["content_idx"], 30);
|
|
97
|
-
}
|
|
98
|
-
})().catch((error) => {
|
|
99
|
-
promise = null;
|
|
100
|
-
throw error;
|
|
101
|
-
});
|
|
102
|
-
|
|
103
|
-
return promise;
|
|
104
|
-
};
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
/**
|
|
108
|
-
* Creates a promise-mutex for RRFReranker instantiation.
|
|
109
|
-
* Same pattern as createFtsMutex: create once, cache forever, reset on error.
|
|
110
|
-
*/
|
|
111
|
-
export function createRerankerMutex(
|
|
112
|
-
k: number = RRF_K
|
|
113
|
-
): () => Promise<rerankers.RRFReranker> {
|
|
114
|
-
let promise: Promise<rerankers.RRFReranker> | null = null;
|
|
115
|
-
|
|
116
|
-
return () => {
|
|
117
|
-
if (!promise) {
|
|
118
|
-
promise = rerankers.RRFReranker.create(k).catch((e) => {
|
|
119
|
-
promise = null;
|
|
120
|
-
throw e;
|
|
121
|
-
});
|
|
122
|
-
}
|
|
123
|
-
return promise;
|
|
124
|
-
};
|
|
125
|
-
}
|
package/src/db/schema.ts
DELETED
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
Schema,
|
|
3
|
-
Field,
|
|
4
|
-
FixedSizeList,
|
|
5
|
-
Float32,
|
|
6
|
-
Utf8,
|
|
7
|
-
Timestamp,
|
|
8
|
-
TimeUnit,
|
|
9
|
-
Int32,
|
|
10
|
-
} from "apache-arrow";
|
|
11
|
-
|
|
12
|
-
// Shared field helpers — used by both memory and conversation history schemas
|
|
13
|
-
export const EMBEDDING_DIMENSION = 384;
|
|
14
|
-
|
|
15
|
-
export const vectorField = () =>
|
|
16
|
-
new Field(
|
|
17
|
-
"vector",
|
|
18
|
-
new FixedSizeList(EMBEDDING_DIMENSION, new Field("item", new Float32())),
|
|
19
|
-
false
|
|
20
|
-
);
|
|
21
|
-
|
|
22
|
-
export const timestampField = (name: string, nullable = false) =>
|
|
23
|
-
new Field(name, new Timestamp(TimeUnit.MILLISECOND), nullable);
|
|
24
|
-
|
|
25
|
-
export const TABLE_NAME = "memories";
|
|
26
|
-
|
|
27
|
-
export const memorySchema = new Schema([
|
|
28
|
-
new Field("id", new Utf8(), false),
|
|
29
|
-
vectorField(),
|
|
30
|
-
new Field("content", new Utf8(), false),
|
|
31
|
-
new Field("metadata", new Utf8(), false), // JSON string
|
|
32
|
-
timestampField("created_at"),
|
|
33
|
-
timestampField("updated_at"),
|
|
34
|
-
new Field("superseded_by", new Utf8(), true), // Nullable
|
|
35
|
-
new Field("usefulness", new Float32(), false),
|
|
36
|
-
new Field("access_count", new Int32(), false),
|
|
37
|
-
timestampField("last_accessed", true),
|
|
38
|
-
]);
|