@memextend/core 0.1.9 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/memory/index.d.ts +1 -1
- package/dist/memory/index.d.ts.map +1 -1
- package/dist/memory/retrieve.d.ts +10 -4
- package/dist/memory/retrieve.d.ts.map +1 -1
- package/dist/memory/retrieve.js +5 -5
- package/dist/memory/retrieve.js.map +1 -1
- package/dist/memory/retrieve.test.js +8 -8
- package/dist/memory/retrieve.test.js.map +1 -1
- package/dist/storage/index.d.ts +2 -2
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +1 -1
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/{lancedb.d.ts → sqlite-vec.d.ts} +4 -12
- package/dist/storage/sqlite-vec.d.ts.map +1 -0
- package/dist/storage/sqlite-vec.js +131 -0
- package/dist/storage/sqlite-vec.js.map +1 -0
- package/package.json +4 -5
- package/dist/storage/lancedb.d.ts.map +0 -1
- package/dist/storage/lancedb.js +0 -131
- package/dist/storage/lancedb.js.map +0 -1
- package/dist/storage/lancedb.test.d.ts +0 -2
- package/dist/storage/lancedb.test.d.ts.map +0 -1
- package/dist/storage/lancedb.test.js +0 -67
- package/dist/storage/lancedb.test.js.map +0 -1
package/dist/memory/index.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ export * from './types.js';
|
|
|
2
2
|
export { TranscriptParser, formatMemoryContent, formatToolMemoryContent, formatCaptureContent, isTextCapture, isToolCapture, CONFIGURABLE_TOOLS } from './capture.js';
|
|
3
3
|
export type { ToolCapture, TextCapture, Capture, TranscriptEntry, TranscriptParserOptions, ToolCaptureConfig, ConfigurableTool } from './capture.js';
|
|
4
4
|
export { MemoryRetriever, formatContextForInjection } from './retrieve.js';
|
|
5
|
-
export type { EmbedFunction, MemoryRetrieverOptions, FormatOptions } from './retrieve.js';
|
|
5
|
+
export type { EmbedFunction, MemoryRetrieverOptions, FormatOptions, VectorStorage } from './retrieve.js';
|
|
6
6
|
export { deduplicateMemories, getDeduplicationStats } from './deduplicate.js';
|
|
7
7
|
export type { DeduplicationOptions } from './deduplicate.js';
|
|
8
8
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/memory/index.ts"],"names":[],"mappings":"AAGA,cAAc,YAAY,CAAC;AAC3B,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,uBAAuB,EACvB,oBAAoB,EACpB,aAAa,EACb,aAAa,EACb,kBAAkB,EACnB,MAAM,cAAc,CAAC;AACtB,YAAY,EACV,WAAW,EACX,WAAW,EACX,OAAO,EACP,eAAe,EACf,uBAAuB,EACvB,iBAAiB,EACjB,gBAAgB,EACjB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,eAAe,EAAE,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAC3E,YAAY,EAAE,aAAa,EAAE,sBAAsB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/memory/index.ts"],"names":[],"mappings":"AAGA,cAAc,YAAY,CAAC;AAC3B,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,uBAAuB,EACvB,oBAAoB,EACpB,aAAa,EACb,aAAa,EACb,kBAAkB,EACnB,MAAM,cAAc,CAAC;AACtB,YAAY,EACV,WAAW,EACX,WAAW,EACX,OAAO,EACP,eAAe,EACf,uBAAuB,EACvB,iBAAiB,EACjB,gBAAgB,EACjB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,eAAe,EAAE,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAC3E,YAAY,EAAE,aAAa,EAAE,sBAAsB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACzG,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAC9E,YAAY,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC"}
|
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
import type { SQLiteStorage } from '../storage/sqlite.js';
|
|
2
|
-
import type {
|
|
2
|
+
import type { VectorSearchResult } from '../storage/sqlite-vec.js';
|
|
3
3
|
import type { Memory, SearchResult, RetrievalOptions, GlobalProfile } from './types.js';
|
|
4
4
|
export type EmbedFunction = (text: string) => Promise<number[]>;
|
|
5
|
+
export interface VectorStorage {
|
|
6
|
+
search(vector: number[], limit?: number): Promise<VectorSearchResult[]>;
|
|
7
|
+
insertVector(id: string, vector: number[]): Promise<void>;
|
|
8
|
+
deleteVector(id: string): Promise<void>;
|
|
9
|
+
close(): Promise<void>;
|
|
10
|
+
}
|
|
5
11
|
export interface MemoryRetrieverOptions {
|
|
6
12
|
defaultLimit?: number;
|
|
7
13
|
defaultRecentDays?: number;
|
|
@@ -9,16 +15,16 @@ export interface MemoryRetrieverOptions {
|
|
|
9
15
|
}
|
|
10
16
|
export declare class MemoryRetriever {
|
|
11
17
|
private sqlite;
|
|
12
|
-
private
|
|
18
|
+
private vectorStore;
|
|
13
19
|
private embed;
|
|
14
20
|
private options;
|
|
15
|
-
constructor(sqlite: SQLiteStorage,
|
|
21
|
+
constructor(sqlite: SQLiteStorage, vectorStore: VectorStorage, embed: EmbedFunction, options?: MemoryRetrieverOptions);
|
|
16
22
|
/**
|
|
17
23
|
* Full-text search using FTS5
|
|
18
24
|
*/
|
|
19
25
|
search(query: string, options?: RetrievalOptions): Promise<SearchResult[]>;
|
|
20
26
|
/**
|
|
21
|
-
* Vector similarity search
|
|
27
|
+
* Vector similarity search
|
|
22
28
|
*/
|
|
23
29
|
vectorSearch(query: string, options?: RetrievalOptions): Promise<SearchResult[]>;
|
|
24
30
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retrieve.d.ts","sourceRoot":"","sources":["../../src/memory/retrieve.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"retrieve.d.ts","sourceRoot":"","sources":["../../src/memory/retrieve.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AACnE,OAAO,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAExF,MAAM,MAAM,aAAa,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;AAGhE,MAAM,WAAW,aAAa;IAC5B,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC;IACxE,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1D,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,MAAM,WAAW,sBAAsB;IACrC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,WAAW,CAAgB;IACnC,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,OAAO,CAAmC;gBAGhD,MAAM,EAAE,aAAa,EACrB,WAAW,EAAE,aAAa,EAC1B,KAAK,EAAE,aAAa,EACpB,OAAO,GAAE,sBAA2B;IAYtC;;OAEG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAKpF;;OAEG;IACG,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAuB1F;;OAEG;IACG,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAyC1F;;OAEG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,gBAAqB,GAAG,MAAM,EAAE;IAM7E;;OAEG;IACG,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC;QACrF,cAAc,EAAE,MAAM,EAAE,CAAC;QACzB,aAAa,EAAE,aAAa,EAAE,CAAC;QAC/B,gBAAgB,EAAE,YAAY,EAAE,CAAC;KAClC,CAAC;CA+BH;AAED,MAAM,WAAW,aAAa;IAC5B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE;IACP,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,aAAa,EAAE,aAAa,EAAE,CAAC;IAC/B,gBAAgB,CAAC,EAAE,YAAY,EAAE,CAAC;CACnC,EACD,OAAO,GAAE,aAAkB,GAC1B,MAAM,CAwFR"}
|
package/dist/memory/retrieve.js
CHANGED
|
@@ -2,12 +2,12 @@
|
|
|
2
2
|
// Copyright (c) 2026 ZodTTD LLC. MIT License.
|
|
3
3
|
export class MemoryRetriever {
|
|
4
4
|
sqlite;
|
|
5
|
-
|
|
5
|
+
vectorStore;
|
|
6
6
|
embed;
|
|
7
7
|
options;
|
|
8
|
-
constructor(sqlite,
|
|
8
|
+
constructor(sqlite, vectorStore, embed, options = {}) {
|
|
9
9
|
this.sqlite = sqlite;
|
|
10
|
-
this.
|
|
10
|
+
this.vectorStore = vectorStore;
|
|
11
11
|
this.embed = embed;
|
|
12
12
|
this.options = {
|
|
13
13
|
defaultLimit: options.defaultLimit ?? 0, // 0 = unlimited
|
|
@@ -23,12 +23,12 @@ export class MemoryRetriever {
|
|
|
23
23
|
return this.sqlite.searchFTS(query, limit);
|
|
24
24
|
}
|
|
25
25
|
/**
|
|
26
|
-
* Vector similarity search
|
|
26
|
+
* Vector similarity search
|
|
27
27
|
*/
|
|
28
28
|
async vectorSearch(query, options = {}) {
|
|
29
29
|
const limit = options.limit ?? this.options.defaultLimit;
|
|
30
30
|
const queryVector = await this.embed(query);
|
|
31
|
-
const vectorResults = await this.
|
|
31
|
+
const vectorResults = await this.vectorStore.search(queryVector, limit * 2);
|
|
32
32
|
const results = [];
|
|
33
33
|
for (const vr of vectorResults) {
|
|
34
34
|
const memory = this.sqlite.getMemory(vr.id);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retrieve.js","sourceRoot":"","sources":["../../src/memory/retrieve.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,8CAA8C;
|
|
1
|
+
{"version":3,"file":"retrieve.js","sourceRoot":"","sources":["../../src/memory/retrieve.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,8CAA8C;AAsB9C,MAAM,OAAO,eAAe;IAClB,MAAM,CAAgB;IACtB,WAAW,CAAgB;IAC3B,KAAK,CAAgB;IACrB,OAAO,CAAmC;IAElD,YACE,MAAqB,EACrB,WAA0B,EAC1B,KAAoB,EACpB,UAAkC,EAAE;QAEpC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG;YACb,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,CAAC,EAAE,gBAAgB;YACzD,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,CAAC,EAAE,gBAAgB;YACnE,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,EAAE;SACzB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAA4B,EAAE;QACxD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC;QACzD,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,UAA4B,EAAE;QAC9D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC;QACzD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QAE5E,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,KAAK,MAAM,EAAE,IAAI,aAAa,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAC5C,IAAI,MAAM,EAAE,CAAC;gBACX,iCAAiC;gBACjC,IAAI,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,KAAK,OAAO,CAAC,SAAS,EAAE,CAAC;oBAChE,SAAS;gBACX,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC;oBACX,MAAM;oBACN,KAAK,EAAE,EAAE,CAAC,KAAK;oBACf,MAAM,EAAE,QAAQ;iBACjB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,UAA4B,EAAE;QAC9D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC;QACzD,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QAE5B,4CAA4C;QAC5C,MAAM,CAAC,UAAU,EAAE,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACpD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC;YACpD,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC;SAC3D,CAAC,CAAC;QAEH,qCAAqC;QACrC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE3C,oBAAoB;QACpB,UAAU,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE;YAClC,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;YACpC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC;YAC7E,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,uBAAuB;QACvB,aAAa,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE;YACrC,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;YACpC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC;YAC7E,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,gDAAgD;QAChD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;aAC1C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;YACrB,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAE;YACzB,KAAK;YACL,MAAM,EAAE,QAAiB;SAC1B,CAAC,CAAC,CAAC;QAEN,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,SAAwB,EAAE,UAA4B,EAAE;QAChE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC;QACzD,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC;QAClE,OAAO,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;IAC/D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB,CAAC,SAAiB,EAAE,UAA4B,EAAE;QAK1E,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC;QAEpD,8BAA8B;QAC9B,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE;YAC/C,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;YAC5B,UAAU,EAAE,OAAO,CAAC,UAAU;SAC/B,CAAC,CAAC;QAEH,kCAAkC;QAClC,MAAM,aAAa,GAAG,aAAa;YACjC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC;YAClC,CAAC,CAAC,EAAE,CAAC;QAEP,uCAAuC;QACvC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAClD,MAAM,WAAW,GAAG,OAAO,EAAE,IAAI,IAAI,SAAS,CAAC;QAE/C,wCAAwC;QACxC,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE;YAC5D,SAAS;YACT,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;SAC7B,CAAC,CAAC;QAEH,OAAO;YACL,cAAc;YACd,aAAa;YACb,gBAAgB;SACjB,CAAC;IACJ,CAAC;CACF;AAiBD;;GAEG;AACH,MAAM,UAAU,yBAAyB,CACvC,OAIC,EACD,UAAyB,EAAE;IAE3B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,gBAAgB;IACxD,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,GAAG,CAAC;IAEnD,MAAM,KAAK,GAAa,CAAC,qBAAqB,EAAE,iCAAiC,EAAE,EAAE,CAAC,CAAC;IACvF,IAAI,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAE5C,mCAAmC;IACnC,MAAM,MAAM,GAAG;QACb,4EAA4E;QAC5E,2EAA2E;QAC3E,sBAAsB;KACvB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,+BAA+B;IAEvE,6CAA6C;IAC7C,MAAM,MAAM,GAAG,CAAC,IAAY,EAAW,EAAE;QACvC,IAAI,QAAQ,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,YAAY;QAC7C,OAAO,aAAa,GAAG,IAAI,CAAC,MAAM,GAAG,YAAY,IAAI,QAAQ,CAAC;IAChE,CAAC,CAAC;IAEF,sBAAsB;IACtB,IAAI,OAAO,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtC,MAAM,aAAa,GAAG,mBAAmB,CAAC;QAC1C,IAAI,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAC9B,aAAa,IAAI,aAAa,CAAC,MAAM,CAAC;YAEtC,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;gBAC5C,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBAClD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;gBACtE,MAAM,IAAI,GAAG,MAAM,IAAI,KAAK,OAAO,IAAI,CAAC;gBAExC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;oBAAE,MAAM,CAAC,4BAA4B;gBAEtD,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,KAAK,OAAO,EAAE,CAAC,CAAC;gBACrC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC;YAC/B,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,aAAa,IAAI,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,6BAA6B;IAC7B,IAAI,OAAO,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,MAAM,aAAa,GAAG,iCAAiC,CAAC;QACxD,IAAI,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;YAC5C,aAAa,IAAI,aAAa,CAAC,MAAM,CAAC;YAEtC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;gBAC5C,MAAM,IAAI,GAAG,KAAK,OAAO,CAAC,OAAO,IAAI,CAAC;gBAEtC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;oBAAE,MAAM;gBAEzB,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;gBACnC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC;YAC/B,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,aAAa,IAAI,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,0CAA0C;IAC1C,IAAI,OAAO,CAAC,gBAAgB,IAAI,OAAO,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpE,MAAM,aAAa,GAAG,0BAA0B,CAAC;QACjD,IAAI,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YACrC,aAAa,IAAI,aAAa,CAAC,MAAM,CAAC;YAEtC,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;gBAC9C,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;gBAC7E,MAAM,IAAI,GAAG,KAAK,OAAO,IAAI,CAAC;gBAE9B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;oBAAE,MAAM;gBAEzB,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC;gBAC3B,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC;YAC/B,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,4EAA4E,CAAC,CAAC;IACzF,KAAK,CAAC,IAAI,CAAC,2EAA2E,CAAC,CAAC;IACxF,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IAEnC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAe;IACzC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IAE5D,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IACnC,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,WAAW,CAAC;IACvC,IAAI,QAAQ,GAAG,CAAC;QAAE,OAAO,GAAG,QAAQ,WAAW,CAAC;IAChD,IAAI,QAAQ,GAAG,EAAE;QAAE,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,YAAY,CAAC;IAClE,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,EAAE,CAAC,aAAa,CAAC;AACnD,CAAC"}
|
|
@@ -6,11 +6,11 @@ import { join } from 'path';
|
|
|
6
6
|
import { tmpdir } from 'os';
|
|
7
7
|
import { MemoryRetriever, formatContextForInjection } from './retrieve.js';
|
|
8
8
|
import { SQLiteStorage } from '../storage/sqlite.js';
|
|
9
|
-
import {
|
|
9
|
+
import { SQLiteVecStorage } from '../storage/sqlite-vec.js';
|
|
10
10
|
describe('MemoryRetriever', () => {
|
|
11
11
|
let tempDir;
|
|
12
12
|
let sqlite;
|
|
13
|
-
let
|
|
13
|
+
let vectorStore;
|
|
14
14
|
let retriever;
|
|
15
15
|
// Mock embedding function for tests
|
|
16
16
|
const mockEmbed = async (text) => {
|
|
@@ -25,14 +25,14 @@ describe('MemoryRetriever', () => {
|
|
|
25
25
|
beforeEach(async () => {
|
|
26
26
|
tempDir = await mkdtemp(join(tmpdir(), 'memextend-retrieve-test-'));
|
|
27
27
|
sqlite = new SQLiteStorage(join(tempDir, 'test.db'));
|
|
28
|
-
|
|
29
|
-
retriever = new MemoryRetriever(sqlite,
|
|
28
|
+
vectorStore = await SQLiteVecStorage.create(join(tempDir, 'vectors.db'));
|
|
29
|
+
retriever = new MemoryRetriever(sqlite, vectorStore, mockEmbed);
|
|
30
30
|
// Seed test data
|
|
31
|
-
await seedTestMemories(sqlite,
|
|
31
|
+
await seedTestMemories(sqlite, vectorStore, mockEmbed);
|
|
32
32
|
});
|
|
33
33
|
afterEach(async () => {
|
|
34
34
|
sqlite.close();
|
|
35
|
-
await
|
|
35
|
+
await vectorStore.close();
|
|
36
36
|
await rm(tempDir, { recursive: true });
|
|
37
37
|
});
|
|
38
38
|
it('should retrieve memories by FTS', async () => {
|
|
@@ -123,7 +123,7 @@ describe('formatContextForInjection', () => {
|
|
|
123
123
|
});
|
|
124
124
|
});
|
|
125
125
|
// Helper functions
|
|
126
|
-
async function seedTestMemories(sqlite,
|
|
126
|
+
async function seedTestMemories(sqlite, vectorStore, embed) {
|
|
127
127
|
const memories = [
|
|
128
128
|
{
|
|
129
129
|
id: 'mem-1',
|
|
@@ -159,7 +159,7 @@ async function seedTestMemories(sqlite, lancedb, embed) {
|
|
|
159
159
|
for (const memory of memories) {
|
|
160
160
|
sqlite.insertMemory(memory);
|
|
161
161
|
const vector = await embed(memory.content);
|
|
162
|
-
await
|
|
162
|
+
await vectorStore.insertVector(memory.id, vector);
|
|
163
163
|
}
|
|
164
164
|
}
|
|
165
165
|
function createMemory(id, projectId, content) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retrieve.test.js","sourceRoot":"","sources":["../../src/memory/retrieve.test.ts"],"names":[],"mappings":"AAAA,4CAA4C;AAC5C,8CAA8C;AAE9C,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,eAAe,EAAE,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"retrieve.test.js","sourceRoot":"","sources":["../../src/memory/retrieve.test.ts"],"names":[],"mappings":"AAAA,4CAA4C;AAC5C,8CAA8C;AAE9C,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,eAAe,EAAE,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAG5D,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,IAAI,OAAe,CAAC;IACpB,IAAI,MAAqB,CAAC;IAC1B,IAAI,WAA6B,CAAC;IAClC,IAAI,SAA0B,CAAC;IAE/B,oCAAoC;IACpC,MAAM,SAAS,GAAG,KAAK,EAAE,IAAY,EAAqB,EAAE;QAC1D,yEAAyE;QACzE,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YACjD,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;QACrB,CAAC;QACD,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;IAC9E,CAAC,CAAC;IAEF,UAAU,CAAC,KAAK,IAAI,EAAE;QACpB,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,0BAA0B,CAAC,CAAC,CAAC;QACpE,MAAM,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;QACrD,WAAW,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;QACzE,SAAS,GAAG,IAAI,eAAe,CAAC,MAAM,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QAEhE,iBAAiB;QACjB,MAAM,gBAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC;QAC1B,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;QAC/C,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,eAAe,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACtE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,kBAAkB,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QAC/E,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QAC/D,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,oBAAoB,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACjF,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAC9C,sBAAsB;QACtB,MAAM,CAAC,aAAa,CAAC;YACnB,EAAE,EAAE,WAAW;YACf,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,uBAAuB;YAC7B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;QAEH,qBAAqB;QACrB,MAAM,CAAC,mBAAmB,CAAC;YACzB,EAAE,EAAE,QAAQ;YACZ,GAAG,EAAE,YAAY;YACjB,OAAO,EAAE,0BAA0B;YACnC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAC,WAAW,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAE3F,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAChE,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9F,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,OAAO,GAAG;YACd,cAAc,EAAE;gBACd,YAAY,CAAC,OAAO,EAAE,WAAW,EAAE,2CAA2C,CAAC;gBAC/E,YAAY,CAAC,OAAO,EAAE,WAAW,EAAE,yBAAyB,CAAC;aAC9D;YACD,aAAa,EAAE,EAAE;YACjB,gBAAgB,EAAE,EAAE;SACrB,CAAC;QAEF,MAAM,SAAS,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;QAErD,MAAM,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;QACnD,MAAM,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;QACpD,MAAM,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QAC3C,MAAM,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,OAAO,GAAG;YACd,cAAc,EAAE,EAAE;YAClB,aAAa,EAAE;gBACb,EAAE,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,YAAqB,EAAE,OAAO,EAAE,oBAAoB,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE;aACjH;YACD,gBAAgB,EAAE,EAAE;SACrB,CAAC;QAEF,MAAM,SAAS,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;QAErD,MAAM,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;QAChD,MAAM,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,OAAO,GAAG;YACd,cAAc,EAAE,EAAE;YAClB,aAAa,EAAE,EAAE;YACjB,gBAAgB,EAAE;gBAChB,EAAE,MAAM,EAAE,YAAY,CAAC,OAAO,EAAE,WAAW,EAAE,4BAA4B,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,QAAiB,EAAE;aACpH;SACF,CAAC;QAEF,MAAM,SAAS,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;QAErD,MAAM,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;QAClD,MAAM,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,mBAAmB;AACnB,KAAK,UAAU,gBAAgB,CAC7B,MAAqB,EACrB,WAA6B,EAC7B,KAA0C;IAE1C,MAAM,QAAQ,GAAa;QACzB;YACE,EAAE,EAAE,OAAO;YACX,SAAS,EAAE,WAAW;YACtB,OAAO,EAAE,iDAAiD;YAC1D,IAAI,EAAE,cAAc;YACpB,UAAU,EAAE,MAAM;YAClB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,SAAS,EAAE,QAAQ;YACnB,QAAQ,EAAE,IAAI;SACf;QACD;YACE,EAAE,EAAE,OAAO;YACX,SAAS,EAAE,WAAW;YACtB,OAAO,EAAE,qDAAqD;YAC9D,IAAI,EAAE,cAAc;YACpB,UAAU,EAAE,MAAM;YAClB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,SAAS,EAAE,QAAQ;YACnB,QAAQ,EAAE,IAAI;SACf;QACD;YACE,EAAE,EAAE,OAAO;YACX,SAAS,EAAE,WAAW;YACtB,OAAO,EAAE,4BAA4B;YACrC,IAAI,EAAE,cAAc;YACpB,UAAU,EAAE,MAAM;YAClB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,SAAS,EAAE,QAAQ;YACnB,QAAQ,EAAE,IAAI;SACf;KACF,CAAC;IAEF,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,WAAW,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACpD,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,EAAU,EAAE,SAAiB,EAAE,OAAe;IAClE,OAAO;QACL,EAAE;QACF,SAAS;QACT,OAAO;QACP,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,MAAM;QAClB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,SAAS,EAAE,IAAI;QACf,QAAQ,EAAE,IAAI;KACf,CAAC;AACJ,CAAC"}
|
package/dist/storage/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { SQLiteStorage } from './sqlite.js';
|
|
2
|
-
export {
|
|
3
|
-
export type { VectorSearchResult } from './
|
|
2
|
+
export { SQLiteVecStorage } from './sqlite-vec.js';
|
|
3
|
+
export type { VectorSearchResult } from './sqlite-vec.js';
|
|
4
4
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,YAAY,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC"}
|
package/dist/storage/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// packages/core/src/storage/index.ts
|
|
2
2
|
// Copyright (c) 2026 ZodTTD LLC. MIT License.
|
|
3
3
|
export { SQLiteStorage } from './sqlite.js';
|
|
4
|
-
export {
|
|
4
|
+
export { SQLiteVecStorage } from './sqlite-vec.js';
|
|
5
5
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAAA,qCAAqC;AACrC,8CAA8C;AAE9C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAAA,qCAAqC;AACrC,8CAA8C;AAE9C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC"}
|
|
@@ -2,13 +2,12 @@ export interface VectorSearchResult {
|
|
|
2
2
|
id: string;
|
|
3
3
|
score: number;
|
|
4
4
|
}
|
|
5
|
-
export declare class
|
|
5
|
+
export declare class SQLiteVecStorage {
|
|
6
6
|
private db;
|
|
7
|
-
private table;
|
|
8
7
|
private readonly tableName;
|
|
9
8
|
private readonly dimensions;
|
|
10
9
|
private constructor();
|
|
11
|
-
static create(dbPath: string): Promise<
|
|
10
|
+
static create(dbPath: string): Promise<SQLiteVecStorage>;
|
|
12
11
|
private initialize;
|
|
13
12
|
insertVector(id: string, vector: number[]): Promise<void>;
|
|
14
13
|
insertVectors(items: Array<{
|
|
@@ -20,16 +19,9 @@ export declare class LanceDBStorage {
|
|
|
20
19
|
getVectorCount(): Promise<number>;
|
|
21
20
|
getVectorsByIds(ids: string[]): Promise<Map<string, number[]>>;
|
|
22
21
|
close(): Promise<void>;
|
|
23
|
-
|
|
24
|
-
* Optimize the LanceDB table to reduce storage.
|
|
25
|
-
* This compacts files, prunes old versions, and optimizes indices.
|
|
26
|
-
* Should be called periodically (e.g., after many inserts or on cleanup command).
|
|
27
|
-
*
|
|
28
|
-
* @param cleanupOlderThan - Date before which old versions should be pruned (default: now)
|
|
29
|
-
*/
|
|
30
|
-
optimize(cleanupOlderThan?: Date): Promise<{
|
|
22
|
+
optimize(): Promise<{
|
|
31
23
|
compacted: number;
|
|
32
24
|
pruned: number;
|
|
33
25
|
} | null>;
|
|
34
26
|
}
|
|
35
|
-
//# sourceMappingURL=
|
|
27
|
+
//# sourceMappingURL=sqlite-vec.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sqlite-vec.d.ts","sourceRoot":"","sources":["../../src/storage/sqlite-vec.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,EAAE,CAAoB;IAC9B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAoB;IAC9C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAO;IAElC,OAAO;WAIM,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAkB9D,OAAO,CAAC,UAAU;IAWZ,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBzD,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAoB5E,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IA6B3E,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKvC,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC;IAMjC,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IA4B9D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAKtB,QAAQ,IAAI,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CAKxE"}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
// packages/core/src/storage/sqlite-vec.ts
|
|
2
|
+
// Copyright (c) 2026 ZodTTD LLC. MIT License.
|
|
3
|
+
import Database from 'better-sqlite3';
|
|
4
|
+
import * as sqliteVec from 'sqlite-vec';
|
|
5
|
+
export class SQLiteVecStorage {
|
|
6
|
+
db;
|
|
7
|
+
tableName = 'memory_vectors';
|
|
8
|
+
dimensions = 384;
|
|
9
|
+
constructor(db) {
|
|
10
|
+
this.db = db;
|
|
11
|
+
}
|
|
12
|
+
static async create(dbPath) {
|
|
13
|
+
// Handle both directory paths (legacy directory format) and file paths
|
|
14
|
+
// If path ends with 'vectors' (directory), convert to vectors.db file
|
|
15
|
+
let actualPath = dbPath;
|
|
16
|
+
if (dbPath.endsWith('vectors') || dbPath.endsWith('vectors/')) {
|
|
17
|
+
actualPath = dbPath.replace(/\/?$/, '.db');
|
|
18
|
+
}
|
|
19
|
+
const db = new Database(actualPath);
|
|
20
|
+
// Load sqlite-vec extension
|
|
21
|
+
sqliteVec.load(db);
|
|
22
|
+
const storage = new SQLiteVecStorage(db);
|
|
23
|
+
storage.initialize();
|
|
24
|
+
return storage;
|
|
25
|
+
}
|
|
26
|
+
initialize() {
|
|
27
|
+
// Create virtual table for vector search
|
|
28
|
+
// vec0 is the virtual table module from sqlite-vec
|
|
29
|
+
this.db.exec(`
|
|
30
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS ${this.tableName} USING vec0(
|
|
31
|
+
id TEXT PRIMARY KEY,
|
|
32
|
+
vector FLOAT[${this.dimensions}]
|
|
33
|
+
)
|
|
34
|
+
`);
|
|
35
|
+
}
|
|
36
|
+
async insertVector(id, vector) {
|
|
37
|
+
if (vector.length !== this.dimensions) {
|
|
38
|
+
throw new Error(`Vector must have ${this.dimensions} dimensions, got ${vector.length}`);
|
|
39
|
+
}
|
|
40
|
+
// Convert to Buffer for better-sqlite3 binding
|
|
41
|
+
const float32 = new Float32Array(vector);
|
|
42
|
+
const vectorBuffer = Buffer.from(float32.buffer);
|
|
43
|
+
const stmt = this.db.prepare(`
|
|
44
|
+
INSERT OR REPLACE INTO ${this.tableName} (id, vector)
|
|
45
|
+
VALUES (?, ?)
|
|
46
|
+
`);
|
|
47
|
+
stmt.run(id, vectorBuffer);
|
|
48
|
+
}
|
|
49
|
+
async insertVectors(items) {
|
|
50
|
+
const stmt = this.db.prepare(`
|
|
51
|
+
INSERT OR REPLACE INTO ${this.tableName} (id, vector)
|
|
52
|
+
VALUES (?, ?)
|
|
53
|
+
`);
|
|
54
|
+
const insertMany = this.db.transaction((items) => {
|
|
55
|
+
for (const item of items) {
|
|
56
|
+
if (item.vector.length !== this.dimensions) {
|
|
57
|
+
throw new Error(`Vector must have ${this.dimensions} dimensions, got ${item.vector.length}`);
|
|
58
|
+
}
|
|
59
|
+
const float32 = new Float32Array(item.vector);
|
|
60
|
+
const vectorBuffer = Buffer.from(float32.buffer);
|
|
61
|
+
stmt.run(item.id, vectorBuffer);
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
insertMany(items);
|
|
65
|
+
}
|
|
66
|
+
async search(vector, limit = 10) {
|
|
67
|
+
if (vector.length !== this.dimensions) {
|
|
68
|
+
throw new Error(`Query vector must have ${this.dimensions} dimensions, got ${vector.length}`);
|
|
69
|
+
}
|
|
70
|
+
const effectiveLimit = limit > 0 ? limit : 100;
|
|
71
|
+
const float32 = new Float32Array(vector);
|
|
72
|
+
const vectorBuffer = Buffer.from(float32.buffer);
|
|
73
|
+
// KNN search using sqlite-vec
|
|
74
|
+
// distance is L2 (Euclidean) by default, lower = more similar
|
|
75
|
+
const stmt = this.db.prepare(`
|
|
76
|
+
SELECT id, distance
|
|
77
|
+
FROM ${this.tableName}
|
|
78
|
+
WHERE vector MATCH ?
|
|
79
|
+
ORDER BY distance
|
|
80
|
+
LIMIT ?
|
|
81
|
+
`);
|
|
82
|
+
const rows = stmt.all(vectorBuffer, effectiveLimit);
|
|
83
|
+
// Convert distance to similarity score (1 / (1 + distance))
|
|
84
|
+
// This gives us a score between 0 and 1, where 1 is most similar
|
|
85
|
+
return rows.map(row => ({
|
|
86
|
+
id: row.id,
|
|
87
|
+
score: 1 / (1 + row.distance)
|
|
88
|
+
}));
|
|
89
|
+
}
|
|
90
|
+
async deleteVector(id) {
|
|
91
|
+
const stmt = this.db.prepare(`DELETE FROM ${this.tableName} WHERE id = ?`);
|
|
92
|
+
stmt.run(id);
|
|
93
|
+
}
|
|
94
|
+
async getVectorCount() {
|
|
95
|
+
const stmt = this.db.prepare(`SELECT COUNT(*) as count FROM ${this.tableName}`);
|
|
96
|
+
const result = stmt.get();
|
|
97
|
+
return result.count;
|
|
98
|
+
}
|
|
99
|
+
async getVectorsByIds(ids) {
|
|
100
|
+
const result = new Map();
|
|
101
|
+
if (ids.length === 0)
|
|
102
|
+
return result;
|
|
103
|
+
// Query in batches to avoid SQL limits
|
|
104
|
+
const BATCH_SIZE = 100;
|
|
105
|
+
for (let i = 0; i < ids.length; i += BATCH_SIZE) {
|
|
106
|
+
const batch = ids.slice(i, i + BATCH_SIZE);
|
|
107
|
+
const placeholders = batch.map(() => '?').join(',');
|
|
108
|
+
const stmt = this.db.prepare(`
|
|
109
|
+
SELECT id, vector FROM ${this.tableName}
|
|
110
|
+
WHERE id IN (${placeholders})
|
|
111
|
+
`);
|
|
112
|
+
const rows = stmt.all(...batch);
|
|
113
|
+
for (const row of rows) {
|
|
114
|
+
// Convert ArrayBuffer back to number array
|
|
115
|
+
const vector = Array.from(new Float32Array(row.vector));
|
|
116
|
+
result.set(row.id, vector);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return result;
|
|
120
|
+
}
|
|
121
|
+
async close() {
|
|
122
|
+
this.db.close();
|
|
123
|
+
}
|
|
124
|
+
// No optimize needed - SQLite handles this automatically!
|
|
125
|
+
async optimize() {
|
|
126
|
+
// Run VACUUM to reclaim space (optional, SQLite auto-manages)
|
|
127
|
+
this.db.exec('VACUUM');
|
|
128
|
+
return { compacted: 0, pruned: 0 };
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
//# sourceMappingURL=sqlite-vec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sqlite-vec.js","sourceRoot":"","sources":["../../src/storage/sqlite-vec.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,8CAA8C;AAE9C,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,KAAK,SAAS,MAAM,YAAY,CAAC;AAOxC,MAAM,OAAO,gBAAgB;IACnB,EAAE,CAAoB;IACb,SAAS,GAAG,gBAAgB,CAAC;IAC7B,UAAU,GAAG,GAAG,CAAC;IAElC,YAAoB,EAAqB;QACvC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACf,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAc;QAChC,uEAAuE;QACvE,sEAAsE;QACtE,IAAI,UAAU,GAAG,MAAM,CAAC;QACxB,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9D,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,UAAU,CAAC,CAAC;QAEpC,4BAA4B;QAC5B,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEnB,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC,EAAE,CAAC,CAAC;QACzC,OAAO,CAAC,UAAU,EAAE,CAAC;QACrB,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,UAAU;QAChB,yCAAyC;QACzC,mDAAmD;QACnD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;2CAC0B,IAAI,CAAC,SAAS;;uBAElC,IAAI,CAAC,UAAU;;KAEjC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,EAAU,EAAE,MAAgB;QAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,CAAC,UAAU,oBAAoB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,+CAA+C;QAC/C,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAEjD,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;+BACF,IAAI,CAAC,SAAS;;KAExC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAA8C;QAChE,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;+BACF,IAAI,CAAC,SAAS;;KAExC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,KAA8C,EAAE,EAAE;YACxF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;oBAC3C,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,CAAC,UAAU,oBAAoB,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC/F,CAAC;gBACD,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBACjD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;YAClC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,UAAU,CAAC,KAAK,CAAC,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAgB,EAAE,QAAgB,EAAE;QAC/C,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,UAAU,oBAAoB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAChG,CAAC;QAED,MAAM,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAEjD,8BAA8B;QAC9B,8DAA8D;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;aAEpB,IAAI,CAAC,SAAS;;;;KAItB,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,cAAc,CAA4C,CAAC;QAE/F,4DAA4D;QAC5D,iEAAiE;QACjE,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACtB,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC;SAC9B,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,EAAU;QAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,IAAI,CAAC,SAAS,eAAe,CAAC,CAAC;QAC3E,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACf,CAAC;IAED,KAAK,CAAC,cAAc;QAClB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,iCAAiC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAChF,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAuB,CAAC;QAC/C,OAAO,MAAM,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,GAAa;QACjC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC3C,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC;QAEpC,uCAAuC;QACvC,MAAM,UAAU,GAAG,GAAG,CAAC;QAEvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;YAC3C,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEpD,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;iCACF,IAAI,CAAC,SAAS;uBACxB,YAAY;OAC5B,CAAC,CAAC;YAEH,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAA+C,CAAC;YAE9E,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,2CAA2C;gBAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;gBACxD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;IAED,0DAA0D;IAC1D,KAAK,CAAC,QAAQ;QACZ,8DAA8D;QAC9D,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvB,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IACrC,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@memextend/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Core memory storage and retrieval for memextend - local AI memory extension",
|
|
5
5
|
"author": "ZodTTD LLC <repo@zodttd.com>",
|
|
6
6
|
"license": "MIT",
|
|
@@ -19,13 +19,13 @@
|
|
|
19
19
|
"test:watch": "vitest"
|
|
20
20
|
},
|
|
21
21
|
"dependencies": {
|
|
22
|
-
"
|
|
23
|
-
"better-sqlite3": "^12.6.2",
|
|
22
|
+
"better-sqlite3": "^11.10.0",
|
|
24
23
|
"node-llama-cpp": "^3.0.0",
|
|
24
|
+
"sqlite-vec": "^0.1.6",
|
|
25
25
|
"uuid": "^9.0.0"
|
|
26
26
|
},
|
|
27
27
|
"devDependencies": {
|
|
28
|
-
"@types/better-sqlite3": "^7.6.
|
|
28
|
+
"@types/better-sqlite3": "^7.6.13",
|
|
29
29
|
"@types/uuid": "^9.0.7"
|
|
30
30
|
},
|
|
31
31
|
"repository": {
|
|
@@ -44,7 +44,6 @@
|
|
|
44
44
|
"claude",
|
|
45
45
|
"vector-search",
|
|
46
46
|
"sqlite",
|
|
47
|
-
"lancedb",
|
|
48
47
|
"embeddings",
|
|
49
48
|
"semantic-search"
|
|
50
49
|
],
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"lancedb.d.ts","sourceRoot":"","sources":["../../src/storage/lancedb.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,EAAE,CAAqB;IAC/B,OAAO,CAAC,KAAK,CAA8B;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAc;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAO;IAElC,OAAO;WAIM,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;YAO9C,UAAU;IAQlB,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAczD,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAc5E,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAmB3E,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOvC,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC;IAKjC,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IA8B9D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;;;;;OAMG;IACG,QAAQ,CAAC,gBAAgB,CAAC,EAAE,IAAI,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CAsB/F"}
|
package/dist/storage/lancedb.js
DELETED
|
@@ -1,131 +0,0 @@
|
|
|
1
|
-
// packages/core/src/storage/lancedb.ts
|
|
2
|
-
// Copyright (c) 2026 ZodTTD LLC. MIT License.
|
|
3
|
-
import * as lancedb from '@lancedb/lancedb';
|
|
4
|
-
export class LanceDBStorage {
|
|
5
|
-
db;
|
|
6
|
-
table = null;
|
|
7
|
-
tableName = 'memories';
|
|
8
|
-
dimensions = 384;
|
|
9
|
-
constructor(db) {
|
|
10
|
-
this.db = db;
|
|
11
|
-
}
|
|
12
|
-
static async create(dbPath) {
|
|
13
|
-
const db = await lancedb.connect(dbPath);
|
|
14
|
-
const storage = new LanceDBStorage(db);
|
|
15
|
-
await storage.initialize();
|
|
16
|
-
return storage;
|
|
17
|
-
}
|
|
18
|
-
async initialize() {
|
|
19
|
-
const tableNames = await this.db.tableNames();
|
|
20
|
-
if (tableNames.includes(this.tableName)) {
|
|
21
|
-
this.table = await this.db.openTable(this.tableName);
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
async insertVector(id, vector) {
|
|
25
|
-
if (vector.length !== this.dimensions) {
|
|
26
|
-
throw new Error(`Vector must have ${this.dimensions} dimensions, got ${vector.length}`);
|
|
27
|
-
}
|
|
28
|
-
const data = [{ id, vector }];
|
|
29
|
-
if (!this.table) {
|
|
30
|
-
this.table = await this.db.createTable(this.tableName, data);
|
|
31
|
-
}
|
|
32
|
-
else {
|
|
33
|
-
await this.table.add(data);
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
async insertVectors(items) {
|
|
37
|
-
for (const item of items) {
|
|
38
|
-
if (item.vector.length !== this.dimensions) {
|
|
39
|
-
throw new Error(`Vector must have ${this.dimensions} dimensions, got ${item.vector.length}`);
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
if (!this.table) {
|
|
43
|
-
this.table = await this.db.createTable(this.tableName, items);
|
|
44
|
-
}
|
|
45
|
-
else {
|
|
46
|
-
await this.table.add(items);
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
async search(vector, limit = 10) {
|
|
50
|
-
if (!this.table) {
|
|
51
|
-
return [];
|
|
52
|
-
}
|
|
53
|
-
// LanceDB requires k > 0, so default 0 to a reasonable max
|
|
54
|
-
const effectiveLimit = limit > 0 ? limit : 100;
|
|
55
|
-
const results = await this.table
|
|
56
|
-
.vectorSearch(vector)
|
|
57
|
-
.limit(effectiveLimit)
|
|
58
|
-
.toArray();
|
|
59
|
-
return results.map(row => ({
|
|
60
|
-
id: row.id,
|
|
61
|
-
score: 1 - row._distance // Convert distance to similarity
|
|
62
|
-
}));
|
|
63
|
-
}
|
|
64
|
-
async deleteVector(id) {
|
|
65
|
-
if (!this.table)
|
|
66
|
-
return;
|
|
67
|
-
// Sanitize the ID to prevent injection (escape single quotes)
|
|
68
|
-
const sanitizedId = id.replace(/'/g, "''");
|
|
69
|
-
await this.table.delete(`id = '${sanitizedId}'`);
|
|
70
|
-
}
|
|
71
|
-
async getVectorCount() {
|
|
72
|
-
if (!this.table)
|
|
73
|
-
return 0;
|
|
74
|
-
return await this.table.countRows();
|
|
75
|
-
}
|
|
76
|
-
async getVectorsByIds(ids) {
|
|
77
|
-
const result = new Map();
|
|
78
|
-
if (!this.table || ids.length === 0)
|
|
79
|
-
return result;
|
|
80
|
-
// Batch queries to avoid extremely long WHERE clauses
|
|
81
|
-
const BATCH_SIZE = 100;
|
|
82
|
-
try {
|
|
83
|
-
for (let i = 0; i < ids.length; i += BATCH_SIZE) {
|
|
84
|
-
const batch = ids.slice(i, i + BATCH_SIZE);
|
|
85
|
-
// Sanitize IDs to prevent injection
|
|
86
|
-
const sanitizedIds = batch.map(id => id.replace(/'/g, "''"));
|
|
87
|
-
const filter = sanitizedIds.map(id => `id = '${id}'`).join(' OR ');
|
|
88
|
-
const rows = await this.table
|
|
89
|
-
.query()
|
|
90
|
-
.where(filter)
|
|
91
|
-
.toArray();
|
|
92
|
-
for (const row of rows) {
|
|
93
|
-
result.set(row.id, row.vector);
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
catch {
|
|
98
|
-
// Return empty map on error
|
|
99
|
-
}
|
|
100
|
-
return result;
|
|
101
|
-
}
|
|
102
|
-
async close() {
|
|
103
|
-
// LanceDB doesn't require explicit close
|
|
104
|
-
}
|
|
105
|
-
/**
|
|
106
|
-
* Optimize the LanceDB table to reduce storage.
|
|
107
|
-
* This compacts files, prunes old versions, and optimizes indices.
|
|
108
|
-
* Should be called periodically (e.g., after many inserts or on cleanup command).
|
|
109
|
-
*
|
|
110
|
-
* @param cleanupOlderThan - Date before which old versions should be pruned (default: now)
|
|
111
|
-
*/
|
|
112
|
-
async optimize(cleanupOlderThan) {
|
|
113
|
-
if (!this.table)
|
|
114
|
-
return null;
|
|
115
|
-
try {
|
|
116
|
-
// Use type assertion since optimize isn't in the type definitions
|
|
117
|
-
const table = this.table;
|
|
118
|
-
// Default to now to prune all old versions immediately
|
|
119
|
-
const stats = await table.optimize({ cleanupOlderThan: cleanupOlderThan ?? new Date() });
|
|
120
|
-
return {
|
|
121
|
-
compacted: stats?.compaction?.filesRemoved ?? 0,
|
|
122
|
-
pruned: stats?.prune?.versionsRemoved ?? 0,
|
|
123
|
-
};
|
|
124
|
-
}
|
|
125
|
-
catch (error) {
|
|
126
|
-
console.error('[memextend] LanceDB optimize failed:', error);
|
|
127
|
-
return null;
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
//# sourceMappingURL=lancedb.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"lancedb.js","sourceRoot":"","sources":["../../src/storage/lancedb.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,8CAA8C;AAE9C,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAO5C,MAAM,OAAO,cAAc;IACjB,EAAE,CAAqB;IACvB,KAAK,GAAyB,IAAI,CAAC;IAC1B,SAAS,GAAG,UAAU,CAAC;IACvB,UAAU,GAAG,GAAG,CAAC;IAElC,YAAoB,EAAsB;QACxC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACf,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAc;QAChC,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC3B,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,UAAU;QACtB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC;QAE9C,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACxC,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,EAAU,EAAE,MAAgB;QAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,CAAC,UAAU,oBAAoB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAE9B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAC/D,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAA8C;QAChE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC3C,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,CAAC,UAAU,oBAAoB,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/F,CAAC;QACH,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAChE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAgB,EAAE,QAAgB,EAAE;QAC/C,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,2DAA2D;QAC3D,MAAM,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;QAE/C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK;aAC7B,YAAY,CAAC,MAAM,CAAC;aACpB,KAAK,CAAC,cAAc,CAAC;aACrB,OAAO,EAAE,CAAC;QAEb,OAAQ,OAA0C,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7D,EAAE,EAAE,GAAG,CAAC,EAAY;YACpB,KAAK,EAAE,CAAC,GAAI,GAAG,CAAC,SAAoB,CAAC,iCAAiC;SACvE,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,EAAU;QAC3B,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QACxB,8DAA8D;QAC9D,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3C,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,WAAW,GAAG,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,cAAc;QAClB,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,CAAC,CAAC;QAC1B,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,GAAa;QACjC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC3C,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC;QAEnD,sDAAsD;QACtD,MAAM,UAAU,GAAG,GAAG,CAAC;QAEvB,IAAI,CAAC;YACH,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;gBAChD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;gBAC3C,oCAAoC;gBACpC,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC7D,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAEnE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK;qBAC1B,KAAK,EAAE;qBACP,KAAK,CAAC,MAAM,CAAC;qBACb,OAAO,EAAE,CAAC;gBAEb,KAAK,MAAM,GAAG,IAAI,IAAsC,EAAE,CAAC;oBACzD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAY,EAAE,GAAG,CAAC,MAAkB,CAAC,CAAC;gBACvD,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4BAA4B;QAC9B,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,yCAAyC;IAC3C,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,gBAAuB;QACpC,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QAE7B,IAAI,CAAC;YACH,kEAAkE;YAClE,MAAM,KAAK,GAAG,IAAI,CAAC,KAKlB,CAAC;YACF,uDAAuD;YACvD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,EAAE,gBAAgB,EAAE,gBAAgB,IAAI,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;YACzF,OAAO;gBACL,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,YAAY,IAAI,CAAC;gBAC/C,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,eAAe,IAAI,CAAC;aAC3C,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,sCAAsC,EAAE,KAAK,CAAC,CAAC;YAC7D,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"lancedb.test.d.ts","sourceRoot":"","sources":["../../src/storage/lancedb.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,67 +0,0 @@
|
|
|
1
|
-
// packages/core/src/storage/lancedb.test.ts
|
|
2
|
-
// Copyright (c) 2026 ZodTTD LLC. MIT License.
|
|
3
|
-
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
4
|
-
import { mkdtemp, rm } from 'fs/promises';
|
|
5
|
-
import { join } from 'path';
|
|
6
|
-
import { tmpdir } from 'os';
|
|
7
|
-
import { LanceDBStorage } from './lancedb.js';
|
|
8
|
-
describe('LanceDBStorage', () => {
|
|
9
|
-
let tempDir;
|
|
10
|
-
let storage;
|
|
11
|
-
beforeEach(async () => {
|
|
12
|
-
tempDir = await mkdtemp(join(tmpdir(), 'memextend-lance-test-'));
|
|
13
|
-
storage = await LanceDBStorage.create(tempDir);
|
|
14
|
-
});
|
|
15
|
-
afterEach(async () => {
|
|
16
|
-
await storage.close();
|
|
17
|
-
await rm(tempDir, { recursive: true });
|
|
18
|
-
});
|
|
19
|
-
it('should insert and search vectors', async () => {
|
|
20
|
-
const vector = new Array(384).fill(0).map(() => Math.random());
|
|
21
|
-
await storage.insertVector('mem-1', vector);
|
|
22
|
-
const results = await storage.search(vector, 5);
|
|
23
|
-
expect(results.length).toBeGreaterThan(0);
|
|
24
|
-
expect(results[0].id).toBe('mem-1');
|
|
25
|
-
});
|
|
26
|
-
it('should return similar vectors first', async () => {
|
|
27
|
-
const baseVector = new Array(384).fill(0.5);
|
|
28
|
-
const similarVector = new Array(384).fill(0.5).map(v => v + (Math.random() - 0.5) * 0.1);
|
|
29
|
-
const differentVector = new Array(384).fill(0).map(() => Math.random());
|
|
30
|
-
await storage.insertVector('similar', similarVector);
|
|
31
|
-
await storage.insertVector('different', differentVector);
|
|
32
|
-
const results = await storage.search(baseVector, 2);
|
|
33
|
-
expect(results[0].id).toBe('similar');
|
|
34
|
-
});
|
|
35
|
-
it('should validate vector dimensions', async () => {
|
|
36
|
-
const wrongDimensions = new Array(100).fill(0.5);
|
|
37
|
-
await expect(storage.insertVector('wrong', wrongDimensions))
|
|
38
|
-
.rejects.toThrow('Vector must have 384 dimensions');
|
|
39
|
-
});
|
|
40
|
-
it('should batch insert vectors', async () => {
|
|
41
|
-
const items = [
|
|
42
|
-
{ id: 'batch-1', vector: new Array(384).fill(0.1) },
|
|
43
|
-
{ id: 'batch-2', vector: new Array(384).fill(0.2) },
|
|
44
|
-
{ id: 'batch-3', vector: new Array(384).fill(0.3) },
|
|
45
|
-
];
|
|
46
|
-
await storage.insertVectors(items);
|
|
47
|
-
const count = await storage.getVectorCount();
|
|
48
|
-
expect(count).toBe(3);
|
|
49
|
-
});
|
|
50
|
-
it('should delete vectors', async () => {
|
|
51
|
-
const vector = new Array(384).fill(0.5);
|
|
52
|
-
await storage.insertVector('to-delete', vector);
|
|
53
|
-
await storage.deleteVector('to-delete');
|
|
54
|
-
const results = await storage.search(vector, 5);
|
|
55
|
-
const found = results.find(r => r.id === 'to-delete');
|
|
56
|
-
expect(found).toBeUndefined();
|
|
57
|
-
});
|
|
58
|
-
it('should return empty results when no table exists', async () => {
|
|
59
|
-
const freshDir = await mkdtemp(join(tmpdir(), 'memextend-lance-empty-'));
|
|
60
|
-
const freshStorage = await LanceDBStorage.create(freshDir);
|
|
61
|
-
const results = await freshStorage.search(new Array(384).fill(0.5), 5);
|
|
62
|
-
expect(results).toEqual([]);
|
|
63
|
-
await freshStorage.close();
|
|
64
|
-
await rm(freshDir, { recursive: true });
|
|
65
|
-
});
|
|
66
|
-
});
|
|
67
|
-
//# sourceMappingURL=lancedb.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"lancedb.test.js","sourceRoot":"","sources":["../../src/storage/lancedb.test.ts"],"names":[],"mappings":"AAAA,4CAA4C;AAC5C,8CAA8C;AAE9C,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,IAAI,OAAe,CAAC;IACpB,IAAI,OAAuB,CAAC;IAE5B,UAAU,CAAC,KAAK,IAAI,EAAE;QACpB,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,uBAAuB,CAAC,CAAC,CAAC;QACjE,OAAO,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACtB,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QAChD,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAE/D,MAAM,OAAO,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAE5C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;QACnD,MAAM,UAAU,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,aAAa,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;QACzF,MAAM,eAAe,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAExE,MAAM,OAAO,CAAC,YAAY,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;QACrD,MAAM,OAAO,CAAC,YAAY,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;QAEzD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QACpD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QACjD,MAAM,eAAe,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEjD,MAAM,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;aACzD,OAAO,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC3C,MAAM,KAAK,GAAG;YACZ,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;YACnD,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;YACnD,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;SACpD,CAAC;QAEF,MAAM,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAEnC,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,cAAc,EAAE,CAAC;QAC7C,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uBAAuB,EAAE,KAAK,IAAI,EAAE;QACrC,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,OAAO,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAEhD,MAAM,OAAO,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;QAExC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,WAAW,CAAC,CAAC;QACtD,MAAM,CAAC,KAAK,CAAC,CAAC,aAAa,EAAE,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QAChE,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,wBAAwB,CAAC,CAAC,CAAC;QACzE,MAAM,YAAY,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAE3D,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACvE,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAE5B,MAAM,YAAY,CAAC,KAAK,EAAE,CAAC;QAC3B,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|