@memextend/core 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -20,5 +20,16 @@ export declare class LanceDBStorage {
|
|
|
20
20
|
getVectorCount(): Promise<number>;
|
|
21
21
|
getVectorsByIds(ids: string[]): Promise<Map<string, number[]>>;
|
|
22
22
|
close(): Promise<void>;
|
|
23
|
+
/**
|
|
24
|
+
* Optimize the LanceDB table to reduce storage.
|
|
25
|
+
* This compacts files, prunes old versions, and optimizes indices.
|
|
26
|
+
* Should be called periodically (e.g., after many inserts or on cleanup command).
|
|
27
|
+
*
|
|
28
|
+
* @param cleanupOlderThan - Date before which old versions should be pruned (default: now)
|
|
29
|
+
*/
|
|
30
|
+
optimize(cleanupOlderThan?: Date): Promise<{
|
|
31
|
+
compacted: number;
|
|
32
|
+
pruned: number;
|
|
33
|
+
} | null>;
|
|
23
34
|
}
|
|
24
35
|
//# sourceMappingURL=lancedb.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lancedb.d.ts","sourceRoot":"","sources":["../../src/storage/lancedb.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,EAAE,CAAqB;IAC/B,OAAO,CAAC,KAAK,CAA8B;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAc;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAO;IAElC,OAAO;WAIM,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;YAO9C,UAAU;IAQlB,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"lancedb.d.ts","sourceRoot":"","sources":["../../src/storage/lancedb.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,EAAE,CAAqB;IAC/B,OAAO,CAAC,KAAK,CAA8B;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAc;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAO;IAElC,OAAO;WAIM,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;YAO9C,UAAU;IAQlB,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBzD,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAiB5E,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAmB3E,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOvC,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC;IAKjC,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IA8B9D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;;;;;OAMG;IACG,QAAQ,CAAC,gBAAgB,CAAC,EAAE,IAAI,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CAsB/F"}
|
package/dist/storage/lancedb.js
CHANGED
|
@@ -32,6 +32,8 @@ export class LanceDBStorage {
|
|
|
32
32
|
else {
|
|
33
33
|
await this.table.add(data);
|
|
34
34
|
}
|
|
35
|
+
// Auto-optimize after every insert to prevent version bloat
|
|
36
|
+
await this.optimize();
|
|
35
37
|
}
|
|
36
38
|
async insertVectors(items) {
|
|
37
39
|
for (const item of items) {
|
|
@@ -45,6 +47,8 @@ export class LanceDBStorage {
|
|
|
45
47
|
else {
|
|
46
48
|
await this.table.add(items);
|
|
47
49
|
}
|
|
50
|
+
// Auto-optimize after every insert to prevent version bloat
|
|
51
|
+
await this.optimize();
|
|
48
52
|
}
|
|
49
53
|
async search(vector, limit = 10) {
|
|
50
54
|
if (!this.table) {
|
|
@@ -102,5 +106,30 @@ export class LanceDBStorage {
|
|
|
102
106
|
async close() {
|
|
103
107
|
// LanceDB doesn't require explicit close
|
|
104
108
|
}
|
|
109
|
+
/**
|
|
110
|
+
* Optimize the LanceDB table to reduce storage.
|
|
111
|
+
* This compacts files, prunes old versions, and optimizes indices.
|
|
112
|
+
* Should be called periodically (e.g., after many inserts or on cleanup command).
|
|
113
|
+
*
|
|
114
|
+
* @param cleanupOlderThan - Date before which old versions should be pruned (default: now)
|
|
115
|
+
*/
|
|
116
|
+
async optimize(cleanupOlderThan) {
|
|
117
|
+
if (!this.table)
|
|
118
|
+
return null;
|
|
119
|
+
try {
|
|
120
|
+
// Use type assertion since optimize isn't in the type definitions
|
|
121
|
+
const table = this.table;
|
|
122
|
+
// Default to now to prune all old versions immediately
|
|
123
|
+
const stats = await table.optimize({ cleanupOlderThan: cleanupOlderThan ?? new Date() });
|
|
124
|
+
return {
|
|
125
|
+
compacted: stats?.compaction?.filesRemoved ?? 0,
|
|
126
|
+
pruned: stats?.prune?.versionsRemoved ?? 0,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
console.error('[memextend] LanceDB optimize failed:', error);
|
|
131
|
+
return null;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
105
134
|
}
|
|
106
135
|
//# sourceMappingURL=lancedb.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lancedb.js","sourceRoot":"","sources":["../../src/storage/lancedb.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,8CAA8C;AAE9C,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAO5C,MAAM,OAAO,cAAc;IACjB,EAAE,CAAqB;IACvB,KAAK,GAAyB,IAAI,CAAC;IAC1B,SAAS,GAAG,UAAU,CAAC;IACvB,UAAU,GAAG,GAAG,CAAC;IAElC,YAAoB,EAAsB;QACxC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACf,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAc;QAChC,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC3B,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,UAAU;QACtB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC;QAE9C,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACxC,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,EAAU,EAAE,MAAgB;QAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,CAAC,UAAU,oBAAoB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAE9B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAC/D,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;
|
|
1
|
+
{"version":3,"file":"lancedb.js","sourceRoot":"","sources":["../../src/storage/lancedb.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,8CAA8C;AAE9C,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAO5C,MAAM,OAAO,cAAc;IACjB,EAAE,CAAqB;IACvB,KAAK,GAAyB,IAAI,CAAC;IAC1B,SAAS,GAAG,UAAU,CAAC;IACvB,UAAU,GAAG,GAAG,CAAC;IAElC,YAAoB,EAAsB;QACxC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACf,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAc;QAChC,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC3B,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,UAAU;QACtB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC;QAE9C,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACxC,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,EAAU,EAAE,MAAgB;QAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,CAAC,UAAU,oBAAoB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAE9B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAC/D,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QAED,4DAA4D;QAC5D,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAA8C;QAChE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC3C,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,CAAC,UAAU,oBAAoB,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/F,CAAC;QACH,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAChE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;QAED,4DAA4D;QAC5D,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAgB,EAAE,QAAgB,EAAE;QAC/C,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,2DAA2D;QAC3D,MAAM,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;QAE/C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK;aAC7B,YAAY,CAAC,MAAM,CAAC;aACpB,KAAK,CAAC,cAAc,CAAC;aACrB,OAAO,EAAE,CAAC;QAEb,OAAQ,OAA0C,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC7D,EAAE,EAAE,GAAG,CAAC,EAAY;YACpB,KAAK,EAAE,CAAC,GAAI,GAAG,CAAC,SAAoB,CAAC,iCAAiC;SACvE,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,EAAU;QAC3B,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QACxB,8DAA8D;QAC9D,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3C,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,WAAW,GAAG,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,cAAc;QAClB,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,CAAC,CAAC;QAC1B,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,GAAa;QACjC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC3C,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC;QAEnD,sDAAsD;QACtD,MAAM,UAAU,GAAG,GAAG,CAAC;QAEvB,IAAI,CAAC;YACH,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;gBAChD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;gBAC3C,oCAAoC;gBACpC,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC7D,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAEnE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK;qBAC1B,KAAK,EAAE;qBACP,KAAK,CAAC,MAAM,CAAC;qBACb,OAAO,EAAE,CAAC;gBAEb,KAAK,MAAM,GAAG,IAAI,IAAsC,EAAE,CAAC;oBACzD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAY,EAAE,GAAG,CAAC,MAAkB,CAAC,CAAC;gBACvD,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4BAA4B;QAC9B,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,yCAAyC;IAC3C,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,QAAQ,CAAC,gBAAuB;QACpC,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QAE7B,IAAI,CAAC;YACH,kEAAkE;YAClE,MAAM,KAAK,GAAG,IAAI,CAAC,KAKlB,CAAC;YACF,uDAAuD;YACvD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,EAAE,gBAAgB,EAAE,gBAAgB,IAAI,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;YACzF,OAAO;gBACL,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,YAAY,IAAI,CAAC;gBAC/C,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,eAAe,IAAI,CAAC;aAC3C,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,sCAAsC,EAAE,KAAK,CAAC,CAAC;YAC7D,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@memextend/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Core memory storage and retrieval for memextend - local AI memory extension",
|
|
5
5
|
"author": "ZodTTD LLC <repo@zodttd.com>",
|
|
6
6
|
"license": "MIT",
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
"test:watch": "vitest"
|
|
20
20
|
},
|
|
21
21
|
"dependencies": {
|
|
22
|
-
"@lancedb/lancedb": "^0.
|
|
22
|
+
"@lancedb/lancedb": "^0.23.0",
|
|
23
23
|
"better-sqlite3": "^12.6.2",
|
|
24
24
|
"node-llama-cpp": "^3.0.0",
|
|
25
25
|
"uuid": "^9.0.0"
|