om-memory-system 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +587 -0
  3. package/dist/adapters/opencode/auto-capture-host.d.ts +3 -0
  4. package/dist/adapters/opencode/auto-capture-host.js +29 -0
  5. package/dist/adapters/opencode/auto-capture-summary.d.ts +4 -0
  6. package/dist/adapters/opencode/auto-capture-summary.js +147 -0
  7. package/dist/adapters/opencode/conversation.d.ts +2 -0
  8. package/dist/adapters/opencode/conversation.js +50 -0
  9. package/dist/adapters/opencode/user-prompt.d.ts +9 -0
  10. package/dist/adapters/opencode/user-prompt.js +26 -0
  11. package/dist/adapters/pi/capture.d.ts +41 -0
  12. package/dist/adapters/pi/capture.js +91 -0
  13. package/dist/adapters/pi/conversation.d.ts +38 -0
  14. package/dist/adapters/pi/conversation.js +118 -0
  15. package/dist/adapters/pi/extension.d.ts +18 -0
  16. package/dist/adapters/pi/extension.js +234 -0
  17. package/dist/adapters/pi/import-command.d.ts +29 -0
  18. package/dist/adapters/pi/import-command.js +232 -0
  19. package/dist/adapters/pi/profile.d.ts +39 -0
  20. package/dist/adapters/pi/profile.js +148 -0
  21. package/dist/adapters/pi/provider.d.ts +50 -0
  22. package/dist/adapters/pi/provider.js +77 -0
  23. package/dist/config.d.ts +290 -0
  24. package/dist/config.js +742 -0
  25. package/dist/core/capture-context.d.ts +4 -0
  26. package/dist/core/capture-context.js +132 -0
  27. package/dist/core/capture.d.ts +15 -0
  28. package/dist/core/capture.js +68 -0
  29. package/dist/core/extraction.d.ts +80 -0
  30. package/dist/core/extraction.js +140 -0
  31. package/dist/core/host.d.ts +52 -0
  32. package/dist/core/host.js +1 -0
  33. package/dist/core/memory-operations.d.ts +27 -0
  34. package/dist/core/memory-operations.js +247 -0
  35. package/dist/core/retrieval.d.ts +12 -0
  36. package/dist/core/retrieval.js +72 -0
  37. package/dist/importer/discovery.d.ts +31 -0
  38. package/dist/importer/discovery.js +88 -0
  39. package/dist/importer/importer.d.ts +87 -0
  40. package/dist/importer/importer.js +260 -0
  41. package/dist/importer/ledger.d.ts +44 -0
  42. package/dist/importer/ledger.js +137 -0
  43. package/dist/importer/session-loader.d.ts +19 -0
  44. package/dist/importer/session-loader.js +12 -0
  45. package/dist/index.d.ts +24 -0
  46. package/dist/index.js +601 -0
  47. package/dist/plugin.d.ts +9 -0
  48. package/dist/plugin.js +11 -0
  49. package/dist/services/ai/ai-provider-factory.d.ts +7 -0
  50. package/dist/services/ai/ai-provider-factory.js +41 -0
  51. package/dist/services/ai/api-key-placeholder.d.ts +1 -0
  52. package/dist/services/ai/api-key-placeholder.js +14 -0
  53. package/dist/services/ai/internal-capture-sessions.d.ts +6 -0
  54. package/dist/services/ai/internal-capture-sessions.js +33 -0
  55. package/dist/services/ai/opencode-diagnostics.d.ts +7 -0
  56. package/dist/services/ai/opencode-diagnostics.js +36 -0
  57. package/dist/services/ai/opencode-host-config.d.ts +10 -0
  58. package/dist/services/ai/opencode-host-config.js +51 -0
  59. package/dist/services/ai/opencode-provider-loader.d.ts +3 -0
  60. package/dist/services/ai/opencode-provider-loader.js +4 -0
  61. package/dist/services/ai/opencode-provider.d.ts +92 -0
  62. package/dist/services/ai/opencode-provider.js +419 -0
  63. package/dist/services/ai/opencode-sdk-client.d.ts +6 -0
  64. package/dist/services/ai/opencode-sdk-client.js +38 -0
  65. package/dist/services/ai/profile-llm-client.d.ts +2 -0
  66. package/dist/services/ai/profile-llm-client.js +27 -0
  67. package/dist/services/ai/provider-config.d.ts +17 -0
  68. package/dist/services/ai/provider-config.js +30 -0
  69. package/dist/services/ai/providers/anthropic-messages.d.ts +34 -0
  70. package/dist/services/ai/providers/anthropic-messages.js +220 -0
  71. package/dist/services/ai/providers/base-provider.d.ts +24 -0
  72. package/dist/services/ai/providers/base-provider.js +24 -0
  73. package/dist/services/ai/providers/google-gemini.d.ts +15 -0
  74. package/dist/services/ai/providers/google-gemini.js +228 -0
  75. package/dist/services/ai/providers/minimax.d.ts +35 -0
  76. package/dist/services/ai/providers/minimax.js +55 -0
  77. package/dist/services/ai/providers/openai-chat-completion.d.ts +22 -0
  78. package/dist/services/ai/providers/openai-chat-completion.js +387 -0
  79. package/dist/services/ai/providers/openai-responses.d.ts +13 -0
  80. package/dist/services/ai/providers/openai-responses.js +182 -0
  81. package/dist/services/ai/providers/orcarouter.d.ts +51 -0
  82. package/dist/services/ai/providers/orcarouter.js +66 -0
  83. package/dist/services/ai/session/ai-session-manager.d.ts +24 -0
  84. package/dist/services/ai/session/ai-session-manager.js +211 -0
  85. package/dist/services/ai/session/session-types.d.ts +42 -0
  86. package/dist/services/ai/session/session-types.js +1 -0
  87. package/dist/services/ai/tools/tool-schema.d.ts +40 -0
  88. package/dist/services/ai/tools/tool-schema.js +24 -0
  89. package/dist/services/ai/validators/user-profile-validator.d.ts +12 -0
  90. package/dist/services/ai/validators/user-profile-validator.js +114 -0
  91. package/dist/services/api-handlers.d.ts +186 -0
  92. package/dist/services/api-handlers.js +1284 -0
  93. package/dist/services/auth-token.d.ts +14 -0
  94. package/dist/services/auth-token.js +59 -0
  95. package/dist/services/auto-capture.d.ts +3 -0
  96. package/dist/services/auto-capture.js +127 -0
  97. package/dist/services/cleanup-service.d.ts +22 -0
  98. package/dist/services/cleanup-service.js +109 -0
  99. package/dist/services/client.d.ts +134 -0
  100. package/dist/services/client.js +327 -0
  101. package/dist/services/context.d.ts +10 -0
  102. package/dist/services/context.js +27 -0
  103. package/dist/services/cors.d.ts +12 -0
  104. package/dist/services/cors.js +52 -0
  105. package/dist/services/deduplication-service.d.ts +28 -0
  106. package/dist/services/deduplication-service.js +111 -0
  107. package/dist/services/embedding.d.ts +36 -0
  108. package/dist/services/embedding.js +200 -0
  109. package/dist/services/jsonc.d.ts +6 -0
  110. package/dist/services/jsonc.js +76 -0
  111. package/dist/services/language-detector.d.ts +2 -0
  112. package/dist/services/language-detector.js +33 -0
  113. package/dist/services/legacy-migration.d.ts +77 -0
  114. package/dist/services/legacy-migration.js +343 -0
  115. package/dist/services/logger.d.ts +1 -0
  116. package/dist/services/logger.js +119 -0
  117. package/dist/services/memory-portability-service.d.ts +37 -0
  118. package/dist/services/memory-portability-service.js +340 -0
  119. package/dist/services/memory-scope.d.ts +16 -0
  120. package/dist/services/memory-scope.js +43 -0
  121. package/dist/services/migration-service.d.ts +42 -0
  122. package/dist/services/migration-service.js +307 -0
  123. package/dist/services/onnxruntime-resolve.d.ts +24 -0
  124. package/dist/services/onnxruntime-resolve.js +174 -0
  125. package/dist/services/privacy.d.ts +26 -0
  126. package/dist/services/privacy.js +69 -0
  127. package/dist/services/request-timeouts.d.ts +9 -0
  128. package/dist/services/request-timeouts.js +11 -0
  129. package/dist/services/runtime-require.d.ts +17 -0
  130. package/dist/services/runtime-require.js +75 -0
  131. package/dist/services/secret-resolver.d.ts +1 -0
  132. package/dist/services/secret-resolver.js +65 -0
  133. package/dist/services/shard-inventory-service.d.ts +69 -0
  134. package/dist/services/shard-inventory-service.js +315 -0
  135. package/dist/services/shard-path-migration-service.d.ts +31 -0
  136. package/dist/services/shard-path-migration-service.js +367 -0
  137. package/dist/services/tag-prefix-migration.d.ts +60 -0
  138. package/dist/services/tag-prefix-migration.js +439 -0
  139. package/dist/services/tag-prefix-scan.d.ts +26 -0
  140. package/dist/services/tag-prefix-scan.js +91 -0
  141. package/dist/services/tags.d.ts +29 -0
  142. package/dist/services/tags.js +251 -0
  143. package/dist/services/turso/connection-manager.d.ts +15 -0
  144. package/dist/services/turso/connection-manager.js +163 -0
  145. package/dist/services/turso/cross-process-write-lock.d.ts +1 -0
  146. package/dist/services/turso/cross-process-write-lock.js +89 -0
  147. package/dist/services/turso/legacy-migrator.d.ts +1 -0
  148. package/dist/services/turso/legacy-migrator.js +540 -0
  149. package/dist/services/turso/lifecycle.d.ts +1 -0
  150. package/dist/services/turso/lifecycle.js +14 -0
  151. package/dist/services/turso/operation-lock.d.ts +2 -0
  152. package/dist/services/turso/operation-lock.js +61 -0
  153. package/dist/services/turso/ready.d.ts +2 -0
  154. package/dist/services/turso/ready.js +36 -0
  155. package/dist/services/turso/shard-manager.d.ts +38 -0
  156. package/dist/services/turso/shard-manager.js +496 -0
  157. package/dist/services/turso/sqlite-handle-release.d.ts +16 -0
  158. package/dist/services/turso/sqlite-handle-release.js +56 -0
  159. package/dist/services/turso/turso-db.d.ts +18 -0
  160. package/dist/services/turso/turso-db.js +53 -0
  161. package/dist/services/turso/types.d.ts +42 -0
  162. package/dist/services/turso/types.js +1 -0
  163. package/dist/services/turso/vector-search.d.ts +41 -0
  164. package/dist/services/turso/vector-search.js +384 -0
  165. package/dist/services/turso/vector-utils.d.ts +7 -0
  166. package/dist/services/turso/vector-utils.js +67 -0
  167. package/dist/services/user-memory-learning.d.ts +97 -0
  168. package/dist/services/user-memory-learning.js +673 -0
  169. package/dist/services/user-profile/ai-cleanup.d.ts +58 -0
  170. package/dist/services/user-profile/ai-cleanup.js +486 -0
  171. package/dist/services/user-profile/profile-context.d.ts +1 -0
  172. package/dist/services/user-profile/profile-context.js +105 -0
  173. package/dist/services/user-profile/profile-utils.d.ts +2 -0
  174. package/dist/services/user-profile/profile-utils.js +45 -0
  175. package/dist/services/user-profile/types.d.ts +92 -0
  176. package/dist/services/user-profile/types.js +1 -0
  177. package/dist/services/user-profile/user-profile-manager.d.ts +65 -0
  178. package/dist/services/user-profile/user-profile-manager.js +1660 -0
  179. package/dist/services/user-prompt/user-prompt-manager.d.ts +53 -0
  180. package/dist/services/user-prompt/user-prompt-manager.js +303 -0
  181. package/dist/services/web-api-auth.d.ts +3 -0
  182. package/dist/services/web-api-auth.js +25 -0
  183. package/dist/services/web-auth.d.ts +46 -0
  184. package/dist/services/web-auth.js +105 -0
  185. package/dist/services/web-server.d.ts +57 -0
  186. package/dist/services/web-server.js +659 -0
  187. package/dist/shared/api/index.d.ts +4 -0
  188. package/dist/shared/api/index.js +2 -0
  189. package/dist/shared/api/portability-schemas.d.ts +56 -0
  190. package/dist/shared/api/portability-schemas.js +39 -0
  191. package/dist/shared/api/schemas.d.ts +295 -0
  192. package/dist/shared/api/schemas.js +134 -0
  193. package/dist/shared/api/types.d.ts +28 -0
  194. package/dist/shared/api/types.js +1 -0
  195. package/dist/types/index.d.ts +27 -0
  196. package/dist/types/index.js +1 -0
  197. package/dist/utils/context-limit.d.ts +10 -0
  198. package/dist/utils/context-limit.js +43 -0
  199. package/dist/utils/math.d.ts +7 -0
  200. package/dist/utils/math.js +48 -0
  201. package/dist/utils/profile.d.ts +2 -0
  202. package/dist/utils/profile.js +15 -0
  203. package/dist/v2/adapter.d.ts +5 -0
  204. package/dist/v2/adapter.js +218 -0
  205. package/dist/v2/legacy-client.d.ts +15 -0
  206. package/dist/v2/legacy-client.js +257 -0
  207. package/dist/v2/memory-bridge.d.ts +18 -0
  208. package/dist/v2/memory-bridge.js +29 -0
  209. package/dist/v2/plugin.d.ts +3 -0
  210. package/dist/v2/plugin.js +24 -0
  211. package/dist/web/assets/index-DPjEpD1h.css +2 -0
  212. package/dist/web/assets/index-do26c4JJ.js +122 -0
  213. package/dist/web/assets/jetbrains-mono-cyrillic-wght-normal-D73BlboJ.woff2 +0 -0
  214. package/dist/web/assets/jetbrains-mono-greek-wght-normal-Bw9x6K1M.woff2 +0 -0
  215. package/dist/web/assets/jetbrains-mono-latin-ext-wght-normal-DBQx-q_a.woff2 +0 -0
  216. package/dist/web/assets/jetbrains-mono-latin-wght-normal-B9CIFXIH.woff2 +0 -0
  217. package/dist/web/assets/jetbrains-mono-vietnamese-wght-normal-Bt-aOZkq.woff2 +0 -0
  218. package/dist/web/favicon.ico +0 -0
  219. package/dist/web/index.html +14 -0
  220. package/dist/web/omms-icon.png +0 -0
  221. package/package.json +138 -0
@@ -0,0 +1,38 @@
1
+ import type { ShardInfo } from "./types.js";
2
+ import type { TursoDb } from "./turso-db.js";
3
+ export declare class TursoShardManager {
4
+ private metadataDb;
5
+ private metadataPath;
6
+ private initPromise;
7
+ private readonly writeLocks;
8
+ reset(): void;
9
+ withScopeWriteLock<T>(scope: "user" | "project", scopeHash: string, fn: () => Promise<T>): Promise<T>;
10
+ private ensureInitialized;
11
+ private initMetadataDb;
12
+ getShardPath(scope: "user" | "project", scopeHash: string, shardIndex: number): string;
13
+ private resolveStoredPath;
14
+ getActiveShard(scope: "user" | "project", scopeHash: string): Promise<ShardInfo | null>;
15
+ getAllShards(scope: "user" | "project", scopeHash: string): Promise<ShardInfo[]>;
16
+ createShard(scope: "user" | "project", scopeHash: string, shardIndex: number): Promise<ShardInfo>;
17
+ registerExistingShard(scope: "user" | "project", scopeHash: string, shardIndex: number, dbPath: string, vectorCount: number, isActive: boolean): Promise<ShardInfo>;
18
+ initShardDb(db: TursoDb, dimensions?: number, embeddingModel?: string): Promise<void>;
19
+ private rowToShardInfo;
20
+ private hasMatchingEmbeddingDimensions;
21
+ private syncShardVectorCount;
22
+ private isShardValid;
23
+ getWriteShard(scope: "user" | "project", scopeHash: string): Promise<ShardInfo>;
24
+ private markShardReadOnly;
25
+ incrementVectorCount(shardId: number): Promise<void>;
26
+ decrementVectorCount(shardId: number): Promise<void>;
27
+ setVectorCount(shardId: number, count: number): Promise<void>;
28
+ getShardById(shardId: number): Promise<ShardInfo | null>;
29
+ getShardByPath(dbPath: string): Promise<ShardInfo | null>;
30
+ deleteShard(shardId: number): Promise<void>;
31
+ archiveShard(shardId: number, reason: string, archivePathOverride?: string): Promise<string | null>;
32
+ /**
33
+ * Reassign a registered shard to a new scope hash and on-disk filename.
34
+ * Caller must close the DB connection and rename the file before calling this.
35
+ */
36
+ reassignShardScope(shardId: number, newScopeHash: string, newDbPath: string, vectorCount: number, isActive: boolean): Promise<ShardInfo>;
37
+ }
38
+ export declare const tursoShardManager: TursoShardManager;
@@ -0,0 +1,496 @@
1
+ import { join, basename, resolve, relative } from "node:path";
2
+ import { existsSync, renameSync, unlinkSync } from "node:fs";
3
+ import { CONFIG } from "../../config.js";
4
+ import { assertSafeScopeHash } from "../memory-scope.js";
5
+ import { tursoConnectionManager } from "./connection-manager.js";
6
+ import { log } from "../logger.js";
7
+ import { assertNoTursoMigrationInProgress } from "./operation-lock.js";
8
+ import { withCrossProcessWriteLock } from "./cross-process-write-lock.js";
9
+ import { withSqliteFileLockRetry } from "./sqlite-handle-release.js";
10
+ const METADATA_DB_NAME = "metadata.db";
11
+ function getValidatedEmbeddingDimensions(dimensions = CONFIG.embeddingDimensions) {
12
+ const dims = dimensions;
13
+ if (!Number.isInteger(dims) || dims <= 0 || dims > 65536) {
14
+ throw new Error(`Invalid embeddingDimensions config: ${dims}`);
15
+ }
16
+ return dims;
17
+ }
18
+ function isUniqueConstraintError(error) {
19
+ const message = error instanceof Error ? error.message : String(error);
20
+ return message.includes("UNIQUE constraint failed") || message.includes("SQLITE_CONSTRAINT");
21
+ }
22
+ export class TursoShardManager {
23
+ metadataDb = null;
24
+ metadataPath = "";
25
+ initPromise = null;
26
+ writeLocks = new Map();
27
+ reset() {
28
+ this.metadataDb = null;
29
+ this.initPromise = null;
30
+ this.metadataPath = "";
31
+ this.writeLocks.clear();
32
+ }
33
+ async withScopeWriteLock(scope, scopeHash, fn) {
34
+ const key = `${scope}:${scopeHash}`;
35
+ const previous = this.writeLocks.get(key) ?? Promise.resolve();
36
+ let release;
37
+ const gate = new Promise((resolve) => {
38
+ release = resolve;
39
+ });
40
+ const next = previous.catch(() => undefined).then(() => gate);
41
+ this.writeLocks.set(key, next);
42
+ await previous.catch(() => undefined);
43
+ try {
44
+ // In-process queue first, then the cross-process advisory lock: every
45
+ // writer takes them in the same order, and the file lock arbitrates
46
+ // between processes (OpenCode + Pi, or any two writers) so that shard
47
+ // allocation, vector-count sync, and the insert itself run as one
48
+ // critical section per scope.
49
+ return await withCrossProcessWriteLock(scope, scopeHash, fn);
50
+ }
51
+ finally {
52
+ release();
53
+ if (this.writeLocks.get(key) === next) {
54
+ this.writeLocks.delete(key);
55
+ }
56
+ }
57
+ }
58
+ async ensureInitialized() {
59
+ if (this.metadataDb && this.initPromise) {
60
+ await this.initPromise;
61
+ return this.metadataDb;
62
+ }
63
+ if (this.initPromise) {
64
+ await this.initPromise;
65
+ if (this.metadataDb)
66
+ return this.metadataDb;
67
+ }
68
+ this.initPromise = (async () => {
69
+ try {
70
+ this.metadataPath = join(CONFIG.storagePath, METADATA_DB_NAME);
71
+ this.metadataDb = await tursoConnectionManager.getConnection(this.metadataPath);
72
+ await this.initMetadataDb(this.metadataDb);
73
+ }
74
+ catch (error) {
75
+ this.initPromise = null;
76
+ this.metadataDb = null;
77
+ throw error;
78
+ }
79
+ })();
80
+ await this.initPromise;
81
+ return this.metadataDb;
82
+ }
83
+ async initMetadataDb(db) {
84
+ await db.batch([
85
+ {
86
+ sql: `
87
+ CREATE TABLE IF NOT EXISTS shards (
88
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
89
+ scope TEXT NOT NULL,
90
+ scope_hash TEXT NOT NULL,
91
+ shard_index INTEGER NOT NULL,
92
+ db_path TEXT NOT NULL,
93
+ vector_count INTEGER DEFAULT 0,
94
+ is_active INTEGER DEFAULT 1,
95
+ created_at INTEGER NOT NULL,
96
+ UNIQUE(scope, scope_hash, shard_index)
97
+ )
98
+ `,
99
+ },
100
+ {
101
+ sql: `
102
+ CREATE INDEX IF NOT EXISTS idx_active_shards
103
+ ON shards(scope, scope_hash, is_active)
104
+ `,
105
+ },
106
+ ]);
107
+ }
108
+ getShardPath(scope, scopeHash, shardIndex) {
109
+ assertSafeScopeHash(scopeHash);
110
+ const scopeDir = resolve(CONFIG.storagePath, `${scope}s`);
111
+ const fullPath = resolve(join(scopeDir, `${scope}_${scopeHash}_shard_${shardIndex}.db`));
112
+ const relativePath = relative(scopeDir, fullPath);
113
+ if (relativePath.startsWith("..") || relativePath.includes("..")) {
114
+ throw new Error(`Shard path escapes storage directory: ${fullPath}`);
115
+ }
116
+ return fullPath;
117
+ }
118
+ resolveStoredPath(storedPath, scope) {
119
+ const fileName = basename(storedPath);
120
+ return join(CONFIG.storagePath, `${scope}s`, fileName);
121
+ }
122
+ async getActiveShard(scope, scopeHash) {
123
+ const metadataDb = await this.ensureInitialized();
124
+ const row = await metadataDb.get(`
125
+ SELECT * FROM shards
126
+ WHERE scope = ? AND scope_hash = ? AND is_active = 1
127
+ ORDER BY shard_index DESC LIMIT 1
128
+ `, [scope, scopeHash]);
129
+ if (!row)
130
+ return null;
131
+ return this.rowToShardInfo(row);
132
+ }
133
+ async getAllShards(scope, scopeHash) {
134
+ const metadataDb = await this.ensureInitialized();
135
+ const rows = scopeHash === ""
136
+ ? await metadataDb.all(`
137
+ SELECT * FROM shards
138
+ WHERE scope = ?
139
+ ORDER BY shard_index ASC
140
+ `, [scope])
141
+ : await metadataDb.all(`
142
+ SELECT * FROM shards
143
+ WHERE scope = ? AND scope_hash = ?
144
+ ORDER BY shard_index ASC
145
+ `, [scope, scopeHash]);
146
+ return rows.map((row) => this.rowToShardInfo(row));
147
+ }
148
+ async createShard(scope, scopeHash, shardIndex) {
149
+ const metadataDb = await this.ensureInitialized();
150
+ const fullPath = this.getShardPath(scope, scopeHash, shardIndex);
151
+ const storedPath = join(`${scope}s`, basename(fullPath)).replace(/\\/g, "/");
152
+ const now = Date.now();
153
+ // Initialize the shard file BEFORE inserting the registry row. If init throws
154
+ // (disk full, permissions), the registry stays free of an orphan row that points
155
+ // at an uninitialized file — such a row would later fail isShardValid on every
156
+ // getWriteShard and brick all writes to this scope. initShardDb is idempotent.
157
+ const shardDb = await tursoConnectionManager.getConnection(fullPath);
158
+ await this.initShardDb(shardDb);
159
+ let result;
160
+ try {
161
+ result = await metadataDb.execute(`
162
+ INSERT INTO shards (scope, scope_hash, shard_index, db_path, vector_count, is_active, created_at)
163
+ VALUES (?, ?, ?, ?, 0, 1, ?)
164
+ `, [scope, scopeHash, shardIndex, storedPath, now]);
165
+ }
166
+ catch (error) {
167
+ if (isUniqueConstraintError(error)) {
168
+ const existing = await metadataDb.get(`
169
+ SELECT * FROM shards
170
+ WHERE scope = ? AND scope_hash = ? AND shard_index = ?
171
+ `, [scope, scopeHash, shardIndex]);
172
+ if (existing) {
173
+ return this.rowToShardInfo(existing);
174
+ }
175
+ }
176
+ throw error;
177
+ }
178
+ return {
179
+ id: Number(result.lastInsertRowid),
180
+ scope,
181
+ scopeHash,
182
+ shardIndex,
183
+ dbPath: fullPath,
184
+ vectorCount: 0,
185
+ isActive: true,
186
+ createdAt: now,
187
+ };
188
+ }
189
+ async registerExistingShard(scope, scopeHash, shardIndex, dbPath, vectorCount, isActive) {
190
+ assertSafeScopeHash(scopeHash);
191
+ const metadataDb = await this.ensureInitialized();
192
+ const storedPath = join(`${scope}s`, basename(dbPath)).replace(/\\/g, "/");
193
+ const now = Date.now();
194
+ await metadataDb.execute(`
195
+ INSERT INTO shards (
196
+ scope, scope_hash, shard_index, db_path, vector_count, is_active, created_at
197
+ ) VALUES (?, ?, ?, ?, ?, ?, ?)
198
+ ON CONFLICT(scope, scope_hash, shard_index) DO UPDATE SET
199
+ db_path = excluded.db_path,
200
+ vector_count = excluded.vector_count,
201
+ is_active = excluded.is_active
202
+ `, [scope, scopeHash, shardIndex, storedPath, vectorCount, isActive ? 1 : 0, now]);
203
+ const row = await metadataDb.get(`SELECT * FROM shards WHERE scope = ? AND scope_hash = ? AND shard_index = ?`, [scope, scopeHash, shardIndex]);
204
+ if (!row) {
205
+ throw new Error(`Failed to register shard ${scope}/${scopeHash}#${shardIndex}`);
206
+ }
207
+ return this.rowToShardInfo(row);
208
+ }
209
+ async initShardDb(db, dimensions = CONFIG.embeddingDimensions, embeddingModel = CONFIG.embeddingModel) {
210
+ const dims = getValidatedEmbeddingDimensions(dimensions);
211
+ await db.batch([
212
+ {
213
+ sql: `
214
+ CREATE TABLE IF NOT EXISTS shard_metadata (
215
+ key TEXT PRIMARY KEY,
216
+ value TEXT NOT NULL
217
+ )
218
+ `,
219
+ },
220
+ {
221
+ sql: `
222
+ INSERT OR REPLACE INTO shard_metadata (key, value)
223
+ VALUES ('embedding_dimensions', ?)
224
+ `,
225
+ args: [String(dims)],
226
+ },
227
+ {
228
+ sql: `
229
+ INSERT OR REPLACE INTO shard_metadata (key, value)
230
+ VALUES ('embedding_model', ?)
231
+ `,
232
+ args: [embeddingModel],
233
+ },
234
+ {
235
+ sql: `
236
+ CREATE TABLE IF NOT EXISTS memories (
237
+ id TEXT PRIMARY KEY,
238
+ content TEXT NOT NULL,
239
+ vector F32_BLOB(${dims}) NOT NULL,
240
+ tags_vector F32_BLOB(${dims}),
241
+ container_tag TEXT NOT NULL,
242
+ tags TEXT,
243
+ type TEXT,
244
+ created_at INTEGER NOT NULL,
245
+ updated_at INTEGER NOT NULL,
246
+ metadata TEXT,
247
+ display_name TEXT,
248
+ user_name TEXT,
249
+ user_email TEXT,
250
+ project_path TEXT,
251
+ project_name TEXT,
252
+ git_repo_url TEXT,
253
+ is_pinned INTEGER DEFAULT 0
254
+ )
255
+ `,
256
+ },
257
+ {
258
+ sql: `CREATE INDEX IF NOT EXISTS idx_container_tag ON memories(container_tag)`,
259
+ },
260
+ {
261
+ sql: `CREATE INDEX IF NOT EXISTS idx_type ON memories(type)`,
262
+ },
263
+ {
264
+ sql: `CREATE INDEX IF NOT EXISTS idx_created_at ON memories(created_at DESC)`,
265
+ },
266
+ {
267
+ sql: `CREATE INDEX IF NOT EXISTS idx_is_pinned ON memories(is_pinned)`,
268
+ },
269
+ {
270
+ sql: `
271
+ CREATE INDEX IF NOT EXISTS memories_vec_idx
272
+ ON memories (libsql_vector_idx(
273
+ vector,
274
+ 'metric=cosine',
275
+ 'compress_neighbors=float8',
276
+ 'max_neighbors=20'
277
+ ))
278
+ `,
279
+ },
280
+ {
281
+ sql: `
282
+ CREATE INDEX IF NOT EXISTS memories_tags_vec_idx
283
+ ON memories (libsql_vector_idx(
284
+ tags_vector,
285
+ 'metric=cosine',
286
+ 'compress_neighbors=float8',
287
+ 'max_neighbors=20'
288
+ ))
289
+ WHERE tags_vector IS NOT NULL
290
+ `,
291
+ },
292
+ ]);
293
+ }
294
+ rowToShardInfo(row) {
295
+ return {
296
+ id: Number(row.id),
297
+ scope: row.scope,
298
+ scopeHash: String(row.scope_hash),
299
+ shardIndex: Number(row.shard_index),
300
+ dbPath: this.resolveStoredPath(String(row.db_path), String(row.scope)),
301
+ vectorCount: Number(row.vector_count),
302
+ isActive: Number(row.is_active) === 1,
303
+ createdAt: Number(row.created_at),
304
+ };
305
+ }
306
+ async hasMatchingEmbeddingDimensions(db, shard) {
307
+ const row = await db.get(`SELECT value FROM shard_metadata WHERE key = 'embedding_dimensions'`);
308
+ if (!row?.value) {
309
+ log("Shard missing embedding_dimensions metadata", {
310
+ dbPath: shard.dbPath,
311
+ shardId: shard.id,
312
+ });
313
+ return false;
314
+ }
315
+ const storedDimensions = Number(row.value);
316
+ if (storedDimensions !== getValidatedEmbeddingDimensions()) {
317
+ log("Shard embedding dimensions mismatch", {
318
+ dbPath: shard.dbPath,
319
+ shardId: shard.id,
320
+ storedDimensions,
321
+ configDimensions: CONFIG.embeddingDimensions,
322
+ });
323
+ return false;
324
+ }
325
+ return true;
326
+ }
327
+ async syncShardVectorCount(shard) {
328
+ const db = await tursoConnectionManager.getConnection(shard.dbPath);
329
+ const row = await db.get(`SELECT COUNT(*) as count FROM memories`);
330
+ const count = Number(row?.count ?? 0);
331
+ if (count === shard.vectorCount) {
332
+ return shard;
333
+ }
334
+ const metadataDb = await this.ensureInitialized();
335
+ await metadataDb.run(`UPDATE shards SET vector_count = ? WHERE id = ?`, [count, shard.id]);
336
+ return { ...shard, vectorCount: count };
337
+ }
338
+ async isShardValid(shard) {
339
+ if (!existsSync(shard.dbPath)) {
340
+ log("Shard DB file missing", { dbPath: shard.dbPath, shardId: shard.id });
341
+ return false;
342
+ }
343
+ try {
344
+ const db = await tursoConnectionManager.getConnection(shard.dbPath);
345
+ const result = await db.get(`SELECT name FROM sqlite_master WHERE type='table' AND name='memories'`);
346
+ if (!result) {
347
+ log("Shard DB missing 'memories' table", {
348
+ dbPath: shard.dbPath,
349
+ shardId: shard.id,
350
+ });
351
+ return false;
352
+ }
353
+ if (!(await this.hasMatchingEmbeddingDimensions(db, shard))) {
354
+ return false;
355
+ }
356
+ return true;
357
+ }
358
+ catch (error) {
359
+ log("Error validating shard DB", {
360
+ dbPath: shard.dbPath,
361
+ error: String(error),
362
+ });
363
+ return false;
364
+ }
365
+ }
366
+ async getWriteShard(scope, scopeHash) {
367
+ assertNoTursoMigrationInProgress();
368
+ for (let attempt = 0; attempt < 3; attempt++) {
369
+ let shard = await this.getActiveShard(scope, scopeHash);
370
+ if (!shard) {
371
+ return this.createShard(scope, scopeHash, 0);
372
+ }
373
+ if (!(await this.isShardValid(shard))) {
374
+ throw new Error(`Shard ${shard.scope}/${shard.scopeHash}#${shard.shardIndex} is incompatible or corrupt. ` +
375
+ `The original database was left untouched at ${shard.dbPath}; run the migration or restore it before writing.`);
376
+ }
377
+ shard = await this.syncShardVectorCount(shard);
378
+ if (shard.vectorCount >= CONFIG.maxVectorsPerShard) {
379
+ await this.markShardReadOnly(shard.id);
380
+ return this.createShard(scope, scopeHash, shard.shardIndex + 1);
381
+ }
382
+ return shard;
383
+ }
384
+ throw new Error(`Failed to resolve write shard for ${scope}/${scopeHash}`);
385
+ }
386
+ async markShardReadOnly(shardId) {
387
+ const metadataDb = await this.ensureInitialized();
388
+ await metadataDb.run(`UPDATE shards SET is_active = 0 WHERE id = ?`, [shardId]);
389
+ }
390
+ async incrementVectorCount(shardId) {
391
+ const metadataDb = await this.ensureInitialized();
392
+ await metadataDb.run(`UPDATE shards SET vector_count = vector_count + 1 WHERE id = ?`, [
393
+ shardId,
394
+ ]);
395
+ }
396
+ async decrementVectorCount(shardId) {
397
+ const metadataDb = await this.ensureInitialized();
398
+ await metadataDb.run(`UPDATE shards SET vector_count = vector_count - 1 WHERE id = ? AND vector_count > 0`, [shardId]);
399
+ }
400
+ async setVectorCount(shardId, count) {
401
+ if (!Number.isInteger(count) || count < 0) {
402
+ throw new Error(`Invalid vector count: ${count}`);
403
+ }
404
+ const metadataDb = await this.ensureInitialized();
405
+ await metadataDb.run(`UPDATE shards SET vector_count = ? WHERE id = ?`, [count, shardId]);
406
+ }
407
+ async getShardById(shardId) {
408
+ const metadataDb = await this.ensureInitialized();
409
+ const row = await metadataDb.get(`SELECT * FROM shards WHERE id = ?`, [shardId]);
410
+ return row ? this.rowToShardInfo(row) : null;
411
+ }
412
+ async getShardByPath(dbPath) {
413
+ const metadataDb = await this.ensureInitialized();
414
+ const fileName = basename(dbPath);
415
+ // Stored db_path is always `<scope>s/<basename>` (see createShard/registerExistingShard),
416
+ // so anchor on the "/" separator. Escape LIKE metacharacters in the filename — otherwise
417
+ // the "_" in shard names like `user_<hash>_0.db` would match any character.
418
+ const escaped = fileName.replace(/[\\%_]/g, "\\$&");
419
+ const row = await metadataDb.get(`SELECT * FROM shards WHERE db_path LIKE '%/' || ? ESCAPE '\\'`, [escaped]);
420
+ if (!row)
421
+ return null;
422
+ return this.rowToShardInfo(row);
423
+ }
424
+ async deleteShard(shardId) {
425
+ const metadataDb = await this.ensureInitialized();
426
+ const row = await metadataDb.get(`SELECT * FROM shards WHERE id = ?`, [shardId]);
427
+ if (!row)
428
+ return;
429
+ const fullPath = this.resolveStoredPath(String(row.db_path), String(row.scope));
430
+ await tursoConnectionManager.closeConnection(fullPath);
431
+ try {
432
+ if (existsSync(fullPath)) {
433
+ await withSqliteFileLockRetry(() => unlinkSync(fullPath));
434
+ }
435
+ }
436
+ catch (error) {
437
+ log("Error deleting shard file", {
438
+ dbPath: fullPath,
439
+ error: String(error),
440
+ });
441
+ }
442
+ await metadataDb.run(`DELETE FROM shards WHERE id = ?`, [shardId]);
443
+ }
444
+ async archiveShard(shardId, reason, archivePathOverride) {
445
+ const metadataDb = await this.ensureInitialized();
446
+ const row = await metadataDb.get(`SELECT * FROM shards WHERE id = ?`, [shardId]);
447
+ if (!row)
448
+ return null;
449
+ const fullPath = this.resolveStoredPath(String(row.db_path), String(row.scope));
450
+ await tursoConnectionManager.closeConnection(fullPath);
451
+ const archivePath = archivePathOverride ?? `${fullPath}.${reason}-${process.pid}-${Date.now()}.bak`;
452
+ const scopeDir = resolve(CONFIG.storagePath, `${String(row.scope)}s`);
453
+ const archiveRelativePath = relative(scopeDir, resolve(archivePath));
454
+ if (archiveRelativePath.startsWith("..") || archiveRelativePath.includes("..")) {
455
+ throw new Error(`Archive path escapes storage directory: ${archivePath}`);
456
+ }
457
+ if (existsSync(fullPath)) {
458
+ await withSqliteFileLockRetry(() => renameSync(fullPath, archivePath));
459
+ }
460
+ try {
461
+ await metadataDb.run(`DELETE FROM shards WHERE id = ?`, [shardId]);
462
+ }
463
+ catch (error) {
464
+ if (existsSync(archivePath) && !existsSync(fullPath)) {
465
+ await withSqliteFileLockRetry(() => renameSync(archivePath, fullPath));
466
+ }
467
+ throw error;
468
+ }
469
+ return existsSync(archivePath) ? archivePath : null;
470
+ }
471
+ /**
472
+ * Reassign a registered shard to a new scope hash and on-disk filename.
473
+ * Caller must close the DB connection and rename the file before calling this.
474
+ */
475
+ async reassignShardScope(shardId, newScopeHash, newDbPath, vectorCount, isActive) {
476
+ assertSafeScopeHash(newScopeHash);
477
+ const metadataDb = await this.ensureInitialized();
478
+ const row = await metadataDb.get(`SELECT * FROM shards WHERE id = ?`, [shardId]);
479
+ if (!row) {
480
+ throw new Error(`Shard ${shardId} not found in metadata registry`);
481
+ }
482
+ const scope = String(row.scope);
483
+ const storedPath = join(`${scope}s`, basename(newDbPath)).replace(/\\/g, "/");
484
+ await metadataDb.run(`
485
+ UPDATE shards
486
+ SET scope_hash = ?, db_path = ?, vector_count = ?, is_active = ?
487
+ WHERE id = ?
488
+ `, [newScopeHash, storedPath, vectorCount, isActive ? 1 : 0, shardId]);
489
+ const updated = await metadataDb.get(`SELECT * FROM shards WHERE id = ?`, [shardId]);
490
+ if (!updated) {
491
+ throw new Error(`Failed to reassign shard ${shardId}`);
492
+ }
493
+ return this.rowToShardInfo(updated);
494
+ }
495
+ }
496
+ export const tursoShardManager = new TursoShardManager();
@@ -0,0 +1,16 @@
1
+ export declare const RETRYABLE_FILE_LOCK_CODES: Set<string>;
2
+ /**
3
+ * libsql 0.5.x leaves prepared-statement handles alive until garbage collection.
4
+ * Collect them before Windows file swaps/removals that require exclusive access.
5
+ *
6
+ * Upstream: https://github.com/tursodatabase/libsql-js/issues/228
7
+ */
8
+ export declare function collectReleasedSqliteHandles(): Promise<void>;
9
+ /**
10
+ * Retries Windows file mutations while stable libsql releases native handles.
11
+ * The operation runs at most `maxRetries + 1` times: the initial attempt plus
12
+ * up to `maxRetries` retries. Non-lock errors and non-Windows platforms fail
13
+ * immediately. The default covers every entry in the delay table, matching the
14
+ * pre-parameter behavior exactly.
15
+ */
16
+ export declare function withSqliteFileLockRetry<T>(operation: () => T | Promise<T>, maxRetries?: number): Promise<T>;
@@ -0,0 +1,56 @@
1
+ const GC_PASSES = 3;
2
+ const FILE_LOCK_RETRY_DELAYS_MS = [25, 50, 100, 200, 400, 800, 1600, 3200];
3
+ export const RETRYABLE_FILE_LOCK_CODES = new Set(["EBUSY", "EPERM", "EACCES"]);
4
+ function delay(ms) {
5
+ return new Promise((resolve) => setTimeout(resolve, ms));
6
+ }
7
+ /**
8
+ * libsql 0.5.x leaves prepared-statement handles alive until garbage collection.
9
+ * Collect them before Windows file swaps/removals that require exclusive access.
10
+ *
11
+ * Upstream: https://github.com/tursodatabase/libsql-js/issues/228
12
+ */
13
+ export async function collectReleasedSqliteHandles() {
14
+ if (process.platform !== "win32")
15
+ return;
16
+ const runtime = globalThis;
17
+ const bunGc = runtime.Bun?.gc;
18
+ const globalGc = runtime.gc;
19
+ if (!bunGc && !globalGc)
20
+ return;
21
+ for (let pass = 0; pass < GC_PASSES; pass += 1) {
22
+ bunGc?.(true);
23
+ globalGc?.();
24
+ await delay(0);
25
+ }
26
+ }
27
+ /**
28
+ * Retries Windows file mutations while stable libsql releases native handles.
29
+ * The operation runs at most `maxRetries + 1` times: the initial attempt plus
30
+ * up to `maxRetries` retries. Non-lock errors and non-Windows platforms fail
31
+ * immediately. The default covers every entry in the delay table, matching the
32
+ * pre-parameter behavior exactly.
33
+ */
34
+ export async function withSqliteFileLockRetry(operation, maxRetries = FILE_LOCK_RETRY_DELAYS_MS.length) {
35
+ if (!Number.isInteger(maxRetries) || maxRetries < 0) {
36
+ throw new TypeError("maxRetries must be a non-negative integer");
37
+ }
38
+ for (let attempt = 0;; attempt += 1) {
39
+ try {
40
+ return await operation();
41
+ }
42
+ catch (error) {
43
+ const code = error?.code;
44
+ const retryDelay = FILE_LOCK_RETRY_DELAYS_MS[attempt];
45
+ if (process.platform !== "win32" ||
46
+ !code ||
47
+ !RETRYABLE_FILE_LOCK_CODES.has(code) ||
48
+ retryDelay === undefined ||
49
+ attempt >= maxRetries) {
50
+ throw error;
51
+ }
52
+ await collectReleasedSqliteHandles();
53
+ await delay(retryDelay);
54
+ }
55
+ }
56
+ }
@@ -0,0 +1,18 @@
1
+ import type { Client, InArgs, ResultSet, Transaction } from "@libsql/client";
2
+ type Row = Record<string, unknown>;
3
+ export declare class TursoDb {
4
+ private readonly client;
5
+ constructor(client: Client);
6
+ getClient(): Client;
7
+ execute(sql: string, args?: InArgs): Promise<ResultSet>;
8
+ batch(statements: Array<{
9
+ sql: string;
10
+ args?: InArgs;
11
+ }>, mode?: "write" | "read"): Promise<ResultSet[]>;
12
+ get<T extends Row = Row>(sql: string, args?: InArgs): Promise<T | null>;
13
+ all<T extends Row = Row>(sql: string, args?: InArgs): Promise<T[]>;
14
+ run(sql: string, args?: InArgs): Promise<number>;
15
+ transaction<T>(mode: "write" | "read", fn: (tx: Transaction) => Promise<T>): Promise<T>;
16
+ close(): Promise<void>;
17
+ }
18
+ export {};
@@ -0,0 +1,53 @@
1
+ export class TursoDb {
2
+ client;
3
+ constructor(client) {
4
+ this.client = client;
5
+ }
6
+ getClient() {
7
+ return this.client;
8
+ }
9
+ async execute(sql, args) {
10
+ return this.client.execute({ sql, args: args ?? [] });
11
+ }
12
+ async batch(statements, mode = "write") {
13
+ return this.client.batch(statements.map((statement) => ({
14
+ sql: statement.sql,
15
+ args: statement.args ?? [],
16
+ })), mode);
17
+ }
18
+ async get(sql, args) {
19
+ const result = await this.execute(sql, args);
20
+ return result.rows[0] ?? null;
21
+ }
22
+ async all(sql, args) {
23
+ const result = await this.execute(sql, args);
24
+ return result.rows;
25
+ }
26
+ async run(sql, args) {
27
+ const result = await this.execute(sql, args);
28
+ return Number(result.rowsAffected ?? 0);
29
+ }
30
+ async transaction(mode, fn) {
31
+ const tx = await this.client.transaction(mode);
32
+ try {
33
+ const value = await fn(tx);
34
+ await tx.commit();
35
+ return value;
36
+ }
37
+ catch (error) {
38
+ try {
39
+ await tx.rollback();
40
+ }
41
+ catch {
42
+ // ignore rollback errors after a failed statement / already-closed tx
43
+ }
44
+ throw error;
45
+ }
46
+ finally {
47
+ tx.close();
48
+ }
49
+ }
50
+ async close() {
51
+ this.client.close();
52
+ }
53
+ }