@haisto/opencode-mem 2.14.3-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -0
- package/dist/config.d.ts +62 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +457 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +486 -0
- package/dist/plugin.d.ts +9 -0
- package/dist/plugin.d.ts.map +1 -0
- package/dist/plugin.js +5 -0
- package/dist/services/ai/ai-provider-factory.d.ts +8 -0
- package/dist/services/ai/ai-provider-factory.d.ts.map +1 -0
- package/dist/services/ai/ai-provider-factory.js +28 -0
- package/dist/services/ai/opencode-provider.d.ts +36 -0
- package/dist/services/ai/opencode-provider.d.ts.map +1 -0
- package/dist/services/ai/opencode-provider.js +92 -0
- package/dist/services/ai/provider-config.d.ts +17 -0
- package/dist/services/ai/provider-config.d.ts.map +1 -0
- package/dist/services/ai/provider-config.js +14 -0
- package/dist/services/ai/providers/anthropic-messages.d.ts +12 -0
- package/dist/services/ai/providers/anthropic-messages.d.ts.map +1 -0
- package/dist/services/ai/providers/anthropic-messages.js +184 -0
- package/dist/services/ai/providers/base-provider.d.ts +25 -0
- package/dist/services/ai/providers/base-provider.d.ts.map +1 -0
- package/dist/services/ai/providers/base-provider.js +23 -0
- package/dist/services/ai/providers/google-gemini.d.ts +16 -0
- package/dist/services/ai/providers/google-gemini.d.ts.map +1 -0
- package/dist/services/ai/providers/google-gemini.js +228 -0
- package/dist/services/ai/providers/openai-chat-completion.d.ts +14 -0
- package/dist/services/ai/providers/openai-chat-completion.d.ts.map +1 -0
- package/dist/services/ai/providers/openai-chat-completion.js +318 -0
- package/dist/services/ai/providers/openai-responses.d.ts +14 -0
- package/dist/services/ai/providers/openai-responses.d.ts.map +1 -0
- package/dist/services/ai/providers/openai-responses.js +182 -0
- package/dist/services/ai/session/ai-session-manager.d.ts +21 -0
- package/dist/services/ai/session/ai-session-manager.d.ts.map +1 -0
- package/dist/services/ai/session/ai-session-manager.js +166 -0
- package/dist/services/ai/session/session-types.d.ts +43 -0
- package/dist/services/ai/session/session-types.d.ts.map +1 -0
- package/dist/services/ai/session/session-types.js +1 -0
- package/dist/services/ai/tools/tool-schema.d.ts +41 -0
- package/dist/services/ai/tools/tool-schema.d.ts.map +1 -0
- package/dist/services/ai/tools/tool-schema.js +24 -0
- package/dist/services/ai/validators/user-profile-validator.d.ts +13 -0
- package/dist/services/ai/validators/user-profile-validator.d.ts.map +1 -0
- package/dist/services/ai/validators/user-profile-validator.js +111 -0
- package/dist/services/api-handlers.d.ts +164 -0
- package/dist/services/api-handlers.d.ts.map +1 -0
- package/dist/services/api-handlers.js +927 -0
- package/dist/services/auto-capture.d.ts +3 -0
- package/dist/services/auto-capture.d.ts.map +1 -0
- package/dist/services/auto-capture.js +309 -0
- package/dist/services/cleanup-service.d.ts +23 -0
- package/dist/services/cleanup-service.d.ts.map +1 -0
- package/dist/services/cleanup-service.js +102 -0
- package/dist/services/client.d.ts +119 -0
- package/dist/services/client.d.ts.map +1 -0
- package/dist/services/client.js +257 -0
- package/dist/services/context.d.ts +11 -0
- package/dist/services/context.d.ts.map +1 -0
- package/dist/services/context.js +34 -0
- package/dist/services/deduplication-service.d.ts +30 -0
- package/dist/services/deduplication-service.d.ts.map +1 -0
- package/dist/services/deduplication-service.js +124 -0
- package/dist/services/embedding.d.ts +15 -0
- package/dist/services/embedding.d.ts.map +1 -0
- package/dist/services/embedding.js +127 -0
- package/dist/services/jsonc.d.ts +7 -0
- package/dist/services/jsonc.d.ts.map +1 -0
- package/dist/services/jsonc.js +76 -0
- package/dist/services/language-detector.d.ts +3 -0
- package/dist/services/language-detector.d.ts.map +1 -0
- package/dist/services/language-detector.js +33 -0
- package/dist/services/logger.d.ts +11 -0
- package/dist/services/logger.d.ts.map +1 -0
- package/dist/services/logger.js +97 -0
- package/dist/services/migration-service.d.ts +42 -0
- package/dist/services/migration-service.d.ts.map +1 -0
- package/dist/services/migration-service.js +250 -0
- package/dist/services/privacy.d.ts +3 -0
- package/dist/services/privacy.d.ts.map +1 -0
- package/dist/services/privacy.js +7 -0
- package/dist/services/secret-resolver.d.ts +2 -0
- package/dist/services/secret-resolver.d.ts.map +1 -0
- package/dist/services/secret-resolver.js +55 -0
- package/dist/services/sqlite/connection-manager.d.ts +13 -0
- package/dist/services/sqlite/connection-manager.d.ts.map +1 -0
- package/dist/services/sqlite/connection-manager.js +74 -0
- package/dist/services/sqlite/shard-manager.d.ts +23 -0
- package/dist/services/sqlite/shard-manager.d.ts.map +1 -0
- package/dist/services/sqlite/shard-manager.js +288 -0
- package/dist/services/sqlite/sqlite-bootstrap.d.ts +2 -0
- package/dist/services/sqlite/sqlite-bootstrap.d.ts.map +1 -0
- package/dist/services/sqlite/sqlite-bootstrap.js +8 -0
- package/dist/services/sqlite/types.d.ts +42 -0
- package/dist/services/sqlite/types.d.ts.map +1 -0
- package/dist/services/sqlite/types.js +1 -0
- package/dist/services/sqlite/vector-search.d.ts +29 -0
- package/dist/services/sqlite/vector-search.d.ts.map +1 -0
- package/dist/services/sqlite/vector-search.js +279 -0
- package/dist/services/tags.d.ts +24 -0
- package/dist/services/tags.d.ts.map +1 -0
- package/dist/services/tags.js +145 -0
- package/dist/services/user-memory-learning.d.ts +3 -0
- package/dist/services/user-memory-learning.d.ts.map +1 -0
- package/dist/services/user-memory-learning.js +235 -0
- package/dist/services/user-profile/profile-context.d.ts +2 -0
- package/dist/services/user-profile/profile-context.d.ts.map +1 -0
- package/dist/services/user-profile/profile-context.js +40 -0
- package/dist/services/user-profile/profile-utils.d.ts +3 -0
- package/dist/services/user-profile/profile-utils.d.ts.map +1 -0
- package/dist/services/user-profile/profile-utils.js +45 -0
- package/dist/services/user-profile/types.d.ts +46 -0
- package/dist/services/user-profile/types.d.ts.map +1 -0
- package/dist/services/user-profile/types.js +1 -0
- package/dist/services/user-profile/user-profile-manager.d.ts +23 -0
- package/dist/services/user-profile/user-profile-manager.d.ts.map +1 -0
- package/dist/services/user-profile/user-profile-manager.js +337 -0
- package/dist/services/user-prompt/user-prompt-manager.d.ts +41 -0
- package/dist/services/user-prompt/user-prompt-manager.d.ts.map +1 -0
- package/dist/services/user-prompt/user-prompt-manager.js +192 -0
- package/dist/services/vector-backends/backend-factory.d.ts +3 -0
- package/dist/services/vector-backends/backend-factory.d.ts.map +1 -0
- package/dist/services/vector-backends/backend-factory.js +104 -0
- package/dist/services/vector-backends/exact-scan-backend.d.ts +39 -0
- package/dist/services/vector-backends/exact-scan-backend.d.ts.map +1 -0
- package/dist/services/vector-backends/exact-scan-backend.js +63 -0
- package/dist/services/vector-backends/types.d.ts +51 -0
- package/dist/services/vector-backends/types.d.ts.map +1 -0
- package/dist/services/vector-backends/types.js +1 -0
- package/dist/services/vector-backends/usearch-backend.d.ts +47 -0
- package/dist/services/vector-backends/usearch-backend.d.ts.map +1 -0
- package/dist/services/vector-backends/usearch-backend.js +174 -0
- package/dist/services/web-server-worker.d.ts +2 -0
- package/dist/services/web-server-worker.d.ts.map +1 -0
- package/dist/services/web-server-worker.js +283 -0
- package/dist/services/web-server.d.ts +31 -0
- package/dist/services/web-server.d.ts.map +1 -0
- package/dist/services/web-server.js +356 -0
- package/dist/types/index.d.ts +19 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +1 -0
- package/dist/web/app.d.ts +2 -0
- package/dist/web/app.d.ts.map +1 -0
- package/dist/web/app.js +1238 -0
- package/dist/web/favicon.ico +0 -0
- package/dist/web/i18n.d.ts +2 -0
- package/dist/web/i18n.d.ts.map +1 -0
- package/dist/web/i18n.js +312 -0
- package/dist/web/index.html +293 -0
- package/dist/web/styles.css +1786 -0
- package/package.json +78 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { getDatabase } from "./sqlite-bootstrap.js";
|
|
2
|
+
import { existsSync, mkdirSync } from "node:fs";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
import { log } from "../logger.js";
|
|
5
|
+
import { CONFIG } from "../../config.js";
|
|
6
|
+
const Database = getDatabase();
|
|
7
|
+
export class ConnectionManager {
|
|
8
|
+
connections = new Map();
|
|
9
|
+
initDatabase(db) {
|
|
10
|
+
db.run("PRAGMA busy_timeout = 5000");
|
|
11
|
+
db.run("PRAGMA journal_mode = WAL");
|
|
12
|
+
db.run("PRAGMA synchronous = NORMAL");
|
|
13
|
+
db.run("PRAGMA cache_size = -64000");
|
|
14
|
+
db.run("PRAGMA temp_store = MEMORY");
|
|
15
|
+
db.run("PRAGMA foreign_keys = ON");
|
|
16
|
+
this.migrateSchema(db);
|
|
17
|
+
}
|
|
18
|
+
migrateSchema(db) {
|
|
19
|
+
try {
|
|
20
|
+
const columns = db.prepare("PRAGMA table_info(memories)").all();
|
|
21
|
+
const hasTags = columns.some((c) => c.name === "tags");
|
|
22
|
+
if (!hasTags && columns.length > 0) {
|
|
23
|
+
db.run("ALTER TABLE memories ADD COLUMN tags TEXT");
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
catch (error) {
|
|
27
|
+
log("Schema migration error", { error: String(error) });
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
getConnection(dbPath) {
|
|
31
|
+
if (this.connections.has(dbPath)) {
|
|
32
|
+
return this.connections.get(dbPath);
|
|
33
|
+
}
|
|
34
|
+
const dir = dirname(dbPath);
|
|
35
|
+
if (!existsSync(dir)) {
|
|
36
|
+
mkdirSync(dir, { recursive: true });
|
|
37
|
+
}
|
|
38
|
+
const db = new Database(dbPath);
|
|
39
|
+
this.initDatabase(db);
|
|
40
|
+
this.connections.set(dbPath, db);
|
|
41
|
+
return db;
|
|
42
|
+
}
|
|
43
|
+
closeConnection(dbPath) {
|
|
44
|
+
const db = this.connections.get(dbPath);
|
|
45
|
+
if (db) {
|
|
46
|
+
db.run("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
47
|
+
db.close();
|
|
48
|
+
this.connections.delete(dbPath);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
closeAll() {
|
|
52
|
+
for (const [path, db] of this.connections) {
|
|
53
|
+
try {
|
|
54
|
+
db.run("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
55
|
+
db.close();
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
log("Error closing database", { path, error: String(error) });
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
this.connections.clear();
|
|
62
|
+
}
|
|
63
|
+
checkpointAll() {
|
|
64
|
+
for (const [path, db] of this.connections) {
|
|
65
|
+
try {
|
|
66
|
+
db.run("PRAGMA wal_checkpoint(PASSIVE)");
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
log("Error checkpointing database", { path, error: String(error) });
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
export const connectionManager = new ConnectionManager();
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { ShardInfo } from "./types.js";
|
|
2
|
+
export declare class ShardManager {
|
|
3
|
+
private metadataDb;
|
|
4
|
+
private metadataPath;
|
|
5
|
+
constructor();
|
|
6
|
+
private initMetadataDb;
|
|
7
|
+
private getShardPath;
|
|
8
|
+
private resolveStoredPath;
|
|
9
|
+
getActiveShard(scope: "user" | "project", scopeHash: string): ShardInfo | null;
|
|
10
|
+
getAllShards(scope: "user" | "project", scopeHash: string): ShardInfo[];
|
|
11
|
+
createShard(scope: "user" | "project", scopeHash: string, shardIndex: number): ShardInfo;
|
|
12
|
+
private initShardDb;
|
|
13
|
+
private isShardValid;
|
|
14
|
+
private ensureShardTables;
|
|
15
|
+
getWriteShard(scope: "user" | "project", scopeHash: string): ShardInfo;
|
|
16
|
+
private markShardReadOnly;
|
|
17
|
+
incrementVectorCount(shardId: number): void;
|
|
18
|
+
decrementVectorCount(shardId: number): void;
|
|
19
|
+
getShardByPath(dbPath: string): ShardInfo | null;
|
|
20
|
+
deleteShard(shardId: number): Promise<void>;
|
|
21
|
+
}
|
|
22
|
+
export declare const shardManager: ShardManager;
|
|
23
|
+
//# sourceMappingURL=shard-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"shard-manager.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/shard-manager.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAO5C,qBAAa,YAAY;IACvB,OAAO,CAAC,UAAU,CAAe;IACjC,OAAO,CAAC,YAAY,CAAS;;IAQ7B,OAAO,CAAC,cAAc;IAqBtB,OAAO,CAAC,YAAY;IAKpB,OAAO,CAAC,iBAAiB;IAKzB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAsB9E,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,EAAE;IAgCvE,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS;IA2BxF,OAAO,CAAC,WAAW;IA8CnB,OAAO,CAAC,YAAY;IA4BpB,OAAO,CAAC,iBAAiB;IAYzB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS;IA+BtE,OAAO,CAAC,iBAAiB;IAOzB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAO3C,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAO3C,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAkB1C,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAkClD;AAED,eAAO,MAAM,YAAY,cAAqB,CAAC"}
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
import { getDatabase } from "./sqlite-bootstrap.js";
|
|
2
|
+
import { join, basename } from "node:path";
|
|
3
|
+
import { existsSync } from "node:fs";
|
|
4
|
+
import { CONFIG } from "../../config.js";
|
|
5
|
+
import { connectionManager } from "./connection-manager.js";
|
|
6
|
+
import { log } from "../logger.js";
|
|
7
|
+
import { vectorSearch } from "./vector-search.js";
|
|
8
|
+
const Database = getDatabase();
|
|
9
|
+
const METADATA_DB_NAME = "metadata.db";
|
|
10
|
+
export class ShardManager {
|
|
11
|
+
metadataDb;
|
|
12
|
+
metadataPath;
|
|
13
|
+
constructor() {
|
|
14
|
+
this.metadataPath = join(CONFIG.storagePath, METADATA_DB_NAME);
|
|
15
|
+
this.metadataDb = connectionManager.getConnection(this.metadataPath);
|
|
16
|
+
this.initMetadataDb();
|
|
17
|
+
}
|
|
18
|
+
initMetadataDb() {
|
|
19
|
+
this.metadataDb.run(`
|
|
20
|
+
CREATE TABLE IF NOT EXISTS shards (
|
|
21
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
22
|
+
scope TEXT NOT NULL,
|
|
23
|
+
scope_hash TEXT NOT NULL,
|
|
24
|
+
shard_index INTEGER NOT NULL,
|
|
25
|
+
db_path TEXT NOT NULL,
|
|
26
|
+
vector_count INTEGER DEFAULT 0,
|
|
27
|
+
is_active INTEGER DEFAULT 1,
|
|
28
|
+
created_at INTEGER NOT NULL,
|
|
29
|
+
UNIQUE(scope, scope_hash, shard_index)
|
|
30
|
+
)
|
|
31
|
+
`);
|
|
32
|
+
this.metadataDb.run(`
|
|
33
|
+
CREATE INDEX IF NOT EXISTS idx_active_shards
|
|
34
|
+
ON shards(scope, scope_hash, is_active)
|
|
35
|
+
`);
|
|
36
|
+
}
|
|
37
|
+
getShardPath(scope, scopeHash, shardIndex) {
|
|
38
|
+
const dir = join(CONFIG.storagePath, `${scope}s`);
|
|
39
|
+
return join(dir, `${scope}_${scopeHash}_shard_${shardIndex}.db`);
|
|
40
|
+
}
|
|
41
|
+
resolveStoredPath(storedPath, scope) {
|
|
42
|
+
const fileName = basename(storedPath);
|
|
43
|
+
return join(CONFIG.storagePath, `${scope}s`, fileName);
|
|
44
|
+
}
|
|
45
|
+
getActiveShard(scope, scopeHash) {
|
|
46
|
+
const stmt = this.metadataDb.prepare(`
|
|
47
|
+
SELECT * FROM shards
|
|
48
|
+
WHERE scope = ? AND scope_hash = ? AND is_active = 1
|
|
49
|
+
ORDER BY shard_index DESC LIMIT 1
|
|
50
|
+
`);
|
|
51
|
+
const row = stmt.get(scope, scopeHash);
|
|
52
|
+
if (!row)
|
|
53
|
+
return null;
|
|
54
|
+
return {
|
|
55
|
+
id: row.id,
|
|
56
|
+
scope: row.scope,
|
|
57
|
+
scopeHash: row.scope_hash,
|
|
58
|
+
shardIndex: row.shard_index,
|
|
59
|
+
dbPath: this.resolveStoredPath(row.db_path, row.scope),
|
|
60
|
+
vectorCount: row.vector_count,
|
|
61
|
+
isActive: row.is_active === 1,
|
|
62
|
+
createdAt: row.created_at,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
getAllShards(scope, scopeHash) {
|
|
66
|
+
let stmt;
|
|
67
|
+
let rows;
|
|
68
|
+
if (scopeHash === "") {
|
|
69
|
+
stmt = this.metadataDb.prepare(`
|
|
70
|
+
SELECT * FROM shards
|
|
71
|
+
WHERE scope = ?
|
|
72
|
+
ORDER BY shard_index ASC
|
|
73
|
+
`);
|
|
74
|
+
rows = stmt.all(scope);
|
|
75
|
+
}
|
|
76
|
+
else {
|
|
77
|
+
stmt = this.metadataDb.prepare(`
|
|
78
|
+
SELECT * FROM shards
|
|
79
|
+
WHERE scope = ? AND scope_hash = ?
|
|
80
|
+
ORDER BY shard_index ASC
|
|
81
|
+
`);
|
|
82
|
+
rows = stmt.all(scope, scopeHash);
|
|
83
|
+
}
|
|
84
|
+
return rows.map((row) => ({
|
|
85
|
+
id: row.id,
|
|
86
|
+
scope: row.scope,
|
|
87
|
+
scopeHash: row.scope_hash,
|
|
88
|
+
shardIndex: row.shard_index,
|
|
89
|
+
dbPath: this.resolveStoredPath(row.db_path, row.scope),
|
|
90
|
+
vectorCount: row.vector_count,
|
|
91
|
+
isActive: row.is_active === 1,
|
|
92
|
+
createdAt: row.created_at,
|
|
93
|
+
}));
|
|
94
|
+
}
|
|
95
|
+
createShard(scope, scopeHash, shardIndex) {
|
|
96
|
+
const fullPath = this.getShardPath(scope, scopeHash, shardIndex);
|
|
97
|
+
const storedPath = join(`${scope}s`, basename(fullPath)).replace(/\\/g, "/");
|
|
98
|
+
const now = Date.now();
|
|
99
|
+
const stmt = this.metadataDb.prepare(`
|
|
100
|
+
INSERT INTO shards (scope, scope_hash, shard_index, db_path, vector_count, is_active, created_at)
|
|
101
|
+
VALUES (?, ?, ?, ?, 0, 1, ?)
|
|
102
|
+
`);
|
|
103
|
+
const result = stmt.run(scope, scopeHash, shardIndex, storedPath, now);
|
|
104
|
+
const db = connectionManager.getConnection(fullPath);
|
|
105
|
+
this.initShardDb(db);
|
|
106
|
+
return {
|
|
107
|
+
id: Number(result.lastInsertRowid),
|
|
108
|
+
scope,
|
|
109
|
+
scopeHash,
|
|
110
|
+
shardIndex,
|
|
111
|
+
dbPath: fullPath,
|
|
112
|
+
vectorCount: 0,
|
|
113
|
+
isActive: true,
|
|
114
|
+
createdAt: now,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
initShardDb(db) {
|
|
118
|
+
db.run(`
|
|
119
|
+
CREATE TABLE IF NOT EXISTS shard_metadata (
|
|
120
|
+
key TEXT PRIMARY KEY,
|
|
121
|
+
value TEXT NOT NULL
|
|
122
|
+
)
|
|
123
|
+
`);
|
|
124
|
+
db.run(`
|
|
125
|
+
INSERT OR REPLACE INTO shard_metadata (key, value)
|
|
126
|
+
VALUES ('embedding_dimensions', '${CONFIG.embeddingDimensions}')
|
|
127
|
+
`);
|
|
128
|
+
db.run(`
|
|
129
|
+
INSERT OR REPLACE INTO shard_metadata (key, value)
|
|
130
|
+
VALUES ('embedding_model', '${CONFIG.embeddingModel}')
|
|
131
|
+
`);
|
|
132
|
+
db.run(`
|
|
133
|
+
CREATE TABLE IF NOT EXISTS memories (
|
|
134
|
+
id TEXT PRIMARY KEY,
|
|
135
|
+
content TEXT NOT NULL,
|
|
136
|
+
vector BLOB NOT NULL,
|
|
137
|
+
tags_vector BLOB,
|
|
138
|
+
container_tag TEXT NOT NULL,
|
|
139
|
+
tags TEXT,
|
|
140
|
+
type TEXT,
|
|
141
|
+
created_at INTEGER NOT NULL,
|
|
142
|
+
updated_at INTEGER NOT NULL,
|
|
143
|
+
metadata TEXT,
|
|
144
|
+
display_name TEXT,
|
|
145
|
+
user_name TEXT,
|
|
146
|
+
user_email TEXT,
|
|
147
|
+
project_path TEXT,
|
|
148
|
+
project_name TEXT,
|
|
149
|
+
git_repo_url TEXT,
|
|
150
|
+
is_pinned INTEGER DEFAULT 0
|
|
151
|
+
)
|
|
152
|
+
`);
|
|
153
|
+
db.run(`CREATE INDEX IF NOT EXISTS idx_container_tag ON memories(container_tag)`);
|
|
154
|
+
db.run(`CREATE INDEX IF NOT EXISTS idx_type ON memories(type)`);
|
|
155
|
+
db.run(`CREATE INDEX IF NOT EXISTS idx_created_at ON memories(created_at DESC)`);
|
|
156
|
+
db.run(`CREATE INDEX IF NOT EXISTS idx_is_pinned ON memories(is_pinned)`);
|
|
157
|
+
}
|
|
158
|
+
isShardValid(shard) {
|
|
159
|
+
if (!existsSync(shard.dbPath)) {
|
|
160
|
+
log("Shard DB file missing", { dbPath: shard.dbPath, shardId: shard.id });
|
|
161
|
+
return false;
|
|
162
|
+
}
|
|
163
|
+
try {
|
|
164
|
+
const db = connectionManager.getConnection(shard.dbPath);
|
|
165
|
+
const result = db
|
|
166
|
+
.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='memories'`)
|
|
167
|
+
.get();
|
|
168
|
+
if (!result) {
|
|
169
|
+
log("Shard DB missing 'memories' table", {
|
|
170
|
+
dbPath: shard.dbPath,
|
|
171
|
+
shardId: shard.id,
|
|
172
|
+
});
|
|
173
|
+
return false;
|
|
174
|
+
}
|
|
175
|
+
return true;
|
|
176
|
+
}
|
|
177
|
+
catch (error) {
|
|
178
|
+
log("Error validating shard DB", {
|
|
179
|
+
dbPath: shard.dbPath,
|
|
180
|
+
error: String(error),
|
|
181
|
+
});
|
|
182
|
+
return false;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
ensureShardTables(shard) {
|
|
186
|
+
try {
|
|
187
|
+
const db = connectionManager.getConnection(shard.dbPath);
|
|
188
|
+
this.initShardDb(db);
|
|
189
|
+
}
|
|
190
|
+
catch (error) {
|
|
191
|
+
log("Error ensuring shard tables", {
|
|
192
|
+
dbPath: shard.dbPath,
|
|
193
|
+
error: String(error),
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
getWriteShard(scope, scopeHash) {
|
|
198
|
+
let shard = this.getActiveShard(scope, scopeHash);
|
|
199
|
+
if (!shard) {
|
|
200
|
+
return this.createShard(scope, scopeHash, 0);
|
|
201
|
+
}
|
|
202
|
+
if (!this.isShardValid(shard)) {
|
|
203
|
+
log("Active shard is invalid, recreating", {
|
|
204
|
+
scope,
|
|
205
|
+
scopeHash,
|
|
206
|
+
shardIndex: shard.shardIndex,
|
|
207
|
+
dbPath: shard.dbPath,
|
|
208
|
+
});
|
|
209
|
+
connectionManager.closeConnection(shard.dbPath);
|
|
210
|
+
const deleteStmt = this.metadataDb.prepare(`DELETE FROM shards WHERE id = ?`);
|
|
211
|
+
deleteStmt.run(shard.id);
|
|
212
|
+
return this.createShard(scope, scopeHash, shard.shardIndex);
|
|
213
|
+
}
|
|
214
|
+
if (shard.vectorCount >= CONFIG.maxVectorsPerShard) {
|
|
215
|
+
this.markShardReadOnly(shard.id);
|
|
216
|
+
return this.createShard(scope, scopeHash, shard.shardIndex + 1);
|
|
217
|
+
}
|
|
218
|
+
return shard;
|
|
219
|
+
}
|
|
220
|
+
markShardReadOnly(shardId) {
|
|
221
|
+
const stmt = this.metadataDb.prepare(`
|
|
222
|
+
UPDATE shards SET is_active = 0 WHERE id = ?
|
|
223
|
+
`);
|
|
224
|
+
stmt.run(shardId);
|
|
225
|
+
}
|
|
226
|
+
incrementVectorCount(shardId) {
|
|
227
|
+
const stmt = this.metadataDb.prepare(`
|
|
228
|
+
UPDATE shards SET vector_count = vector_count + 1 WHERE id = ?
|
|
229
|
+
`);
|
|
230
|
+
stmt.run(shardId);
|
|
231
|
+
}
|
|
232
|
+
decrementVectorCount(shardId) {
|
|
233
|
+
const stmt = this.metadataDb.prepare(`
|
|
234
|
+
UPDATE shards SET vector_count = vector_count - 1 WHERE id = ? AND vector_count > 0
|
|
235
|
+
`);
|
|
236
|
+
stmt.run(shardId);
|
|
237
|
+
}
|
|
238
|
+
getShardByPath(dbPath) {
|
|
239
|
+
const fileName = basename(dbPath);
|
|
240
|
+
const stmt = this.metadataDb.prepare(`SELECT * FROM shards WHERE db_path LIKE '%' || ?`);
|
|
241
|
+
const row = stmt.get(fileName);
|
|
242
|
+
if (!row)
|
|
243
|
+
return null;
|
|
244
|
+
return {
|
|
245
|
+
id: row.id,
|
|
246
|
+
scope: row.scope,
|
|
247
|
+
scopeHash: row.scope_hash,
|
|
248
|
+
shardIndex: row.shard_index,
|
|
249
|
+
dbPath: this.resolveStoredPath(row.db_path, row.scope),
|
|
250
|
+
vectorCount: row.vector_count,
|
|
251
|
+
isActive: row.is_active === 1,
|
|
252
|
+
createdAt: row.created_at,
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
async deleteShard(shardId) {
|
|
256
|
+
const stmt = this.metadataDb.prepare(`SELECT * FROM shards WHERE id = ?`);
|
|
257
|
+
const row = stmt.get(shardId);
|
|
258
|
+
if (row) {
|
|
259
|
+
const fullPath = this.resolveStoredPath(row.db_path, row.scope);
|
|
260
|
+
await vectorSearch.deleteShardIndexes({
|
|
261
|
+
id: row.id,
|
|
262
|
+
scope: row.scope,
|
|
263
|
+
scopeHash: row.scope_hash,
|
|
264
|
+
shardIndex: row.shard_index,
|
|
265
|
+
dbPath: fullPath,
|
|
266
|
+
vectorCount: row.vector_count,
|
|
267
|
+
isActive: row.is_active === 1,
|
|
268
|
+
createdAt: row.created_at,
|
|
269
|
+
});
|
|
270
|
+
connectionManager.closeConnection(fullPath);
|
|
271
|
+
try {
|
|
272
|
+
const fs = require("node:fs");
|
|
273
|
+
if (fs.existsSync(fullPath)) {
|
|
274
|
+
fs.unlinkSync(fullPath);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
catch (error) {
|
|
278
|
+
log("Error deleting shard file", {
|
|
279
|
+
dbPath: fullPath,
|
|
280
|
+
error: String(error),
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
const deleteStmt = this.metadataDb.prepare(`DELETE FROM shards WHERE id = ?`);
|
|
284
|
+
deleteStmt.run(shardId);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
export const shardManager = new ShardManager();
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sqlite-bootstrap.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/sqlite-bootstrap.ts"],"names":[],"mappings":"AAEA,wBAAgB,WAAW,IAAI,cAAc,YAAY,EAAE,QAAQ,CAMlE"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
export interface ShardInfo {
|
|
2
|
+
id: number;
|
|
3
|
+
scope: "user" | "project";
|
|
4
|
+
scopeHash: string;
|
|
5
|
+
shardIndex: number;
|
|
6
|
+
dbPath: string;
|
|
7
|
+
vectorCount: number;
|
|
8
|
+
isActive: boolean;
|
|
9
|
+
createdAt: number;
|
|
10
|
+
}
|
|
11
|
+
export interface MemoryRecord {
|
|
12
|
+
id: string;
|
|
13
|
+
content: string;
|
|
14
|
+
vector: Float32Array;
|
|
15
|
+
tagsVector?: Float32Array;
|
|
16
|
+
containerTag: string;
|
|
17
|
+
tags?: string;
|
|
18
|
+
type?: string;
|
|
19
|
+
createdAt: number;
|
|
20
|
+
updatedAt: number;
|
|
21
|
+
metadata?: string;
|
|
22
|
+
displayName?: string;
|
|
23
|
+
userName?: string;
|
|
24
|
+
userEmail?: string;
|
|
25
|
+
projectPath?: string;
|
|
26
|
+
projectName?: string;
|
|
27
|
+
gitRepoUrl?: string;
|
|
28
|
+
}
|
|
29
|
+
export interface SearchResult {
|
|
30
|
+
id: string;
|
|
31
|
+
memory: string;
|
|
32
|
+
similarity: number;
|
|
33
|
+
tags?: string[];
|
|
34
|
+
metadata?: Record<string, unknown>;
|
|
35
|
+
displayName?: string;
|
|
36
|
+
userName?: string;
|
|
37
|
+
userEmail?: string;
|
|
38
|
+
projectPath?: string;
|
|
39
|
+
projectName?: string;
|
|
40
|
+
gitRepoUrl?: string;
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,YAAY,CAAC;IACrB,UAAU,CAAC,EAAE,YAAY,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { MemoryRecord, SearchResult, ShardInfo } from "./types.js";
|
|
2
|
+
import type { VectorBackend } from "../vector-backends/types.js";
|
|
3
|
+
declare const Database: typeof import("bun:sqlite").Database;
|
|
4
|
+
type DatabaseType = typeof Database.prototype;
|
|
5
|
+
export declare class VectorSearch {
|
|
6
|
+
private readonly backendPromise;
|
|
7
|
+
private readonly fallbackBackend;
|
|
8
|
+
constructor(backend?: VectorBackend, fallbackBackend?: VectorBackend);
|
|
9
|
+
private getBackend;
|
|
10
|
+
insertVector(db: DatabaseType, record: MemoryRecord, shard?: ShardInfo): Promise<void>;
|
|
11
|
+
searchInShard(shard: ShardInfo, queryVector: Float32Array, containerTag: string, limit: number, queryText?: string): Promise<SearchResult[]>;
|
|
12
|
+
searchAcrossShards(shards: ShardInfo[], queryVector: Float32Array, containerTag: string, limit: number, similarityThreshold: number, queryText?: string): Promise<SearchResult[]>;
|
|
13
|
+
deleteVector(db: DatabaseType, memoryId: string, shard?: ShardInfo): Promise<void>;
|
|
14
|
+
updateVector(db: DatabaseType, memoryId: string, vector: Float32Array, shard?: ShardInfo, tagsVector?: Float32Array): Promise<void>;
|
|
15
|
+
listMemories(db: DatabaseType, containerTag: string, limit: number): any[];
|
|
16
|
+
getAllMemories(db: DatabaseType): any[];
|
|
17
|
+
getMemoryById(db: DatabaseType, memoryId: string): any | null;
|
|
18
|
+
getMemoriesBySessionID(db: DatabaseType, sessionID: string): any[];
|
|
19
|
+
countVectors(db: DatabaseType, containerTag: string): number;
|
|
20
|
+
countAllVectors(db: DatabaseType): number;
|
|
21
|
+
getDistinctTags(db: DatabaseType): any[];
|
|
22
|
+
pinMemory(db: DatabaseType, memoryId: string): void;
|
|
23
|
+
unpinMemory(db: DatabaseType, memoryId: string): void;
|
|
24
|
+
rebuildIndexForShard(db: DatabaseType, scope: string, scopeHash: string, shardIndex: number): Promise<void>;
|
|
25
|
+
deleteShardIndexes(shard: ShardInfo): Promise<void>;
|
|
26
|
+
}
|
|
27
|
+
export declare const vectorSearch: VectorSearch;
|
|
28
|
+
export {};
|
|
29
|
+
//# sourceMappingURL=vector-search.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector-search.d.ts","sourceRoot":"","sources":["../../../src/services/sqlite/vector-search.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAGxE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAEjE,QAAA,MAAM,QAAQ,sCAAgB,CAAC;AAC/B,KAAK,YAAY,GAAG,OAAO,QAAQ,CAAC,SAAS,CAAC;AAM9C,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAyB;IACxD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgB;gBAEpC,OAAO,CAAC,EAAE,aAAa,EAAE,eAAe,GAAE,aAAsC;YAO9E,UAAU;IAIlB,YAAY,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAyCtF,aAAa,CACjB,KAAK,EAAE,SAAS,EAChB,WAAW,EAAE,YAAY,EACzB,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,MAAM,EACb,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,YAAY,EAAE,CAAC;IA8HpB,kBAAkB,CACtB,MAAM,EAAE,SAAS,EAAE,EACnB,WAAW,EAAE,YAAY,EACzB,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,MAAM,EACb,mBAAmB,EAAE,MAAM,EAC3B,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,YAAY,EAAE,CAAC;IAiBpB,YAAY,CAAC,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAUlF,YAAY,CAChB,EAAE,EAAE,YAAY,EAChB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,YAAY,EACpB,KAAK,CAAC,EAAE,SAAS,EACjB,UAAU,CAAC,EAAE,YAAY,GACxB,OAAO,CAAC,IAAI,CAAC;IAkBhB,YAAY,CAAC,EAAE,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,GAAG,EAAE;IAmB1E,cAAc,CAAC,EAAE,EAAE,YAAY,GAAG,GAAG,EAAE;IAKvC,aAAa,CAAC,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,GAAG,GAAG,GAAG,IAAI;IAK7D,sBAAsB,CAAC,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,GAAG,GAAG,EAAE;IAgBlE,YAAY,CAAC,EAAE,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM;IAM5D,eAAe,CAAC,EAAE,EAAE,YAAY,GAAG,MAAM;IAMzC,eAAe,CAAC,EAAE,EAAE,YAAY,GAAG,GAAG,EAAE;IAexC,SAAS,CAAC,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI;IAKnD,WAAW,CAAC,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK/C,oBAAoB,CACxB,EAAE,EAAE,YAAY,EAChB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC;IAgBV,kBAAkB,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;CAI1D;AAED,eAAO,MAAM,YAAY,cAAqB,CAAC"}
|