@tekmidian/pai 0.25.2 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auto-route-BWGvvpcP.mjs +86 -0
- package/dist/auto-route-BWGvvpcP.mjs.map +1 -0
- package/dist/cli/index.mjs +3 -3
- package/dist/cli/program.mjs +5 -4
- package/dist/cli/program.mjs.map +1 -1
- package/dist/clusters-BdvGIoD-.mjs +201 -0
- package/dist/clusters-BdvGIoD-.mjs.map +1 -0
- package/dist/daemon/index.mjs +6 -5
- package/dist/daemon/index.mjs.map +1 -1
- package/dist/daemon-BaBTOB1P.mjs +1356 -0
- package/dist/daemon-BaBTOB1P.mjs.map +1 -0
- package/dist/detector-DExEQ5cW.mjs +74 -0
- package/dist/detector-DExEQ5cW.mjs.map +1 -0
- package/dist/factory-CAy0N1xR.mjs +142 -0
- package/dist/factory-CAy0N1xR.mjs.map +1 -0
- package/dist/indexer-backend-vOJqSE0U.mjs +299 -0
- package/dist/indexer-backend-vOJqSE0U.mjs.map +1 -0
- package/dist/latent-ideas-DuM_kgkW.mjs +191 -0
- package/dist/latent-ideas-DuM_kgkW.mjs.map +1 -0
- package/dist/main-resolver-Bzloyzi0.mjs +1352 -0
- package/dist/main-resolver-Bzloyzi0.mjs.map +1 -0
- package/dist/neighborhood-DSkvFMAv.mjs +135 -0
- package/dist/neighborhood-DSkvFMAv.mjs.map +1 -0
- package/dist/note-context-DrcY4cWm.mjs +126 -0
- package/dist/note-context-DrcY4cWm.mjs.map +1 -0
- package/dist/pick-C_52sxvh.mjs +13336 -0
- package/dist/pick-C_52sxvh.mjs.map +1 -0
- package/dist/pick-E-q8lf6K.mjs +13336 -0
- package/dist/pick-E-q8lf6K.mjs.map +1 -0
- package/dist/pick-FrfZ_iY7.mjs +13299 -0
- package/dist/pick-FrfZ_iY7.mjs.map +1 -0
- package/dist/pick-sZo7IQi5.mjs +13350 -0
- package/dist/pick-sZo7IQi5.mjs.map +1 -0
- package/dist/postgres-DTyxU4B1.mjs +891 -0
- package/dist/postgres-DTyxU4B1.mjs.map +1 -0
- package/dist/query-feedback-BX5nSyRm.mjs +76 -0
- package/dist/query-feedback-BX5nSyRm.mjs.map +1 -0
- package/dist/router-B2xR3gVP.mjs +228 -0
- package/dist/router-B2xR3gVP.mjs.map +1 -0
- package/dist/sqlite--BBAyXLH.mjs +271 -0
- package/dist/sqlite--BBAyXLH.mjs.map +1 -0
- package/dist/themes-DICajLf-.mjs +148 -0
- package/dist/themes-DICajLf-.mjs.map +1 -0
- package/dist/tools-DMAQxlOk.mjs +1939 -0
- package/dist/tools-DMAQxlOk.mjs.map +1 -0
- package/dist/trace-DfyGmMG_.mjs +137 -0
- package/dist/trace-DfyGmMG_.mjs.map +1 -0
- package/dist/vault-indexer-Dt8qXP-w.mjs +536 -0
- package/dist/vault-indexer-Dt8qXP-w.mjs.map +1 -0
- package/dist/work-queue-worker-Bt_UcbMy.mjs +1856 -0
- package/dist/work-queue-worker-Bt_UcbMy.mjs.map +1 -0
- package/dist/zettelkasten-C8BikWss.mjs +1063 -0
- package/dist/zettelkasten-C8BikWss.mjs.map +1 -0
- package/docs/commands/pause.md +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
import "./embeddings-Bn86ssxR.mjs";
|
|
2
|
+
import { a as searchMemorySemantic, r as searchMemory } from "./search-CpTv1I24.mjs";
|
|
3
|
+
|
|
4
|
+
//#region src/storage/sqlite.ts
|
|
5
|
+
var SQLiteBackend = class {
|
|
6
|
+
backendType = "sqlite";
|
|
7
|
+
db;
|
|
8
|
+
constructor(db) {
|
|
9
|
+
this.db = db;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Expose the raw better-sqlite3 Database handle.
|
|
13
|
+
* Used by the daemon to pass to indexAll() which still uses the synchronous API directly.
|
|
14
|
+
*/
|
|
15
|
+
getRawDb() {
|
|
16
|
+
return this.db;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Alias for getRawDb() — used by the dispatcher for operations that require
|
|
20
|
+
* direct SQLite access (e.g. memory_feedback, touchChunksLastAccessed).
|
|
21
|
+
*/
|
|
22
|
+
getSqliteDb() {
|
|
23
|
+
return this.db;
|
|
24
|
+
}
|
|
25
|
+
async close() {
|
|
26
|
+
try {
|
|
27
|
+
this.db.close();
|
|
28
|
+
} catch {}
|
|
29
|
+
}
|
|
30
|
+
async getStats() {
|
|
31
|
+
return {
|
|
32
|
+
files: this.db.prepare("SELECT COUNT(*) AS n FROM memory_files").get().n,
|
|
33
|
+
chunks: this.db.prepare("SELECT COUNT(*) AS n FROM memory_chunks").get().n
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
async getFileHash(projectId, path) {
|
|
37
|
+
return this.db.prepare("SELECT hash FROM memory_files WHERE project_id = ? AND path = ?").get(projectId, path)?.hash;
|
|
38
|
+
}
|
|
39
|
+
async upsertFile(file) {
|
|
40
|
+
this.db.prepare(`INSERT INTO memory_files (project_id, path, source, tier, hash, mtime, size)
|
|
41
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
42
|
+
ON CONFLICT(project_id, path) DO UPDATE SET
|
|
43
|
+
source = excluded.source,
|
|
44
|
+
tier = excluded.tier,
|
|
45
|
+
hash = excluded.hash,
|
|
46
|
+
mtime = excluded.mtime,
|
|
47
|
+
size = excluded.size`).run(file.projectId, file.path, file.source, file.tier, file.hash, file.mtime, file.size);
|
|
48
|
+
}
|
|
49
|
+
async getChunkIds(projectId, path) {
|
|
50
|
+
return this.db.prepare("SELECT id FROM memory_chunks WHERE project_id = ? AND path = ?").all(projectId, path).map((r) => r.id);
|
|
51
|
+
}
|
|
52
|
+
async deleteChunksForFile(projectId, path) {
|
|
53
|
+
const ids = await this.getChunkIds(projectId, path);
|
|
54
|
+
const deleteFts = this.db.prepare("DELETE FROM memory_fts WHERE id = ?");
|
|
55
|
+
const deleteChunks = this.db.prepare("DELETE FROM memory_chunks WHERE project_id = ? AND path = ?");
|
|
56
|
+
this.db.transaction(() => {
|
|
57
|
+
for (const id of ids) deleteFts.run(id);
|
|
58
|
+
deleteChunks.run(projectId, path);
|
|
59
|
+
})();
|
|
60
|
+
}
|
|
61
|
+
async insertChunks(chunks) {
|
|
62
|
+
if (chunks.length === 0) return;
|
|
63
|
+
const insertChunk = this.db.prepare(`INSERT INTO memory_chunks (id, project_id, source, tier, path, start_line, end_line, hash, text, updated_at)
|
|
64
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
65
|
+
const insertFts = this.db.prepare(`INSERT INTO memory_fts (text, id, project_id, path, source, tier, start_line, end_line)
|
|
66
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
67
|
+
this.db.transaction(() => {
|
|
68
|
+
for (const c of chunks) {
|
|
69
|
+
insertChunk.run(c.id, c.projectId, c.source, c.tier, c.path, c.startLine, c.endLine, c.hash, c.text, c.updatedAt);
|
|
70
|
+
insertFts.run(c.text, c.id, c.projectId, c.path, c.source, c.tier, c.startLine, c.endLine);
|
|
71
|
+
}
|
|
72
|
+
})();
|
|
73
|
+
}
|
|
74
|
+
async getDistinctChunkPaths(projectId) {
|
|
75
|
+
return this.db.prepare("SELECT DISTINCT path FROM memory_chunks WHERE project_id = ?").all(projectId).map((r) => r.path);
|
|
76
|
+
}
|
|
77
|
+
async deletePaths(projectId, paths) {
|
|
78
|
+
if (paths.length === 0) return;
|
|
79
|
+
const deleteFts = this.db.prepare("DELETE FROM memory_fts WHERE id = ?");
|
|
80
|
+
const deleteChunks = this.db.prepare("DELETE FROM memory_chunks WHERE project_id = ? AND path = ?");
|
|
81
|
+
const deleteFile = this.db.prepare("DELETE FROM memory_files WHERE project_id = ? AND path = ?");
|
|
82
|
+
this.db.transaction(() => {
|
|
83
|
+
for (const path of paths) {
|
|
84
|
+
const ids = this.db.prepare("SELECT id FROM memory_chunks WHERE project_id = ? AND path = ?").all(projectId, path);
|
|
85
|
+
for (const { id } of ids) deleteFts.run(id);
|
|
86
|
+
deleteChunks.run(projectId, path);
|
|
87
|
+
deleteFile.run(projectId, path);
|
|
88
|
+
}
|
|
89
|
+
})();
|
|
90
|
+
}
|
|
91
|
+
async getUnembeddedChunkIds(projectId, limit) {
|
|
92
|
+
const conditions = ["embedding IS NULL"];
|
|
93
|
+
const params = [];
|
|
94
|
+
if (projectId !== void 0) {
|
|
95
|
+
conditions.push("project_id = ?");
|
|
96
|
+
params.push(projectId);
|
|
97
|
+
}
|
|
98
|
+
const where = "WHERE " + conditions.join(" AND ");
|
|
99
|
+
return this.db.prepare(`SELECT id, text, project_id, path FROM memory_chunks ${where}
|
|
100
|
+
ORDER BY CASE
|
|
101
|
+
WHEN path LIKE '🧠 Ideaverse/%' THEN 0
|
|
102
|
+
WHEN path LIKE 'Z - Zettelkasten/%' THEN 0
|
|
103
|
+
WHEN path LIKE '💼 Business/%' THEN 0
|
|
104
|
+
WHEN path LIKE '📆 Meetings/%' THEN 1
|
|
105
|
+
WHEN path LIKE '💡 Insights/%' THEN 1
|
|
106
|
+
WHEN path LIKE '👨💻 People/%' THEN 1
|
|
107
|
+
WHEN path LIKE 'University/%' THEN 1
|
|
108
|
+
WHEN path LIKE 'Copilot/%' THEN 1
|
|
109
|
+
WHEN path LIKE '🗓️ Daily Notes/%' THEN 2
|
|
110
|
+
WHEN path LIKE 'PAI/%' THEN 3
|
|
111
|
+
WHEN path LIKE '09-job-search/%' THEN 4
|
|
112
|
+
WHEN path LIKE 'seriousletter/%' THEN 4
|
|
113
|
+
WHEN path LIKE 'Attachments/%' THEN 5
|
|
114
|
+
ELSE 2
|
|
115
|
+
END, id${limit !== void 0 ? " LIMIT ?" : ""}`).all(...params, ...limit !== void 0 ? [limit] : []);
|
|
116
|
+
}
|
|
117
|
+
async updateEmbedding(chunkId, embedding) {
|
|
118
|
+
this.db.prepare("UPDATE memory_chunks SET embedding = ? WHERE id = ?").run(embedding, chunkId);
|
|
119
|
+
}
|
|
120
|
+
async searchKeyword(query, opts) {
|
|
121
|
+
return searchMemory(this.db, query, opts);
|
|
122
|
+
}
|
|
123
|
+
async searchSemantic(queryEmbedding, opts) {
|
|
124
|
+
return searchMemorySemantic(this.db, queryEmbedding, opts);
|
|
125
|
+
}
|
|
126
|
+
vaultNotSupported() {
|
|
127
|
+
throw new Error("Vault operations require the Postgres backend");
|
|
128
|
+
}
|
|
129
|
+
async upsertVaultFile() {
|
|
130
|
+
this.vaultNotSupported();
|
|
131
|
+
}
|
|
132
|
+
async deleteVaultFile() {
|
|
133
|
+
this.vaultNotSupported();
|
|
134
|
+
}
|
|
135
|
+
async getVaultFile() {
|
|
136
|
+
this.vaultNotSupported();
|
|
137
|
+
}
|
|
138
|
+
async getVaultFileByInode() {
|
|
139
|
+
this.vaultNotSupported();
|
|
140
|
+
}
|
|
141
|
+
async getAllVaultFiles() {
|
|
142
|
+
this.vaultNotSupported();
|
|
143
|
+
}
|
|
144
|
+
async getRecentVaultFiles() {
|
|
145
|
+
this.vaultNotSupported();
|
|
146
|
+
}
|
|
147
|
+
async countVaultFiles() {
|
|
148
|
+
this.vaultNotSupported();
|
|
149
|
+
}
|
|
150
|
+
async upsertVaultAliases() {
|
|
151
|
+
this.vaultNotSupported();
|
|
152
|
+
}
|
|
153
|
+
async deleteVaultAliases() {
|
|
154
|
+
this.vaultNotSupported();
|
|
155
|
+
}
|
|
156
|
+
async replaceLinksForSources() {
|
|
157
|
+
this.vaultNotSupported();
|
|
158
|
+
}
|
|
159
|
+
async getLinksFromSource() {
|
|
160
|
+
this.vaultNotSupported();
|
|
161
|
+
}
|
|
162
|
+
async getLinksToTarget() {
|
|
163
|
+
this.vaultNotSupported();
|
|
164
|
+
}
|
|
165
|
+
async getVaultLinkGraph() {
|
|
166
|
+
this.vaultNotSupported();
|
|
167
|
+
}
|
|
168
|
+
async upsertVaultHealth() {
|
|
169
|
+
this.vaultNotSupported();
|
|
170
|
+
}
|
|
171
|
+
async getVaultHealth() {
|
|
172
|
+
this.vaultNotSupported();
|
|
173
|
+
}
|
|
174
|
+
async getOrphans() {
|
|
175
|
+
this.vaultNotSupported();
|
|
176
|
+
}
|
|
177
|
+
async getDeadLinks() {
|
|
178
|
+
this.vaultNotSupported();
|
|
179
|
+
}
|
|
180
|
+
async upsertNameIndex() {
|
|
181
|
+
this.vaultNotSupported();
|
|
182
|
+
}
|
|
183
|
+
async replaceNameIndex() {
|
|
184
|
+
this.vaultNotSupported();
|
|
185
|
+
}
|
|
186
|
+
async resolveVaultName() {
|
|
187
|
+
this.vaultNotSupported();
|
|
188
|
+
}
|
|
189
|
+
async searchVaultNameIndex() {
|
|
190
|
+
this.vaultNotSupported();
|
|
191
|
+
}
|
|
192
|
+
async getVaultFilesByPaths() {
|
|
193
|
+
this.vaultNotSupported();
|
|
194
|
+
}
|
|
195
|
+
async getVaultFilesByPathsAfter() {
|
|
196
|
+
this.vaultNotSupported();
|
|
197
|
+
}
|
|
198
|
+
async getVaultLinksFromPaths() {
|
|
199
|
+
this.vaultNotSupported();
|
|
200
|
+
}
|
|
201
|
+
async getChunksWithEmbeddings() {
|
|
202
|
+
this.vaultNotSupported();
|
|
203
|
+
}
|
|
204
|
+
async getChunksForPath() {
|
|
205
|
+
this.vaultNotSupported();
|
|
206
|
+
}
|
|
207
|
+
async searchChunksByText() {
|
|
208
|
+
this.vaultNotSupported();
|
|
209
|
+
}
|
|
210
|
+
async countVaultFilesWithPrefix() {
|
|
211
|
+
this.vaultNotSupported();
|
|
212
|
+
}
|
|
213
|
+
async countVaultFilesAfter() {
|
|
214
|
+
this.vaultNotSupported();
|
|
215
|
+
}
|
|
216
|
+
async countVaultLinksWithPrefix() {
|
|
217
|
+
this.vaultNotSupported();
|
|
218
|
+
}
|
|
219
|
+
async countVaultLinksAfter() {
|
|
220
|
+
this.vaultNotSupported();
|
|
221
|
+
}
|
|
222
|
+
async getDeadLinksWithLineNumbers() {
|
|
223
|
+
this.vaultNotSupported();
|
|
224
|
+
}
|
|
225
|
+
async getDeadLinksWithPrefix() {
|
|
226
|
+
this.vaultNotSupported();
|
|
227
|
+
}
|
|
228
|
+
async getDeadLinksAfter() {
|
|
229
|
+
this.vaultNotSupported();
|
|
230
|
+
}
|
|
231
|
+
async getOrphansWithPrefix() {
|
|
232
|
+
this.vaultNotSupported();
|
|
233
|
+
}
|
|
234
|
+
async getOrphansAfter() {
|
|
235
|
+
this.vaultNotSupported();
|
|
236
|
+
}
|
|
237
|
+
async getLowConnectivity() {
|
|
238
|
+
this.vaultNotSupported();
|
|
239
|
+
}
|
|
240
|
+
async getLowConnectivityWithPrefix() {
|
|
241
|
+
this.vaultNotSupported();
|
|
242
|
+
}
|
|
243
|
+
async getLowConnectivityAfter() {
|
|
244
|
+
this.vaultNotSupported();
|
|
245
|
+
}
|
|
246
|
+
async getAllVaultFilePaths() {
|
|
247
|
+
this.vaultNotSupported();
|
|
248
|
+
}
|
|
249
|
+
async getVaultFilePathsWithPrefix() {
|
|
250
|
+
this.vaultNotSupported();
|
|
251
|
+
}
|
|
252
|
+
async getVaultFilePathsAfter() {
|
|
253
|
+
this.vaultNotSupported();
|
|
254
|
+
}
|
|
255
|
+
async getVaultLinkEdges() {
|
|
256
|
+
this.vaultNotSupported();
|
|
257
|
+
}
|
|
258
|
+
async getVaultLinkEdgesWithPrefix() {
|
|
259
|
+
this.vaultNotSupported();
|
|
260
|
+
}
|
|
261
|
+
async getVaultLinkEdgesAfter() {
|
|
262
|
+
this.vaultNotSupported();
|
|
263
|
+
}
|
|
264
|
+
async getVaultAlias() {
|
|
265
|
+
this.vaultNotSupported();
|
|
266
|
+
}
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
//#endregion
|
|
270
|
+
export { SQLiteBackend };
|
|
271
|
+
//# sourceMappingURL=sqlite--BBAyXLH.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sqlite--BBAyXLH.mjs","names":[],"sources":["../src/storage/sqlite.ts"],"sourcesContent":["/**\n * SQLiteBackend — wraps the existing better-sqlite3 federation.db\n * behind the StorageBackend interface.\n *\n * This is a thin adapter. The heavy lifting is all in the existing\n * memory/indexer.ts and memory/search.ts code; we just provide a\n * backend-agnostic surface so the daemon and tools can call either\n * SQLite or Postgres transparently.\n */\n\nimport type { Database } from \"better-sqlite3\";\nimport type { StorageBackend, ChunkRow, FileRow, FederationStats } from \"./interface.js\";\nimport type { SearchResult, SearchOptions } from \"../memory/search.js\";\nimport { searchMemory, searchMemorySemantic } from \"../memory/search.js\";\n\nexport class SQLiteBackend implements StorageBackend {\n readonly backendType = \"sqlite\" as const;\n\n private db: Database;\n\n constructor(db: Database) {\n this.db = db;\n }\n\n /**\n * Expose the raw better-sqlite3 Database handle.\n * Used by the daemon to pass to indexAll() which still uses the synchronous API directly.\n */\n getRawDb(): Database {\n return this.db;\n }\n\n /**\n * Alias for getRawDb() — used by the dispatcher for operations that require\n * direct SQLite access (e.g. memory_feedback, touchChunksLastAccessed).\n */\n getSqliteDb(): Database {\n return this.db;\n }\n\n // -------------------------------------------------------------------------\n // Lifecycle\n // -------------------------------------------------------------------------\n\n async close(): Promise<void> {\n try {\n this.db.close();\n } catch {\n // ignore\n }\n }\n\n async getStats(): Promise<FederationStats> {\n const files = (\n this.db.prepare(\"SELECT COUNT(*) AS n FROM memory_files\").get() as { n: number }\n ).n;\n const chunks = (\n this.db.prepare(\"SELECT COUNT(*) AS n FROM memory_chunks\").get() as { n: number }\n ).n;\n return { files, chunks };\n }\n\n // -------------------------------------------------------------------------\n // File tracking\n // -------------------------------------------------------------------------\n\n async getFileHash(projectId: number, path: string): Promise<string | undefined> {\n const row = this.db\n .prepare(\"SELECT hash FROM memory_files WHERE project_id = ? AND path = ?\")\n .get(projectId, path) as { hash: string } | undefined;\n return row?.hash;\n }\n\n async upsertFile(file: FileRow): Promise<void> {\n this.db\n .prepare(\n `INSERT INTO memory_files (project_id, path, source, tier, hash, mtime, size)\n VALUES (?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT(project_id, path) DO UPDATE SET\n source = excluded.source,\n tier = excluded.tier,\n hash = excluded.hash,\n mtime = excluded.mtime,\n size = excluded.size`\n )\n .run(file.projectId, file.path, file.source, file.tier, file.hash, file.mtime, file.size);\n }\n\n // -------------------------------------------------------------------------\n // Chunk management\n // -------------------------------------------------------------------------\n\n async getChunkIds(projectId: number, path: string): Promise<string[]> {\n const rows = this.db\n .prepare(\"SELECT id FROM memory_chunks WHERE project_id = ? AND path = ?\")\n .all(projectId, path) as Array<{ id: string }>;\n return rows.map((r) => r.id);\n }\n\n async deleteChunksForFile(projectId: number, path: string): Promise<void> {\n const ids = await this.getChunkIds(projectId, path);\n const deleteFts = this.db.prepare(\"DELETE FROM memory_fts WHERE id = ?\");\n const deleteChunks = this.db.prepare(\n \"DELETE FROM memory_chunks WHERE project_id = ? AND path = ?\"\n );\n this.db.transaction(() => {\n for (const id of ids) {\n deleteFts.run(id);\n }\n deleteChunks.run(projectId, path);\n })();\n }\n\n async insertChunks(chunks: ChunkRow[]): Promise<void> {\n if (chunks.length === 0) return;\n\n const insertChunk = this.db.prepare(\n `INSERT INTO memory_chunks (id, project_id, source, tier, path, start_line, end_line, hash, text, updated_at)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`\n );\n const insertFts = this.db.prepare(\n `INSERT INTO memory_fts (text, id, project_id, path, source, tier, start_line, end_line)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?)`\n );\n\n this.db.transaction(() => {\n for (const c of chunks) {\n insertChunk.run(\n c.id,\n c.projectId,\n c.source,\n c.tier,\n c.path,\n c.startLine,\n c.endLine,\n c.hash,\n c.text,\n c.updatedAt\n );\n insertFts.run(\n c.text,\n c.id,\n c.projectId,\n c.path,\n c.source,\n c.tier,\n c.startLine,\n c.endLine\n );\n }\n })();\n }\n\n async getDistinctChunkPaths(projectId: number): Promise<string[]> {\n const rows = this.db\n .prepare(\"SELECT DISTINCT path FROM memory_chunks WHERE project_id = ?\")\n .all(projectId) as Array<{ path: string }>;\n return rows.map((r) => r.path);\n }\n\n async deletePaths(projectId: number, paths: string[]): Promise<void> {\n if (paths.length === 0) return;\n const deleteFts = this.db.prepare(\"DELETE FROM memory_fts WHERE id = ?\");\n const deleteChunks = this.db.prepare(\n \"DELETE FROM memory_chunks WHERE project_id = ? AND path = ?\"\n );\n const deleteFile = this.db.prepare(\n \"DELETE FROM memory_files WHERE project_id = ? AND path = ?\"\n );\n this.db.transaction(() => {\n for (const path of paths) {\n const ids = this.db\n .prepare(\"SELECT id FROM memory_chunks WHERE project_id = ? AND path = ?\")\n .all(projectId, path) as Array<{ id: string }>;\n for (const { id } of ids) {\n deleteFts.run(id);\n }\n deleteChunks.run(projectId, path);\n deleteFile.run(projectId, path);\n }\n })();\n }\n\n async getUnembeddedChunkIds(projectId?: number, limit?: number): Promise<Array<{ id: string; text: string; project_id: number; path: string }>> {\n const conditions = [\"embedding IS NULL\"];\n const params: (string | number)[] = [];\n\n if (projectId !== undefined) {\n conditions.push(\"project_id = ?\");\n params.push(projectId);\n }\n\n const where = \"WHERE \" + conditions.join(\" AND \");\n // Prioritize real knowledge notes over PAI session/job-search noise.\n // CASE expression assigns lower priority numbers to knowledge paths.\n const rows = this.db\n .prepare(`SELECT id, text, project_id, path FROM memory_chunks ${where}\n ORDER BY CASE\n WHEN path LIKE '🧠 Ideaverse/%' THEN 0\n WHEN path LIKE 'Z - Zettelkasten/%' THEN 0\n WHEN path LIKE '💼 Business/%' THEN 0\n WHEN path LIKE '📆 Meetings/%' THEN 1\n WHEN path LIKE '💡 Insights/%' THEN 1\n WHEN path LIKE '👨💻 People/%' THEN 1\n WHEN path LIKE 'University/%' THEN 1\n WHEN path LIKE 'Copilot/%' THEN 1\n WHEN path LIKE '🗓️ Daily Notes/%' THEN 2\n WHEN path LIKE 'PAI/%' THEN 3\n WHEN path LIKE '09-job-search/%' THEN 4\n WHEN path LIKE 'seriousletter/%' THEN 4\n WHEN path LIKE 'Attachments/%' THEN 5\n ELSE 2\n END, id${limit !== undefined ? \" LIMIT ?\" : \"\"}`)\n .all(...params, ...(limit !== undefined ? [limit] : [])) as Array<{ id: string; text: string; project_id: number; path: string }>;\n return rows;\n }\n\n async updateEmbedding(chunkId: string, embedding: Buffer): Promise<void> {\n this.db\n .prepare(\"UPDATE memory_chunks SET embedding = ? WHERE id = ?\")\n .run(embedding, chunkId);\n }\n\n // -------------------------------------------------------------------------\n // Search\n // -------------------------------------------------------------------------\n\n async searchKeyword(query: string, opts?: SearchOptions): Promise<SearchResult[]> {\n return searchMemory(this.db, query, opts);\n }\n\n async searchSemantic(queryEmbedding: Float32Array, opts?: SearchOptions): Promise<SearchResult[]> {\n return searchMemorySemantic(this.db, queryEmbedding, opts);\n }\n\n // -------------------------------------------------------------------------\n // Vault operations — not supported on SQLite backend (use Postgres)\n // -------------------------------------------------------------------------\n\n private vaultNotSupported(): never {\n throw new Error(\"Vault operations require the Postgres backend\");\n }\n\n async upsertVaultFile(): Promise<void> { this.vaultNotSupported(); }\n async deleteVaultFile(): Promise<void> { this.vaultNotSupported(); }\n async getVaultFile(): Promise<null> { this.vaultNotSupported(); }\n async getVaultFileByInode(): Promise<null> { this.vaultNotSupported(); }\n async getAllVaultFiles(): Promise<never[]> { this.vaultNotSupported(); }\n async getRecentVaultFiles(): Promise<never[]> { this.vaultNotSupported(); }\n async countVaultFiles(): Promise<number> { this.vaultNotSupported(); }\n async upsertVaultAliases(): Promise<void> { this.vaultNotSupported(); }\n async deleteVaultAliases(): Promise<void> { this.vaultNotSupported(); }\n async replaceLinksForSources(): Promise<void> { this.vaultNotSupported(); }\n async getLinksFromSource(): Promise<never[]> { this.vaultNotSupported(); }\n async getLinksToTarget(): Promise<never[]> { this.vaultNotSupported(); }\n async getVaultLinkGraph(): Promise<never[]> { this.vaultNotSupported(); }\n async upsertVaultHealth(): Promise<void> { this.vaultNotSupported(); }\n async getVaultHealth(): Promise<null> { this.vaultNotSupported(); }\n async getOrphans(): Promise<never[]> { this.vaultNotSupported(); }\n async getDeadLinks(): Promise<never[]> { this.vaultNotSupported(); }\n async upsertNameIndex(): Promise<void> { this.vaultNotSupported(); }\n async replaceNameIndex(): Promise<void> { this.vaultNotSupported(); }\n async resolveVaultName(): Promise<never[]> { this.vaultNotSupported(); }\n async searchVaultNameIndex(): Promise<never[]> { this.vaultNotSupported(); }\n async getVaultFilesByPaths(): Promise<never[]> { this.vaultNotSupported(); }\n async getVaultFilesByPathsAfter(): Promise<never[]> { this.vaultNotSupported(); }\n async getVaultLinksFromPaths(): Promise<never[]> { this.vaultNotSupported(); }\n async getChunksWithEmbeddings(): Promise<never[]> { this.vaultNotSupported(); }\n async getChunksForPath(): Promise<never[]> { this.vaultNotSupported(); }\n async searchChunksByText(): Promise<never[]> { this.vaultNotSupported(); }\n async countVaultFilesWithPrefix(): Promise<number> { this.vaultNotSupported(); }\n async countVaultFilesAfter(): Promise<number> { this.vaultNotSupported(); }\n async countVaultLinksWithPrefix(): Promise<number> { this.vaultNotSupported(); }\n async countVaultLinksAfter(): Promise<number> { this.vaultNotSupported(); }\n async getDeadLinksWithLineNumbers(): Promise<never[]> { this.vaultNotSupported(); }\n async getDeadLinksWithPrefix(): Promise<never[]> { this.vaultNotSupported(); }\n async getDeadLinksAfter(): Promise<never[]> { this.vaultNotSupported(); }\n async getOrphansWithPrefix(): Promise<never[]> { this.vaultNotSupported(); }\n async getOrphansAfter(): Promise<never[]> { this.vaultNotSupported(); }\n async getLowConnectivity(): Promise<never[]> { this.vaultNotSupported(); }\n async getLowConnectivityWithPrefix(): Promise<never[]> { this.vaultNotSupported(); }\n async getLowConnectivityAfter(): Promise<never[]> { this.vaultNotSupported(); }\n async getAllVaultFilePaths(): Promise<never[]> { this.vaultNotSupported(); }\n async getVaultFilePathsWithPrefix(): Promise<never[]> { this.vaultNotSupported(); }\n async getVaultFilePathsAfter(): Promise<never[]> { this.vaultNotSupported(); }\n async getVaultLinkEdges(): Promise<never[]> { this.vaultNotSupported(); }\n async getVaultLinkEdgesWithPrefix(): Promise<never[]> { this.vaultNotSupported(); }\n async getVaultLinkEdgesAfter(): Promise<never[]> { this.vaultNotSupported(); }\n async getVaultAlias(): Promise<null> { this.vaultNotSupported(); }\n}\n"],"mappings":";;;;AAeA,IAAa,gBAAb,MAAqD;CACnD,AAAS,cAAc;CAEvB,AAAQ;CAER,YAAY,IAAc;AACxB,OAAK,KAAK;;;;;;CAOZ,WAAqB;AACnB,SAAO,KAAK;;;;;;CAOd,cAAwB;AACtB,SAAO,KAAK;;CAOd,MAAM,QAAuB;AAC3B,MAAI;AACF,QAAK,GAAG,OAAO;UACT;;CAKV,MAAM,WAAqC;AAOzC,SAAO;GAAE,OALP,KAAK,GAAG,QAAQ,yCAAyC,CAAC,KAAK,CAC/D;GAIc,QAFd,KAAK,GAAG,QAAQ,0CAA0C,CAAC,KAAK,CAChE;GACsB;;CAO1B,MAAM,YAAY,WAAmB,MAA2C;AAI9E,SAHY,KAAK,GACd,QAAQ,kEAAkE,CAC1E,IAAI,WAAW,KAAK,EACX;;CAGd,MAAM,WAAW,MAA8B;AAC7C,OAAK,GACF,QACC;;;;;;;mCAQD,CACA,IAAI,KAAK,WAAW,KAAK,MAAM,KAAK,QAAQ,KAAK,MAAM,KAAK,MAAM,KAAK,OAAO,KAAK,KAAK;;CAO7F,MAAM,YAAY,WAAmB,MAAiC;AAIpE,SAHa,KAAK,GACf,QAAQ,iEAAiE,CACzE,IAAI,WAAW,KAAK,CACX,KAAK,MAAM,EAAE,GAAG;;CAG9B,MAAM,oBAAoB,WAAmB,MAA6B;EACxE,MAAM,MAAM,MAAM,KAAK,YAAY,WAAW,KAAK;EACnD,MAAM,YAAY,KAAK,GAAG,QAAQ,sCAAsC;EACxE,MAAM,eAAe,KAAK,GAAG,QAC3B,8DACD;AACD,OAAK,GAAG,kBAAkB;AACxB,QAAK,MAAM,MAAM,IACf,WAAU,IAAI,GAAG;AAEnB,gBAAa,IAAI,WAAW,KAAK;IACjC,EAAE;;CAGN,MAAM,aAAa,QAAmC;AACpD,MAAI,OAAO,WAAW,EAAG;EAEzB,MAAM,cAAc,KAAK,GAAG,QAC1B;8CAED;EACD,MAAM,YAAY,KAAK,GAAG,QACxB;wCAED;AAED,OAAK,GAAG,kBAAkB;AACxB,QAAK,MAAM,KAAK,QAAQ;AACtB,gBAAY,IACV,EAAE,IACF,EAAE,WACF,EAAE,QACF,EAAE,MACF,EAAE,MACF,EAAE,WACF,EAAE,SACF,EAAE,MACF,EAAE,MACF,EAAE,UACH;AACD,cAAU,IACR,EAAE,MACF,EAAE,IACF,EAAE,WACF,EAAE,MACF,EAAE,QACF,EAAE,MACF,EAAE,WACF,EAAE,QACH;;IAEH,EAAE;;CAGN,MAAM,sBAAsB,WAAsC;AAIhE,SAHa,KAAK,GACf,QAAQ,+DAA+D,CACvE,IAAI,UAAU,CACL,KAAK,MAAM,EAAE,KAAK;;CAGhC,MAAM,YAAY,WAAmB,OAAgC;AACnE,MAAI,MAAM,WAAW,EAAG;EACxB,MAAM,YAAY,KAAK,GAAG,QAAQ,sCAAsC;EACxE,MAAM,eAAe,KAAK,GAAG,QAC3B,8DACD;EACD,MAAM,aAAa,KAAK,GAAG,QACzB,6DACD;AACD,OAAK,GAAG,kBAAkB;AACxB,QAAK,MAAM,QAAQ,OAAO;IACxB,MAAM,MAAM,KAAK,GACd,QAAQ,iEAAiE,CACzE,IAAI,WAAW,KAAK;AACvB,SAAK,MAAM,EAAE,QAAQ,IACnB,WAAU,IAAI,GAAG;AAEnB,iBAAa,IAAI,WAAW,KAAK;AACjC,eAAW,IAAI,WAAW,KAAK;;IAEjC,EAAE;;CAGN,MAAM,sBAAsB,WAAoB,OAAgG;EAC9I,MAAM,aAAa,CAAC,oBAAoB;EACxC,MAAM,SAA8B,EAAE;AAEtC,MAAI,cAAc,QAAW;AAC3B,cAAW,KAAK,iBAAiB;AACjC,UAAO,KAAK,UAAU;;EAGxB,MAAM,QAAQ,WAAW,WAAW,KAAK,QAAQ;AAsBjD,SAnBa,KAAK,GACf,QAAQ,wDAAwD,MAAM;;;;;;;;;;;;;;;;iBAgB5D,UAAU,SAAY,aAAa,KAAK,CAClD,IAAI,GAAG,QAAQ,GAAI,UAAU,SAAY,CAAC,MAAM,GAAG,EAAE,CAAE;;CAI5D,MAAM,gBAAgB,SAAiB,WAAkC;AACvE,OAAK,GACF,QAAQ,sDAAsD,CAC9D,IAAI,WAAW,QAAQ;;CAO5B,MAAM,cAAc,OAAe,MAA+C;AAChF,SAAO,aAAa,KAAK,IAAI,OAAO,KAAK;;CAG3C,MAAM,eAAe,gBAA8B,MAA+C;AAChG,SAAO,qBAAqB,KAAK,IAAI,gBAAgB,KAAK;;CAO5D,AAAQ,oBAA2B;AACjC,QAAM,IAAI,MAAM,gDAAgD;;CAGlE,MAAM,kBAAiC;AAAE,OAAK,mBAAmB;;CACjE,MAAM,kBAAiC;AAAE,OAAK,mBAAmB;;CACjE,MAAM,eAA8B;AAAE,OAAK,mBAAmB;;CAC9D,MAAM,sBAAqC;AAAE,OAAK,mBAAmB;;CACrE,MAAM,mBAAqC;AAAE,OAAK,mBAAmB;;CACrE,MAAM,sBAAwC;AAAE,OAAK,mBAAmB;;CACxE,MAAM,kBAAmC;AAAE,OAAK,mBAAmB;;CACnE,MAAM,qBAAoC;AAAE,OAAK,mBAAmB;;CACpE,MAAM,qBAAoC;AAAE,OAAK,mBAAmB;;CACpE,MAAM,yBAAwC;AAAE,OAAK,mBAAmB;;CACxE,MAAM,qBAAuC;AAAE,OAAK,mBAAmB;;CACvE,MAAM,mBAAqC;AAAE,OAAK,mBAAmB;;CACrE,MAAM,oBAAsC;AAAE,OAAK,mBAAmB;;CACtE,MAAM,oBAAmC;AAAE,OAAK,mBAAmB;;CACnE,MAAM,iBAAgC;AAAE,OAAK,mBAAmB;;CAChE,MAAM,aAA+B;AAAE,OAAK,mBAAmB;;CAC/D,MAAM,eAAiC;AAAE,OAAK,mBAAmB;;CACjE,MAAM,kBAAiC;AAAE,OAAK,mBAAmB;;CACjE,MAAM,mBAAkC;AAAE,OAAK,mBAAmB;;CAClE,MAAM,mBAAqC;AAAE,OAAK,mBAAmB;;CACrE,MAAM,uBAAyC;AAAE,OAAK,mBAAmB;;CACzE,MAAM,uBAAyC;AAAE,OAAK,mBAAmB;;CACzE,MAAM,4BAA8C;AAAE,OAAK,mBAAmB;;CAC9E,MAAM,yBAA2C;AAAE,OAAK,mBAAmB;;CAC3E,MAAM,0BAA4C;AAAE,OAAK,mBAAmB;;CAC5E,MAAM,mBAAqC;AAAE,OAAK,mBAAmB;;CACrE,MAAM,qBAAuC;AAAE,OAAK,mBAAmB;;CACvE,MAAM,4BAA6C;AAAE,OAAK,mBAAmB;;CAC7E,MAAM,uBAAwC;AAAE,OAAK,mBAAmB;;CACxE,MAAM,4BAA6C;AAAE,OAAK,mBAAmB;;CAC7E,MAAM,uBAAwC;AAAE,OAAK,mBAAmB;;CACxE,MAAM,8BAAgD;AAAE,OAAK,mBAAmB;;CAChF,MAAM,yBAA2C;AAAE,OAAK,mBAAmB;;CAC3E,MAAM,oBAAsC;AAAE,OAAK,mBAAmB;;CACtE,MAAM,uBAAyC;AAAE,OAAK,mBAAmB;;CACzE,MAAM,kBAAoC;AAAE,OAAK,mBAAmB;;CACpE,MAAM,qBAAuC;AAAE,OAAK,mBAAmB;;CACvE,MAAM,+BAAiD;AAAE,OAAK,mBAAmB;;CACjF,MAAM,0BAA4C;AAAE,OAAK,mBAAmB;;CAC5E,MAAM,uBAAyC;AAAE,OAAK,mBAAmB;;CACzE,MAAM,8BAAgD;AAAE,OAAK,mBAAmB;;CAChF,MAAM,yBAA2C;AAAE,OAAK,mBAAmB;;CAC3E,MAAM,oBAAsC;AAAE,OAAK,mBAAmB;;CACtE,MAAM,8BAAgD;AAAE,OAAK,mBAAmB;;CAChF,MAAM,yBAA2C;AAAE,OAAK,mBAAmB;;CAC3E,MAAM,gBAA+B;AAAE,OAAK,mBAAmB"}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { n as cosineSimilarity, r as deserializeEmbedding } from "./embeddings-Bn86ssxR.mjs";
|
|
2
|
+
import { t as STOP_WORDS } from "./stop-words-BaMEGVeY.mjs";
|
|
3
|
+
|
|
4
|
+
//#region src/zettelkasten/themes.ts
|
|
5
|
+
const MAX_CHUNKS = 5e3;
|
|
6
|
+
function getTopFolder(vaultPath) {
|
|
7
|
+
const parts = vaultPath.split("/");
|
|
8
|
+
return parts.length > 1 ? parts[0] : "";
|
|
9
|
+
}
|
|
10
|
+
function generateLabel(titles) {
|
|
11
|
+
const wordCounts = /* @__PURE__ */ new Map();
|
|
12
|
+
for (const title of titles) {
|
|
13
|
+
if (!title) continue;
|
|
14
|
+
const words = title.toLowerCase().replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter((w) => w.length > 2 && !STOP_WORDS.has(w));
|
|
15
|
+
for (const word of words) wordCounts.set(word, (wordCounts.get(word) ?? 0) + 1);
|
|
16
|
+
}
|
|
17
|
+
return [...wordCounts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map(([w]) => w).join(" / ");
|
|
18
|
+
}
|
|
19
|
+
async function computeLinkedRatio(backend, paths) {
|
|
20
|
+
if (paths.length < 2) return 0;
|
|
21
|
+
const totalPairs = paths.length * (paths.length - 1) / 2;
|
|
22
|
+
const pathSet = new Set(paths);
|
|
23
|
+
let linkedPairs = 0;
|
|
24
|
+
for (const path of paths) {
|
|
25
|
+
const links = await backend.getLinksFromSource(path);
|
|
26
|
+
for (const link of links) if (link.targetPath && pathSet.has(link.targetPath)) linkedPairs++;
|
|
27
|
+
}
|
|
28
|
+
const uniquePairs = linkedPairs / 2;
|
|
29
|
+
return Math.min(1, uniquePairs / totalPairs);
|
|
30
|
+
}
|
|
31
|
+
function averageEmbeddings(embeddings) {
|
|
32
|
+
if (embeddings.length === 0) return new Float32Array(0);
|
|
33
|
+
const dim = embeddings[0].length;
|
|
34
|
+
const sum = new Float32Array(dim);
|
|
35
|
+
for (const vec of embeddings) for (let i = 0; i < dim; i++) sum[i] += vec[i];
|
|
36
|
+
const avg = new Float32Array(dim);
|
|
37
|
+
for (let i = 0; i < dim; i++) avg[i] = sum[i] / embeddings.length;
|
|
38
|
+
return avg;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Detect emerging themes in recently-modified notes using agglomerative single-linkage
|
|
42
|
+
* clustering of note-level embeddings.
|
|
43
|
+
*/
|
|
44
|
+
async function zettelThemes(backend, opts) {
|
|
45
|
+
const lookbackDays = opts.lookbackDays ?? 30;
|
|
46
|
+
const minClusterSize = opts.minClusterSize ?? 3;
|
|
47
|
+
const maxThemes = opts.maxThemes ?? 10;
|
|
48
|
+
const similarityThreshold = opts.similarityThreshold ?? .65;
|
|
49
|
+
const now = Date.now();
|
|
50
|
+
const from = now - lookbackDays * 864e5;
|
|
51
|
+
const recentNotes = (await backend.getRecentVaultFiles(from)).map((f) => ({
|
|
52
|
+
vault_path: f.vaultPath,
|
|
53
|
+
title: f.title,
|
|
54
|
+
indexed_at: f.indexedAt
|
|
55
|
+
}));
|
|
56
|
+
const chunkRows = await backend.getChunksWithEmbeddings(opts.vaultProjectId, MAX_CHUNKS);
|
|
57
|
+
const embeddingsByPath = /* @__PURE__ */ new Map();
|
|
58
|
+
for (const row of chunkRows) {
|
|
59
|
+
const vec = deserializeEmbedding(row.embedding);
|
|
60
|
+
const arr = embeddingsByPath.get(row.path);
|
|
61
|
+
if (!arr) embeddingsByPath.set(row.path, [vec]);
|
|
62
|
+
else arr.push(vec);
|
|
63
|
+
}
|
|
64
|
+
const fileEmbeddings = /* @__PURE__ */ new Map();
|
|
65
|
+
for (const [path, vecs] of embeddingsByPath) fileEmbeddings.set(path, averageEmbeddings(vecs));
|
|
66
|
+
const clusters = [];
|
|
67
|
+
for (const note of recentNotes) {
|
|
68
|
+
const embedding = fileEmbeddings.get(note.vault_path);
|
|
69
|
+
if (!embedding) continue;
|
|
70
|
+
clusters.push({
|
|
71
|
+
paths: [note.vault_path],
|
|
72
|
+
titles: [note.title],
|
|
73
|
+
indexedAts: [note.indexed_at],
|
|
74
|
+
centroid: embedding
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
const totalNotesAnalyzed = clusters.length;
|
|
78
|
+
let merged = true;
|
|
79
|
+
while (merged && clusters.length > 1) {
|
|
80
|
+
merged = false;
|
|
81
|
+
let bestSim = similarityThreshold;
|
|
82
|
+
let bestI = -1;
|
|
83
|
+
let bestJ = -1;
|
|
84
|
+
for (let i = 0; i < clusters.length; i++) for (let j = i + 1; j < clusters.length; j++) {
|
|
85
|
+
const sim = cosineSimilarity(clusters[i].centroid, clusters[j].centroid);
|
|
86
|
+
if (sim > bestSim) {
|
|
87
|
+
bestSim = sim;
|
|
88
|
+
bestI = i;
|
|
89
|
+
bestJ = j;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
if (bestI === -1) break;
|
|
93
|
+
const ci = clusters[bestI];
|
|
94
|
+
const cj = clusters[bestJ];
|
|
95
|
+
const mergedPaths = [...ci.paths, ...cj.paths];
|
|
96
|
+
const mergedTitles = [...ci.titles, ...cj.titles];
|
|
97
|
+
const mergedIndexedAts = [...ci.indexedAts, ...cj.indexedAts];
|
|
98
|
+
const memberEmbeddings = [];
|
|
99
|
+
for (const p of mergedPaths) {
|
|
100
|
+
const emb = fileEmbeddings.get(p);
|
|
101
|
+
if (emb) memberEmbeddings.push(emb);
|
|
102
|
+
}
|
|
103
|
+
clusters[bestI] = {
|
|
104
|
+
paths: mergedPaths,
|
|
105
|
+
titles: mergedTitles,
|
|
106
|
+
indexedAts: mergedIndexedAts,
|
|
107
|
+
centroid: averageEmbeddings(memberEmbeddings)
|
|
108
|
+
};
|
|
109
|
+
clusters.splice(bestJ, 1);
|
|
110
|
+
merged = true;
|
|
111
|
+
}
|
|
112
|
+
const themes = [];
|
|
113
|
+
let clusterIndex = 0;
|
|
114
|
+
for (const cluster of clusters) {
|
|
115
|
+
if (cluster.paths.length < minClusterSize) continue;
|
|
116
|
+
const label = generateLabel(cluster.titles) || `Theme ${clusterIndex + 1}`;
|
|
117
|
+
const avgRecency = cluster.indexedAts.reduce((sum, t) => sum + t, 0) / cluster.indexedAts.length;
|
|
118
|
+
const folderDiversity = new Set(cluster.paths.map(getTopFolder)).size / cluster.paths.length;
|
|
119
|
+
const linkedRatio = await computeLinkedRatio(backend, cluster.paths);
|
|
120
|
+
const suggestIndexNote = linkedRatio < .3 && cluster.paths.length >= 5;
|
|
121
|
+
themes.push({
|
|
122
|
+
id: clusterIndex++,
|
|
123
|
+
label,
|
|
124
|
+
notes: cluster.paths.map((path, idx) => ({
|
|
125
|
+
path,
|
|
126
|
+
title: cluster.titles[idx]
|
|
127
|
+
})),
|
|
128
|
+
size: cluster.paths.length,
|
|
129
|
+
folderDiversity,
|
|
130
|
+
avgRecency,
|
|
131
|
+
linkedRatio,
|
|
132
|
+
suggestIndexNote
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
themes.sort((a, b) => b.size * b.folderDiversity * (b.avgRecency / now) - a.size * a.folderDiversity * (a.avgRecency / now));
|
|
136
|
+
return {
|
|
137
|
+
themes: themes.slice(0, maxThemes),
|
|
138
|
+
totalNotesAnalyzed,
|
|
139
|
+
timeWindow: {
|
|
140
|
+
from,
|
|
141
|
+
to: now
|
|
142
|
+
}
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
//#endregion
|
|
147
|
+
export { zettelThemes as t };
|
|
148
|
+
//# sourceMappingURL=themes-DICajLf-.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"themes-DICajLf-.mjs","names":[],"sources":["../src/zettelkasten/themes.ts"],"sourcesContent":["import type { StorageBackend } from \"../storage/interface.js\";\nimport { deserializeEmbedding, cosineSimilarity } from \"../memory/embeddings.js\";\nimport { STOP_WORDS } from \"../utils/stop-words.js\";\n\nexport interface ThemeOptions {\n vaultProjectId: number;\n lookbackDays?: number;\n minClusterSize?: number;\n maxThemes?: number;\n similarityThreshold?: number;\n}\n\nexport interface ThemeCluster {\n id: number;\n label: string;\n notes: Array<{\n path: string;\n title: string | null;\n }>;\n size: number;\n folderDiversity: number;\n avgRecency: number;\n linkedRatio: number;\n suggestIndexNote: boolean;\n}\n\nexport interface ThemeResult {\n themes: ThemeCluster[];\n totalNotesAnalyzed: number;\n timeWindow: { from: number; to: number };\n}\n\nconst MAX_CHUNKS = 5000;\n\n// STOP_WORDS imported from utils/stop-words.ts\n\nfunction getTopFolder(vaultPath: string): string {\n const parts = vaultPath.split(\"/\");\n return parts.length > 1 ? parts[0] : \"\";\n}\n\nfunction generateLabel(titles: Array<string | null>): string {\n const wordCounts = new Map<string, number>();\n for (const title of titles) {\n if (!title) continue;\n const words = title\n .toLowerCase()\n .replace(/[^a-z0-9\\s]/g, \" \")\n .split(/\\s+/)\n .filter((w) => w.length > 2 && !STOP_WORDS.has(w));\n for (const word of words) {\n wordCounts.set(word, (wordCounts.get(word) ?? 0) + 1);\n }\n }\n const sorted = [...wordCounts.entries()].sort((a, b) => b[1] - a[1]);\n return sorted\n .slice(0, 3)\n .map(([w]) => w)\n .join(\" / \");\n}\n\nasync function computeLinkedRatio(backend: StorageBackend, paths: string[]): Promise<number> {\n if (paths.length < 2) return 0;\n const totalPairs = (paths.length * (paths.length - 1)) / 2;\n const pathSet = new Set(paths);\n let linkedPairs = 0;\n\n for (const path of paths) {\n const links = await backend.getLinksFromSource(path);\n for (const link of links) {\n if (link.targetPath && pathSet.has(link.targetPath)) {\n linkedPairs++;\n }\n }\n }\n\n // Each bidirectional pair might be counted once per direction; divide by 2 to normalize\n const uniquePairs = linkedPairs / 2;\n return Math.min(1, uniquePairs / totalPairs);\n}\n\ntype ClusterNode = {\n paths: string[];\n titles: Array<string | null>;\n indexedAts: number[];\n centroid: Float32Array;\n};\n\nfunction averageEmbeddings(embeddings: Float32Array[]): Float32Array {\n if (embeddings.length === 0) return new Float32Array(0);\n const dim = embeddings[0].length;\n const sum = new Float32Array(dim);\n for (const vec of embeddings) {\n for (let i = 0; i < dim; i++) {\n sum[i] += vec[i];\n }\n }\n const avg = new Float32Array(dim);\n for (let i = 0; i < dim; i++) {\n avg[i] = sum[i] / embeddings.length;\n }\n return avg;\n}\n\n/**\n * Detect emerging themes in recently-modified notes using agglomerative single-linkage\n * clustering of note-level embeddings.\n */\nexport async function zettelThemes(\n backend: StorageBackend,\n opts: ThemeOptions,\n): Promise<ThemeResult> {\n const lookbackDays = opts.lookbackDays ?? 30;\n const minClusterSize = opts.minClusterSize ?? 3;\n const maxThemes = opts.maxThemes ?? 10;\n const similarityThreshold = opts.similarityThreshold ?? 0.65;\n\n const now = Date.now();\n const from = now - lookbackDays * 86400000;\n\n // Step 1: get recent notes\n const recentFiles = await backend.getRecentVaultFiles(from);\n const recentNotes = recentFiles.map(f => ({ vault_path: f.vaultPath, title: f.title, indexed_at: f.indexedAt }));\n\n // Step 2: get file-level embeddings from memory_chunks\n const chunkRows = await backend.getChunksWithEmbeddings(opts.vaultProjectId, MAX_CHUNKS);\n\n const embeddingsByPath = new Map<string, Float32Array[]>();\n for (const row of chunkRows) {\n const vec = deserializeEmbedding(row.embedding);\n const arr = embeddingsByPath.get(row.path);\n if (!arr) {\n embeddingsByPath.set(row.path, [vec]);\n } else {\n arr.push(vec);\n }\n }\n\n const fileEmbeddings = new Map<string, Float32Array>();\n for (const [path, vecs] of embeddingsByPath) {\n fileEmbeddings.set(path, averageEmbeddings(vecs));\n }\n\n // Step 3: build initial clusters — only include notes that have embeddings\n const clusters: ClusterNode[] = [];\n for (const note of recentNotes) {\n const embedding = fileEmbeddings.get(note.vault_path);\n if (!embedding) continue;\n clusters.push({\n paths: [note.vault_path],\n titles: [note.title],\n indexedAts: [note.indexed_at],\n centroid: embedding,\n });\n }\n\n const totalNotesAnalyzed = clusters.length;\n\n // Step 4: agglomerative single-linkage clustering\n // Stop when no two clusters have similarity >= threshold\n let merged = true;\n while (merged && clusters.length > 1) {\n merged = false;\n let bestSim = similarityThreshold;\n let bestI = -1;\n let bestJ = -1;\n\n for (let i = 0; i < clusters.length; i++) {\n for (let j = i + 1; j < clusters.length; j++) {\n const sim = cosineSimilarity(clusters[i].centroid, clusters[j].centroid);\n if (sim > bestSim) {\n bestSim = sim;\n bestI = i;\n bestJ = j;\n }\n }\n }\n\n if (bestI === -1) break;\n\n // Merge cluster j into cluster i\n const ci = clusters[bestI];\n const cj = clusters[bestJ];\n const mergedPaths = [...ci.paths, ...cj.paths];\n const mergedTitles = [...ci.titles, ...cj.titles];\n const mergedIndexedAts = [...ci.indexedAts, ...cj.indexedAts];\n\n // Recompute centroid from averaged embeddings of all member paths\n const memberEmbeddings: Float32Array[] = [];\n for (const p of mergedPaths) {\n const emb = fileEmbeddings.get(p);\n if (emb) memberEmbeddings.push(emb);\n }\n\n clusters[bestI] = {\n paths: mergedPaths,\n titles: mergedTitles,\n indexedAts: mergedIndexedAts,\n centroid: averageEmbeddings(memberEmbeddings),\n };\n\n clusters.splice(bestJ, 1);\n merged = true;\n }\n\n // Step 5: filter and annotate clusters\n const themes: ThemeCluster[] = [];\n let clusterIndex = 0;\n\n for (const cluster of clusters) {\n if (cluster.paths.length < minClusterSize) continue;\n\n const label = generateLabel(cluster.titles) || `Theme ${clusterIndex + 1}`;\n const avgRecency =\n cluster.indexedAts.reduce((sum, t) => sum + t, 0) / cluster.indexedAts.length;\n\n const uniqueFolders = new Set(cluster.paths.map(getTopFolder));\n const folderDiversity = uniqueFolders.size / cluster.paths.length;\n\n const linkedRatio = await computeLinkedRatio(backend, cluster.paths);\n const suggestIndexNote = linkedRatio < 0.3 && cluster.paths.length >= 5;\n\n themes.push({\n id: clusterIndex++,\n label,\n notes: cluster.paths.map((path, idx) => ({\n path,\n title: cluster.titles[idx],\n })),\n size: cluster.paths.length,\n folderDiversity,\n avgRecency,\n linkedRatio,\n suggestIndexNote,\n });\n }\n\n // Step 6: rank by size * folderDiversity * recency_ratio\n themes.sort(\n (a, b) =>\n b.size * b.folderDiversity * (b.avgRecency / now) -\n a.size * a.folderDiversity * (a.avgRecency / now),\n );\n\n return {\n themes: themes.slice(0, maxThemes),\n totalNotesAnalyzed,\n timeWindow: { from, to: now },\n };\n}\n"],"mappings":";;;;AAgCA,MAAM,aAAa;AAInB,SAAS,aAAa,WAA2B;CAC/C,MAAM,QAAQ,UAAU,MAAM,IAAI;AAClC,QAAO,MAAM,SAAS,IAAI,MAAM,KAAK;;AAGvC,SAAS,cAAc,QAAsC;CAC3D,MAAM,6BAAa,IAAI,KAAqB;AAC5C,MAAK,MAAM,SAAS,QAAQ;AAC1B,MAAI,CAAC,MAAO;EACZ,MAAM,QAAQ,MACX,aAAa,CACb,QAAQ,gBAAgB,IAAI,CAC5B,MAAM,MAAM,CACZ,QAAQ,MAAM,EAAE,SAAS,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC;AACpD,OAAK,MAAM,QAAQ,MACjB,YAAW,IAAI,OAAO,WAAW,IAAI,KAAK,IAAI,KAAK,EAAE;;AAIzD,QADe,CAAC,GAAG,WAAW,SAAS,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,KAAK,EAAE,GAAG,CAEjE,MAAM,GAAG,EAAE,CACX,KAAK,CAAC,OAAO,EAAE,CACf,KAAK,MAAM;;AAGhB,eAAe,mBAAmB,SAAyB,OAAkC;AAC3F,KAAI,MAAM,SAAS,EAAG,QAAO;CAC7B,MAAM,aAAc,MAAM,UAAU,MAAM,SAAS,KAAM;CACzD,MAAM,UAAU,IAAI,IAAI,MAAM;CAC9B,IAAI,cAAc;AAElB,MAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,QAAQ,MAAM,QAAQ,mBAAmB,KAAK;AACpD,OAAK,MAAM,QAAQ,MACjB,KAAI,KAAK,cAAc,QAAQ,IAAI,KAAK,WAAW,CACjD;;CAMN,MAAM,cAAc,cAAc;AAClC,QAAO,KAAK,IAAI,GAAG,cAAc,WAAW;;AAU9C,SAAS,kBAAkB,YAA0C;AACnE,KAAI,WAAW,WAAW,EAAG,QAAO,IAAI,aAAa,EAAE;CACvD,MAAM,MAAM,WAAW,GAAG;CAC1B,MAAM,MAAM,IAAI,aAAa,IAAI;AACjC,MAAK,MAAM,OAAO,WAChB,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,IACvB,KAAI,MAAM,IAAI;CAGlB,MAAM,MAAM,IAAI,aAAa,IAAI;AACjC,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,IACvB,KAAI,KAAK,IAAI,KAAK,WAAW;AAE/B,QAAO;;;;;;AAOT,eAAsB,aACpB,SACA,MACsB;CACtB,MAAM,eAAe,KAAK,gBAAgB;CAC1C,MAAM,iBAAiB,KAAK,kBAAkB;CAC9C,MAAM,YAAY,KAAK,aAAa;CACpC,MAAM,sBAAsB,KAAK,uBAAuB;CAExD,MAAM,MAAM,KAAK,KAAK;CACtB,MAAM,OAAO,MAAM,eAAe;CAIlC,MAAM,eADc,MAAM,QAAQ,oBAAoB,KAAK,EAC3B,KAAI,OAAM;EAAE,YAAY,EAAE;EAAW,OAAO,EAAE;EAAO,YAAY,EAAE;EAAW,EAAE;CAGhH,MAAM,YAAY,MAAM,QAAQ,wBAAwB,KAAK,gBAAgB,WAAW;CAExF,MAAM,mCAAmB,IAAI,KAA6B;AAC1D,MAAK,MAAM,OAAO,WAAW;EAC3B,MAAM,MAAM,qBAAqB,IAAI,UAAU;EAC/C,MAAM,MAAM,iBAAiB,IAAI,IAAI,KAAK;AAC1C,MAAI,CAAC,IACH,kBAAiB,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC;MAErC,KAAI,KAAK,IAAI;;CAIjB,MAAM,iCAAiB,IAAI,KAA2B;AACtD,MAAK,MAAM,CAAC,MAAM,SAAS,iBACzB,gBAAe,IAAI,MAAM,kBAAkB,KAAK,CAAC;CAInD,MAAM,WAA0B,EAAE;AAClC,MAAK,MAAM,QAAQ,aAAa;EAC9B,MAAM,YAAY,eAAe,IAAI,KAAK,WAAW;AACrD,MAAI,CAAC,UAAW;AAChB,WAAS,KAAK;GACZ,OAAO,CAAC,KAAK,WAAW;GACxB,QAAQ,CAAC,KAAK,MAAM;GACpB,YAAY,CAAC,KAAK,WAAW;GAC7B,UAAU;GACX,CAAC;;CAGJ,MAAM,qBAAqB,SAAS;CAIpC,IAAI,SAAS;AACb,QAAO,UAAU,SAAS,SAAS,GAAG;AACpC,WAAS;EACT,IAAI,UAAU;EACd,IAAI,QAAQ;EACZ,IAAI,QAAQ;AAEZ,OAAK,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,IACnC,MAAK,IAAI,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;GAC5C,MAAM,MAAM,iBAAiB,SAAS,GAAG,UAAU,SAAS,GAAG,SAAS;AACxE,OAAI,MAAM,SAAS;AACjB,cAAU;AACV,YAAQ;AACR,YAAQ;;;AAKd,MAAI,UAAU,GAAI;EAGlB,MAAM,KAAK,SAAS;EACpB,MAAM,KAAK,SAAS;EACpB,MAAM,cAAc,CAAC,GAAG,GAAG,OAAO,GAAG,GAAG,MAAM;EAC9C,MAAM,eAAe,CAAC,GAAG,GAAG,QAAQ,GAAG,GAAG,OAAO;EACjD,MAAM,mBAAmB,CAAC,GAAG,GAAG,YAAY,GAAG,GAAG,WAAW;EAG7D,MAAM,mBAAmC,EAAE;AAC3C,OAAK,MAAM,KAAK,aAAa;GAC3B,MAAM,MAAM,eAAe,IAAI,EAAE;AACjC,OAAI,IAAK,kBAAiB,KAAK,IAAI;;AAGrC,WAAS,SAAS;GAChB,OAAO;GACP,QAAQ;GACR,YAAY;GACZ,UAAU,kBAAkB,iBAAiB;GAC9C;AAED,WAAS,OAAO,OAAO,EAAE;AACzB,WAAS;;CAIX,MAAM,SAAyB,EAAE;CACjC,IAAI,eAAe;AAEnB,MAAK,MAAM,WAAW,UAAU;AAC9B,MAAI,QAAQ,MAAM,SAAS,eAAgB;EAE3C,MAAM,QAAQ,cAAc,QAAQ,OAAO,IAAI,SAAS,eAAe;EACvE,MAAM,aACJ,QAAQ,WAAW,QAAQ,KAAK,MAAM,MAAM,GAAG,EAAE,GAAG,QAAQ,WAAW;EAGzE,MAAM,kBADgB,IAAI,IAAI,QAAQ,MAAM,IAAI,aAAa,CAAC,CACxB,OAAO,QAAQ,MAAM;EAE3D,MAAM,cAAc,MAAM,mBAAmB,SAAS,QAAQ,MAAM;EACpE,MAAM,mBAAmB,cAAc,MAAO,QAAQ,MAAM,UAAU;AAEtE,SAAO,KAAK;GACV,IAAI;GACJ;GACA,OAAO,QAAQ,MAAM,KAAK,MAAM,SAAS;IACvC;IACA,OAAO,QAAQ,OAAO;IACvB,EAAE;GACH,MAAM,QAAQ,MAAM;GACpB;GACA;GACA;GACA;GACD,CAAC;;AAIJ,QAAO,MACJ,GAAG,MACF,EAAE,OAAO,EAAE,mBAAmB,EAAE,aAAa,OAC7C,EAAE,OAAO,EAAE,mBAAmB,EAAE,aAAa,KAChD;AAED,QAAO;EACL,QAAQ,OAAO,MAAM,GAAG,UAAU;EAClC;EACA,YAAY;GAAE;GAAM,IAAI;GAAK;EAC9B"}
|