@tekmidian/pai 0.17.0 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auto-route-tE6ymgYb.mjs +86 -0
- package/dist/auto-route-tE6ymgYb.mjs.map +1 -0
- package/dist/checkpoint-block-CkvwYA5y.mjs +1062 -0
- package/dist/checkpoint-block-CkvwYA5y.mjs.map +1 -0
- package/dist/cli/index.mjs +6 -5
- package/dist/cli/index.mjs.map +1 -1
- package/dist/cli/program.d.mts.map +1 -1
- package/dist/cli/program.mjs +7 -5
- package/dist/cli/program.mjs.map +1 -1
- package/dist/clusters-CRXU2T6Z.mjs +201 -0
- package/dist/clusters-CRXU2T6Z.mjs.map +1 -0
- package/dist/config-7UiKVmbE.mjs +203 -0
- package/dist/config-7UiKVmbE.mjs.map +1 -0
- package/dist/config-CcdkNSWa.mjs +204 -0
- package/dist/config-CcdkNSWa.mjs.map +1 -0
- package/dist/daemon/index.mjs +9 -9
- package/dist/daemon-DPHtKQB2.mjs +1576 -0
- package/dist/daemon-DPHtKQB2.mjs.map +1 -0
- package/dist/daemon-Pm6crudb.mjs +1576 -0
- package/dist/daemon-Pm6crudb.mjs.map +1 -0
- package/dist/daemon-mcp/index.mjs +2 -2
- package/dist/detector-BGw8SNWe.mjs +74 -0
- package/dist/detector-BGw8SNWe.mjs.map +1 -0
- package/dist/factory-1UI19STL.mjs +72 -0
- package/dist/factory-1UI19STL.mjs.map +1 -0
- package/dist/indexer-backend-B93q9ICi.mjs +299 -0
- package/dist/indexer-backend-B93q9ICi.mjs.map +1 -0
- package/dist/ipc-client-aVKVERjJ.mjs +152 -0
- package/dist/ipc-client-aVKVERjJ.mjs.map +1 -0
- package/dist/kg-entity-r8duqhi9.mjs +176 -0
- package/dist/kg-entity-r8duqhi9.mjs.map +1 -0
- package/dist/latent-ideas-CHrgdkSW.mjs +191 -0
- package/dist/latent-ideas-CHrgdkSW.mjs.map +1 -0
- package/dist/main-resolver-BozXCqpl.mjs +1340 -0
- package/dist/main-resolver-BozXCqpl.mjs.map +1 -0
- package/dist/neighborhood-jPQBkbB-.mjs +135 -0
- package/dist/neighborhood-jPQBkbB-.mjs.map +1 -0
- package/dist/note-context-D2S2sR38.mjs +126 -0
- package/dist/note-context-D2S2sR38.mjs.map +1 -0
- package/dist/pick-44g9gnCi.mjs +12329 -0
- package/dist/pick-44g9gnCi.mjs.map +1 -0
- package/dist/pick-BLU3NUzF.mjs +12602 -0
- package/dist/pick-BLU3NUzF.mjs.map +1 -0
- package/dist/pick-BROEyWxI.mjs +12665 -0
- package/dist/pick-BROEyWxI.mjs.map +1 -0
- package/dist/pick-BRdJ4gpT.mjs +12666 -0
- package/dist/pick-BRdJ4gpT.mjs.map +1 -0
- package/dist/pick-B_ZSZxR4.mjs +12301 -0
- package/dist/pick-B_ZSZxR4.mjs.map +1 -0
- package/dist/pick-Bl-g6BzM.mjs +12300 -0
- package/dist/pick-Bl-g6BzM.mjs.map +1 -0
- package/dist/pick-C1Gfz3nN.mjs +12587 -0
- package/dist/pick-C1Gfz3nN.mjs.map +1 -0
- package/dist/pick-Cj2seRfQ.mjs +12531 -0
- package/dist/pick-Cj2seRfQ.mjs.map +1 -0
- package/dist/pick-CpZSdQcU.mjs +12587 -0
- package/dist/pick-CpZSdQcU.mjs.map +1 -0
- package/dist/pick-DAHBqsS_.mjs +12665 -0
- package/dist/pick-DAHBqsS_.mjs.map +1 -0
- package/dist/pick-HsJkYu5D.mjs +12500 -0
- package/dist/pick-HsJkYu5D.mjs.map +1 -0
- package/dist/pick-SxeegWzK.mjs +12126 -0
- package/dist/pick-SxeegWzK.mjs.map +1 -0
- package/dist/pick-Y-Bnmlqx.mjs +12571 -0
- package/dist/pick-Y-Bnmlqx.mjs.map +1 -0
- package/dist/pick-sXVKiZhC.mjs +12498 -0
- package/dist/pick-sXVKiZhC.mjs.map +1 -0
- package/dist/postgres-D96XWfwI.mjs +891 -0
- package/dist/postgres-D96XWfwI.mjs.map +1 -0
- package/dist/query-feedback-CdBJ2icj.mjs +76 -0
- package/dist/query-feedback-CdBJ2icj.mjs.map +1 -0
- package/dist/runtime-paths-B0P1TvUr.mjs +50 -0
- package/dist/runtime-paths-B0P1TvUr.mjs.map +1 -0
- package/dist/sqlite-CR9aMImG.mjs +271 -0
- package/dist/sqlite-CR9aMImG.mjs.map +1 -0
- package/dist/state-DTvy-jRB.mjs +102 -0
- package/dist/state-DTvy-jRB.mjs.map +1 -0
- package/dist/themes-HzMKPTas.mjs +148 -0
- package/dist/themes-HzMKPTas.mjs.map +1 -0
- package/dist/tools-DuorP_SS.mjs +1939 -0
- package/dist/tools-DuorP_SS.mjs.map +1 -0
- package/dist/trace-Bf7OprC5.mjs +137 -0
- package/dist/trace-Bf7OprC5.mjs.map +1 -0
- package/dist/vault-indexer-qAfX09RY.mjs +536 -0
- package/dist/vault-indexer-qAfX09RY.mjs.map +1 -0
- package/dist/work-queue-worker-Bov9eWMC.mjs +1856 -0
- package/dist/work-queue-worker-Bov9eWMC.mjs.map +1 -0
- package/dist/zettelkasten-DcZk-XSm.mjs +1063 -0
- package/dist/zettelkasten-DcZk-XSm.mjs.map +1 -0
- package/docs/commands/README.md +7 -0
- package/docs/commands/backup.md +1 -1
- package/docs/commands/clear-names.md +1 -1
- package/docs/commands/daemon.md +1 -1
- package/docs/commands/db.md +1 -1
- package/docs/commands/end.md +1 -1
- package/docs/commands/help.md +1 -1
- package/docs/commands/identity.md +65 -0
- package/docs/commands/kg.md +1 -1
- package/docs/commands/mcp.md +1 -1
- package/docs/commands/memory.md +1 -1
- package/docs/commands/notify.md +1 -1
- package/docs/commands/observation.md +1 -1
- package/docs/commands/obsidian.md +1 -1
- package/docs/commands/pause.md +1 -1
- package/docs/commands/project.md +1 -1
- package/docs/commands/projects.md +1 -1
- package/docs/commands/registry.md +1 -1
- package/docs/commands/restore.md +1 -1
- package/docs/commands/session.md +1 -1
- package/docs/commands/sessions.md +1 -1
- package/docs/commands/setup.md +1 -1
- package/docs/commands/shell-init.md +1 -1
- package/docs/commands/skill.md +1 -1
- package/docs/commands/task.md +15 -2
- package/docs/commands/topic.md +1 -1
- package/docs/commands/update.md +1 -1
- package/docs/commands/zettel.md +1 -1
- package/docs/task-bus.md +185 -0
- package/package.json +1 -1
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { t as __exportAll } from "./rolldown-runtime-95iHPtFO.mjs";
|
|
2
|
+
|
|
3
|
+
//#region src/daemon/daemon/state.ts
|
|
4
|
+
var state_exports = /* @__PURE__ */ __exportAll({
|
|
5
|
+
daemonConfig: () => daemonConfig,
|
|
6
|
+
embedInProgress: () => embedInProgress,
|
|
7
|
+
embedSchedulerTimer: () => embedSchedulerTimer,
|
|
8
|
+
indexInProgress: () => indexInProgress,
|
|
9
|
+
indexSchedulerTimer: () => indexSchedulerTimer,
|
|
10
|
+
lastEmbedTime: () => lastEmbedTime,
|
|
11
|
+
lastIndexTime: () => lastIndexTime,
|
|
12
|
+
lastVaultIndexTime: () => lastVaultIndexTime,
|
|
13
|
+
notificationConfig: () => notificationConfig,
|
|
14
|
+
registryDb: () => registryDb,
|
|
15
|
+
setDaemonConfig: () => setDaemonConfig,
|
|
16
|
+
setEmbedInProgress: () => setEmbedInProgress,
|
|
17
|
+
setEmbedSchedulerTimer: () => setEmbedSchedulerTimer,
|
|
18
|
+
setIndexInProgress: () => setIndexInProgress,
|
|
19
|
+
setIndexSchedulerTimer: () => setIndexSchedulerTimer,
|
|
20
|
+
setLastEmbedTime: () => setLastEmbedTime,
|
|
21
|
+
setLastIndexTime: () => setLastIndexTime,
|
|
22
|
+
setLastVaultIndexTime: () => setLastVaultIndexTime,
|
|
23
|
+
setNotificationConfig: () => setNotificationConfig,
|
|
24
|
+
setRegistryDb: () => setRegistryDb,
|
|
25
|
+
setShutdownRequested: () => setShutdownRequested,
|
|
26
|
+
setStartTime: () => setStartTime,
|
|
27
|
+
setStorageBackend: () => setStorageBackend,
|
|
28
|
+
setVaultIndexInProgress: () => setVaultIndexInProgress,
|
|
29
|
+
shutdownRequested: () => shutdownRequested,
|
|
30
|
+
startTime: () => startTime,
|
|
31
|
+
storageBackend: () => storageBackend,
|
|
32
|
+
vaultIndexInProgress: () => vaultIndexInProgress
|
|
33
|
+
});
|
|
34
|
+
let registryDb;
|
|
35
|
+
let storageBackend;
|
|
36
|
+
let daemonConfig;
|
|
37
|
+
let startTime = Date.now();
|
|
38
|
+
/** True while a project index pass is running. */
|
|
39
|
+
let indexInProgress = false;
|
|
40
|
+
let lastIndexTime = 0;
|
|
41
|
+
let indexSchedulerTimer = null;
|
|
42
|
+
/** True while an embedding pass is running. */
|
|
43
|
+
let embedInProgress = false;
|
|
44
|
+
let lastEmbedTime = 0;
|
|
45
|
+
let embedSchedulerTimer = null;
|
|
46
|
+
/** True while a vault index pass is running. */
|
|
47
|
+
let vaultIndexInProgress = false;
|
|
48
|
+
let lastVaultIndexTime = 0;
|
|
49
|
+
/** Mutable notification config — loaded from disk at startup, patchable at runtime. */
|
|
50
|
+
let notificationConfig;
|
|
51
|
+
/**
|
|
52
|
+
* Set to true when a SIGTERM/SIGINT is received so that long-running loops
|
|
53
|
+
* (embed, index) can detect the signal and exit their inner loops before the
|
|
54
|
+
* pool/backend is closed.
|
|
55
|
+
*/
|
|
56
|
+
let shutdownRequested = false;
|
|
57
|
+
function setRegistryDb(db) {
|
|
58
|
+
registryDb = db;
|
|
59
|
+
}
|
|
60
|
+
function setStorageBackend(b) {
|
|
61
|
+
storageBackend = b;
|
|
62
|
+
}
|
|
63
|
+
function setDaemonConfig(c) {
|
|
64
|
+
daemonConfig = c;
|
|
65
|
+
}
|
|
66
|
+
function setStartTime(t) {
|
|
67
|
+
startTime = t;
|
|
68
|
+
}
|
|
69
|
+
function setNotificationConfig(c) {
|
|
70
|
+
notificationConfig = c;
|
|
71
|
+
}
|
|
72
|
+
function setShutdownRequested(v) {
|
|
73
|
+
shutdownRequested = v;
|
|
74
|
+
}
|
|
75
|
+
function setIndexInProgress(v) {
|
|
76
|
+
indexInProgress = v;
|
|
77
|
+
}
|
|
78
|
+
function setLastIndexTime(v) {
|
|
79
|
+
lastIndexTime = v;
|
|
80
|
+
}
|
|
81
|
+
function setIndexSchedulerTimer(v) {
|
|
82
|
+
indexSchedulerTimer = v;
|
|
83
|
+
}
|
|
84
|
+
function setEmbedInProgress(v) {
|
|
85
|
+
embedInProgress = v;
|
|
86
|
+
}
|
|
87
|
+
function setLastEmbedTime(v) {
|
|
88
|
+
lastEmbedTime = v;
|
|
89
|
+
}
|
|
90
|
+
function setEmbedSchedulerTimer(v) {
|
|
91
|
+
embedSchedulerTimer = v;
|
|
92
|
+
}
|
|
93
|
+
function setVaultIndexInProgress(v) {
|
|
94
|
+
vaultIndexInProgress = v;
|
|
95
|
+
}
|
|
96
|
+
function setLastVaultIndexTime(v) {
|
|
97
|
+
lastVaultIndexTime = v;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
//#endregion
|
|
101
|
+
export { setStorageBackend as C, state_exports as D, startTime as E, storageBackend as O, setStartTime as S, shutdownRequested as T, setLastIndexTime as _, indexSchedulerTimer as a, setRegistryDb as b, lastVaultIndexTime as c, setDaemonConfig as d, setEmbedInProgress as f, setLastEmbedTime as g, setIndexSchedulerTimer as h, indexInProgress as i, vaultIndexInProgress as k, notificationConfig as l, setIndexInProgress as m, embedInProgress as n, lastEmbedTime as o, setEmbedSchedulerTimer as p, embedSchedulerTimer as r, lastIndexTime as s, daemonConfig as t, registryDb as u, setLastVaultIndexTime as v, setVaultIndexInProgress as w, setShutdownRequested as x, setNotificationConfig as y };
|
|
102
|
+
//# sourceMappingURL=state-DTvy-jRB.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"state-DTvy-jRB.mjs","names":[],"sources":["../src/daemon/daemon/state.ts"],"sourcesContent":["/**\n * Shared mutable daemon state — module-level singletons used across all daemon sub-modules.\n */\n\nimport { openRegistry } from \"../../registry/db.js\";\nimport type { StorageBackend } from \"../../storage/interface.js\";\nimport type { NotificationConfig } from \"../../notifications/types.js\";\nimport type { PaiDaemonConfig } from \"../config.js\";\n\n// ---------------------------------------------------------------------------\n// Core singletons (assigned at startup by serve())\n// ---------------------------------------------------------------------------\n\nexport let registryDb: ReturnType<typeof openRegistry>;\nexport let storageBackend: StorageBackend;\nexport let daemonConfig: PaiDaemonConfig;\nexport let startTime = Date.now();\n\n// ---------------------------------------------------------------------------\n// Scheduler state\n// ---------------------------------------------------------------------------\n\n/** True while a project index pass is running. */\nexport let indexInProgress = false;\nexport let lastIndexTime = 0;\nexport let indexSchedulerTimer: ReturnType<typeof setInterval> | null = null;\n\n/** True while an embedding pass is running. */\nexport let embedInProgress = false;\nexport let lastEmbedTime = 0;\nexport let embedSchedulerTimer: ReturnType<typeof setInterval> | null = null;\n\n/** True while a vault index pass is running. */\nexport let vaultIndexInProgress = false;\nexport let lastVaultIndexTime = 0;\n\n// ---------------------------------------------------------------------------\n// Notification state\n// ---------------------------------------------------------------------------\n\n/** Mutable notification config — loaded from disk at startup, patchable at runtime. */\nexport let notificationConfig: NotificationConfig;\n\n// ---------------------------------------------------------------------------\n// Graceful shutdown flag\n// ---------------------------------------------------------------------------\n\n/**\n * Set to true when a SIGTERM/SIGINT is received so that long-running loops\n * (embed, index) can detect the signal and exit their inner loops before the\n * pool/backend is closed.\n */\nexport let shutdownRequested = false;\n\n// ---------------------------------------------------------------------------\n// Setters (TypeScript requires assignment functions for exported `let` vars\n// that need to be mutated across module boundaries)\n// ---------------------------------------------------------------------------\n\nexport function setRegistryDb(db: ReturnType<typeof openRegistry>): void { registryDb = db; }\nexport function setStorageBackend(b: StorageBackend): void { storageBackend = b; }\nexport function setDaemonConfig(c: PaiDaemonConfig): void { daemonConfig = c; }\nexport function setStartTime(t: number): void { startTime = t; }\nexport function setNotificationConfig(c: NotificationConfig): void { notificationConfig = c; }\nexport function setShutdownRequested(v: boolean): void { shutdownRequested = v; }\nexport function setIndexInProgress(v: boolean): void { indexInProgress = v; }\nexport function setLastIndexTime(v: number): void { lastIndexTime = v; }\nexport function setIndexSchedulerTimer(v: ReturnType<typeof setInterval> | null): void { indexSchedulerTimer = v; }\nexport function setEmbedInProgress(v: boolean): void { embedInProgress = v; }\nexport function setLastEmbedTime(v: number): void { lastEmbedTime = v; }\nexport function setEmbedSchedulerTimer(v: ReturnType<typeof setInterval> | null): void { embedSchedulerTimer = v; }\nexport function setVaultIndexInProgress(v: boolean): void { vaultIndexInProgress = v; }\nexport function setLastVaultIndexTime(v: number): void { lastVaultIndexTime = v; }\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAaA,IAAW;AACX,IAAW;AACX,IAAW;AACX,IAAW,YAAY,KAAK,KAAK;;AAOjC,IAAW,kBAAkB;AAC7B,IAAW,gBAAgB;AAC3B,IAAW,sBAA6D;;AAGxE,IAAW,kBAAkB;AAC7B,IAAW,gBAAgB;AAC3B,IAAW,sBAA6D;;AAGxE,IAAW,uBAAuB;AAClC,IAAW,qBAAqB;;AAOhC,IAAW;;;;;;AAWX,IAAW,oBAAoB;AAO/B,SAAgB,cAAc,IAA2C;AAAE,cAAa;;AACxF,SAAgB,kBAAkB,GAAyB;AAAE,kBAAiB;;AAC9E,SAAgB,gBAAgB,GAA0B;AAAE,gBAAe;;AAC3E,SAAgB,aAAa,GAAiB;AAAE,aAAY;;AAC5D,SAAgB,sBAAsB,GAA6B;AAAE,sBAAqB;;AAC1F,SAAgB,qBAAqB,GAAkB;AAAE,qBAAoB;;AAC7E,SAAgB,mBAAmB,GAAkB;AAAE,mBAAkB;;AACzE,SAAgB,iBAAiB,GAAiB;AAAE,iBAAgB;;AACpE,SAAgB,uBAAuB,GAAgD;AAAE,uBAAsB;;AAC/G,SAAgB,mBAAmB,GAAkB;AAAE,mBAAkB;;AACzE,SAAgB,iBAAiB,GAAiB;AAAE,iBAAgB;;AACpE,SAAgB,uBAAuB,GAAgD;AAAE,uBAAsB;;AAC/G,SAAgB,wBAAwB,GAAkB;AAAE,wBAAuB;;AACnF,SAAgB,sBAAsB,GAAiB;AAAE,sBAAqB"}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { n as cosineSimilarity, r as deserializeEmbedding } from "./embeddings-Bn86ssxR.mjs";
|
|
2
|
+
import { t as STOP_WORDS } from "./stop-words-BaMEGVeY.mjs";
|
|
3
|
+
|
|
4
|
+
//#region src/zettelkasten/themes.ts
|
|
5
|
+
const MAX_CHUNKS = 5e3;
|
|
6
|
+
function getTopFolder(vaultPath) {
|
|
7
|
+
const parts = vaultPath.split("/");
|
|
8
|
+
return parts.length > 1 ? parts[0] : "";
|
|
9
|
+
}
|
|
10
|
+
function generateLabel(titles) {
|
|
11
|
+
const wordCounts = /* @__PURE__ */ new Map();
|
|
12
|
+
for (const title of titles) {
|
|
13
|
+
if (!title) continue;
|
|
14
|
+
const words = title.toLowerCase().replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter((w) => w.length > 2 && !STOP_WORDS.has(w));
|
|
15
|
+
for (const word of words) wordCounts.set(word, (wordCounts.get(word) ?? 0) + 1);
|
|
16
|
+
}
|
|
17
|
+
return [...wordCounts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map(([w]) => w).join(" / ");
|
|
18
|
+
}
|
|
19
|
+
async function computeLinkedRatio(backend, paths) {
|
|
20
|
+
if (paths.length < 2) return 0;
|
|
21
|
+
const totalPairs = paths.length * (paths.length - 1) / 2;
|
|
22
|
+
const pathSet = new Set(paths);
|
|
23
|
+
let linkedPairs = 0;
|
|
24
|
+
for (const path of paths) {
|
|
25
|
+
const links = await backend.getLinksFromSource(path);
|
|
26
|
+
for (const link of links) if (link.targetPath && pathSet.has(link.targetPath)) linkedPairs++;
|
|
27
|
+
}
|
|
28
|
+
const uniquePairs = linkedPairs / 2;
|
|
29
|
+
return Math.min(1, uniquePairs / totalPairs);
|
|
30
|
+
}
|
|
31
|
+
function averageEmbeddings(embeddings) {
|
|
32
|
+
if (embeddings.length === 0) return new Float32Array(0);
|
|
33
|
+
const dim = embeddings[0].length;
|
|
34
|
+
const sum = new Float32Array(dim);
|
|
35
|
+
for (const vec of embeddings) for (let i = 0; i < dim; i++) sum[i] += vec[i];
|
|
36
|
+
const avg = new Float32Array(dim);
|
|
37
|
+
for (let i = 0; i < dim; i++) avg[i] = sum[i] / embeddings.length;
|
|
38
|
+
return avg;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Detect emerging themes in recently-modified notes using agglomerative single-linkage
|
|
42
|
+
* clustering of note-level embeddings.
|
|
43
|
+
*/
|
|
44
|
+
async function zettelThemes(backend, opts) {
|
|
45
|
+
const lookbackDays = opts.lookbackDays ?? 30;
|
|
46
|
+
const minClusterSize = opts.minClusterSize ?? 3;
|
|
47
|
+
const maxThemes = opts.maxThemes ?? 10;
|
|
48
|
+
const similarityThreshold = opts.similarityThreshold ?? .65;
|
|
49
|
+
const now = Date.now();
|
|
50
|
+
const from = now - lookbackDays * 864e5;
|
|
51
|
+
const recentNotes = (await backend.getRecentVaultFiles(from)).map((f) => ({
|
|
52
|
+
vault_path: f.vaultPath,
|
|
53
|
+
title: f.title,
|
|
54
|
+
indexed_at: f.indexedAt
|
|
55
|
+
}));
|
|
56
|
+
const chunkRows = await backend.getChunksWithEmbeddings(opts.vaultProjectId, MAX_CHUNKS);
|
|
57
|
+
const embeddingsByPath = /* @__PURE__ */ new Map();
|
|
58
|
+
for (const row of chunkRows) {
|
|
59
|
+
const vec = deserializeEmbedding(row.embedding);
|
|
60
|
+
const arr = embeddingsByPath.get(row.path);
|
|
61
|
+
if (!arr) embeddingsByPath.set(row.path, [vec]);
|
|
62
|
+
else arr.push(vec);
|
|
63
|
+
}
|
|
64
|
+
const fileEmbeddings = /* @__PURE__ */ new Map();
|
|
65
|
+
for (const [path, vecs] of embeddingsByPath) fileEmbeddings.set(path, averageEmbeddings(vecs));
|
|
66
|
+
const clusters = [];
|
|
67
|
+
for (const note of recentNotes) {
|
|
68
|
+
const embedding = fileEmbeddings.get(note.vault_path);
|
|
69
|
+
if (!embedding) continue;
|
|
70
|
+
clusters.push({
|
|
71
|
+
paths: [note.vault_path],
|
|
72
|
+
titles: [note.title],
|
|
73
|
+
indexedAts: [note.indexed_at],
|
|
74
|
+
centroid: embedding
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
const totalNotesAnalyzed = clusters.length;
|
|
78
|
+
let merged = true;
|
|
79
|
+
while (merged && clusters.length > 1) {
|
|
80
|
+
merged = false;
|
|
81
|
+
let bestSim = similarityThreshold;
|
|
82
|
+
let bestI = -1;
|
|
83
|
+
let bestJ = -1;
|
|
84
|
+
for (let i = 0; i < clusters.length; i++) for (let j = i + 1; j < clusters.length; j++) {
|
|
85
|
+
const sim = cosineSimilarity(clusters[i].centroid, clusters[j].centroid);
|
|
86
|
+
if (sim > bestSim) {
|
|
87
|
+
bestSim = sim;
|
|
88
|
+
bestI = i;
|
|
89
|
+
bestJ = j;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
if (bestI === -1) break;
|
|
93
|
+
const ci = clusters[bestI];
|
|
94
|
+
const cj = clusters[bestJ];
|
|
95
|
+
const mergedPaths = [...ci.paths, ...cj.paths];
|
|
96
|
+
const mergedTitles = [...ci.titles, ...cj.titles];
|
|
97
|
+
const mergedIndexedAts = [...ci.indexedAts, ...cj.indexedAts];
|
|
98
|
+
const memberEmbeddings = [];
|
|
99
|
+
for (const p of mergedPaths) {
|
|
100
|
+
const emb = fileEmbeddings.get(p);
|
|
101
|
+
if (emb) memberEmbeddings.push(emb);
|
|
102
|
+
}
|
|
103
|
+
clusters[bestI] = {
|
|
104
|
+
paths: mergedPaths,
|
|
105
|
+
titles: mergedTitles,
|
|
106
|
+
indexedAts: mergedIndexedAts,
|
|
107
|
+
centroid: averageEmbeddings(memberEmbeddings)
|
|
108
|
+
};
|
|
109
|
+
clusters.splice(bestJ, 1);
|
|
110
|
+
merged = true;
|
|
111
|
+
}
|
|
112
|
+
const themes = [];
|
|
113
|
+
let clusterIndex = 0;
|
|
114
|
+
for (const cluster of clusters) {
|
|
115
|
+
if (cluster.paths.length < minClusterSize) continue;
|
|
116
|
+
const label = generateLabel(cluster.titles) || `Theme ${clusterIndex + 1}`;
|
|
117
|
+
const avgRecency = cluster.indexedAts.reduce((sum, t) => sum + t, 0) / cluster.indexedAts.length;
|
|
118
|
+
const folderDiversity = new Set(cluster.paths.map(getTopFolder)).size / cluster.paths.length;
|
|
119
|
+
const linkedRatio = await computeLinkedRatio(backend, cluster.paths);
|
|
120
|
+
const suggestIndexNote = linkedRatio < .3 && cluster.paths.length >= 5;
|
|
121
|
+
themes.push({
|
|
122
|
+
id: clusterIndex++,
|
|
123
|
+
label,
|
|
124
|
+
notes: cluster.paths.map((path, idx) => ({
|
|
125
|
+
path,
|
|
126
|
+
title: cluster.titles[idx]
|
|
127
|
+
})),
|
|
128
|
+
size: cluster.paths.length,
|
|
129
|
+
folderDiversity,
|
|
130
|
+
avgRecency,
|
|
131
|
+
linkedRatio,
|
|
132
|
+
suggestIndexNote
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
themes.sort((a, b) => b.size * b.folderDiversity * (b.avgRecency / now) - a.size * a.folderDiversity * (a.avgRecency / now));
|
|
136
|
+
return {
|
|
137
|
+
themes: themes.slice(0, maxThemes),
|
|
138
|
+
totalNotesAnalyzed,
|
|
139
|
+
timeWindow: {
|
|
140
|
+
from,
|
|
141
|
+
to: now
|
|
142
|
+
}
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
//#endregion
|
|
147
|
+
export { zettelThemes as t };
|
|
148
|
+
//# sourceMappingURL=themes-HzMKPTas.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"themes-HzMKPTas.mjs","names":[],"sources":["../src/zettelkasten/themes.ts"],"sourcesContent":["import type { StorageBackend } from \"../storage/interface.js\";\nimport { deserializeEmbedding, cosineSimilarity } from \"../memory/embeddings.js\";\nimport { STOP_WORDS } from \"../utils/stop-words.js\";\n\nexport interface ThemeOptions {\n vaultProjectId: number;\n lookbackDays?: number;\n minClusterSize?: number;\n maxThemes?: number;\n similarityThreshold?: number;\n}\n\nexport interface ThemeCluster {\n id: number;\n label: string;\n notes: Array<{\n path: string;\n title: string | null;\n }>;\n size: number;\n folderDiversity: number;\n avgRecency: number;\n linkedRatio: number;\n suggestIndexNote: boolean;\n}\n\nexport interface ThemeResult {\n themes: ThemeCluster[];\n totalNotesAnalyzed: number;\n timeWindow: { from: number; to: number };\n}\n\nconst MAX_CHUNKS = 5000;\n\n// STOP_WORDS imported from utils/stop-words.ts\n\nfunction getTopFolder(vaultPath: string): string {\n const parts = vaultPath.split(\"/\");\n return parts.length > 1 ? parts[0] : \"\";\n}\n\nfunction generateLabel(titles: Array<string | null>): string {\n const wordCounts = new Map<string, number>();\n for (const title of titles) {\n if (!title) continue;\n const words = title\n .toLowerCase()\n .replace(/[^a-z0-9\\s]/g, \" \")\n .split(/\\s+/)\n .filter((w) => w.length > 2 && !STOP_WORDS.has(w));\n for (const word of words) {\n wordCounts.set(word, (wordCounts.get(word) ?? 0) + 1);\n }\n }\n const sorted = [...wordCounts.entries()].sort((a, b) => b[1] - a[1]);\n return sorted\n .slice(0, 3)\n .map(([w]) => w)\n .join(\" / \");\n}\n\nasync function computeLinkedRatio(backend: StorageBackend, paths: string[]): Promise<number> {\n if (paths.length < 2) return 0;\n const totalPairs = (paths.length * (paths.length - 1)) / 2;\n const pathSet = new Set(paths);\n let linkedPairs = 0;\n\n for (const path of paths) {\n const links = await backend.getLinksFromSource(path);\n for (const link of links) {\n if (link.targetPath && pathSet.has(link.targetPath)) {\n linkedPairs++;\n }\n }\n }\n\n // Each bidirectional pair might be counted once per direction; divide by 2 to normalize\n const uniquePairs = linkedPairs / 2;\n return Math.min(1, uniquePairs / totalPairs);\n}\n\ntype ClusterNode = {\n paths: string[];\n titles: Array<string | null>;\n indexedAts: number[];\n centroid: Float32Array;\n};\n\nfunction averageEmbeddings(embeddings: Float32Array[]): Float32Array {\n if (embeddings.length === 0) return new Float32Array(0);\n const dim = embeddings[0].length;\n const sum = new Float32Array(dim);\n for (const vec of embeddings) {\n for (let i = 0; i < dim; i++) {\n sum[i] += vec[i];\n }\n }\n const avg = new Float32Array(dim);\n for (let i = 0; i < dim; i++) {\n avg[i] = sum[i] / embeddings.length;\n }\n return avg;\n}\n\n/**\n * Detect emerging themes in recently-modified notes using agglomerative single-linkage\n * clustering of note-level embeddings.\n */\nexport async function zettelThemes(\n backend: StorageBackend,\n opts: ThemeOptions,\n): Promise<ThemeResult> {\n const lookbackDays = opts.lookbackDays ?? 30;\n const minClusterSize = opts.minClusterSize ?? 3;\n const maxThemes = opts.maxThemes ?? 10;\n const similarityThreshold = opts.similarityThreshold ?? 0.65;\n\n const now = Date.now();\n const from = now - lookbackDays * 86400000;\n\n // Step 1: get recent notes\n const recentFiles = await backend.getRecentVaultFiles(from);\n const recentNotes = recentFiles.map(f => ({ vault_path: f.vaultPath, title: f.title, indexed_at: f.indexedAt }));\n\n // Step 2: get file-level embeddings from memory_chunks\n const chunkRows = await backend.getChunksWithEmbeddings(opts.vaultProjectId, MAX_CHUNKS);\n\n const embeddingsByPath = new Map<string, Float32Array[]>();\n for (const row of chunkRows) {\n const vec = deserializeEmbedding(row.embedding);\n const arr = embeddingsByPath.get(row.path);\n if (!arr) {\n embeddingsByPath.set(row.path, [vec]);\n } else {\n arr.push(vec);\n }\n }\n\n const fileEmbeddings = new Map<string, Float32Array>();\n for (const [path, vecs] of embeddingsByPath) {\n fileEmbeddings.set(path, averageEmbeddings(vecs));\n }\n\n // Step 3: build initial clusters — only include notes that have embeddings\n const clusters: ClusterNode[] = [];\n for (const note of recentNotes) {\n const embedding = fileEmbeddings.get(note.vault_path);\n if (!embedding) continue;\n clusters.push({\n paths: [note.vault_path],\n titles: [note.title],\n indexedAts: [note.indexed_at],\n centroid: embedding,\n });\n }\n\n const totalNotesAnalyzed = clusters.length;\n\n // Step 4: agglomerative single-linkage clustering\n // Stop when no two clusters have similarity >= threshold\n let merged = true;\n while (merged && clusters.length > 1) {\n merged = false;\n let bestSim = similarityThreshold;\n let bestI = -1;\n let bestJ = -1;\n\n for (let i = 0; i < clusters.length; i++) {\n for (let j = i + 1; j < clusters.length; j++) {\n const sim = cosineSimilarity(clusters[i].centroid, clusters[j].centroid);\n if (sim > bestSim) {\n bestSim = sim;\n bestI = i;\n bestJ = j;\n }\n }\n }\n\n if (bestI === -1) break;\n\n // Merge cluster j into cluster i\n const ci = clusters[bestI];\n const cj = clusters[bestJ];\n const mergedPaths = [...ci.paths, ...cj.paths];\n const mergedTitles = [...ci.titles, ...cj.titles];\n const mergedIndexedAts = [...ci.indexedAts, ...cj.indexedAts];\n\n // Recompute centroid from averaged embeddings of all member paths\n const memberEmbeddings: Float32Array[] = [];\n for (const p of mergedPaths) {\n const emb = fileEmbeddings.get(p);\n if (emb) memberEmbeddings.push(emb);\n }\n\n clusters[bestI] = {\n paths: mergedPaths,\n titles: mergedTitles,\n indexedAts: mergedIndexedAts,\n centroid: averageEmbeddings(memberEmbeddings),\n };\n\n clusters.splice(bestJ, 1);\n merged = true;\n }\n\n // Step 5: filter and annotate clusters\n const themes: ThemeCluster[] = [];\n let clusterIndex = 0;\n\n for (const cluster of clusters) {\n if (cluster.paths.length < minClusterSize) continue;\n\n const label = generateLabel(cluster.titles) || `Theme ${clusterIndex + 1}`;\n const avgRecency =\n cluster.indexedAts.reduce((sum, t) => sum + t, 0) / cluster.indexedAts.length;\n\n const uniqueFolders = new Set(cluster.paths.map(getTopFolder));\n const folderDiversity = uniqueFolders.size / cluster.paths.length;\n\n const linkedRatio = await computeLinkedRatio(backend, cluster.paths);\n const suggestIndexNote = linkedRatio < 0.3 && cluster.paths.length >= 5;\n\n themes.push({\n id: clusterIndex++,\n label,\n notes: cluster.paths.map((path, idx) => ({\n path,\n title: cluster.titles[idx],\n })),\n size: cluster.paths.length,\n folderDiversity,\n avgRecency,\n linkedRatio,\n suggestIndexNote,\n });\n }\n\n // Step 6: rank by size * folderDiversity * recency_ratio\n themes.sort(\n (a, b) =>\n b.size * b.folderDiversity * (b.avgRecency / now) -\n a.size * a.folderDiversity * (a.avgRecency / now),\n );\n\n return {\n themes: themes.slice(0, maxThemes),\n totalNotesAnalyzed,\n timeWindow: { from, to: now },\n };\n}\n"],"mappings":";;;;AAgCA,MAAM,aAAa;AAInB,SAAS,aAAa,WAA2B;CAC/C,MAAM,QAAQ,UAAU,MAAM,IAAI;AAClC,QAAO,MAAM,SAAS,IAAI,MAAM,KAAK;;AAGvC,SAAS,cAAc,QAAsC;CAC3D,MAAM,6BAAa,IAAI,KAAqB;AAC5C,MAAK,MAAM,SAAS,QAAQ;AAC1B,MAAI,CAAC,MAAO;EACZ,MAAM,QAAQ,MACX,aAAa,CACb,QAAQ,gBAAgB,IAAI,CAC5B,MAAM,MAAM,CACZ,QAAQ,MAAM,EAAE,SAAS,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC;AACpD,OAAK,MAAM,QAAQ,MACjB,YAAW,IAAI,OAAO,WAAW,IAAI,KAAK,IAAI,KAAK,EAAE;;AAIzD,QADe,CAAC,GAAG,WAAW,SAAS,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,KAAK,EAAE,GAAG,CAEjE,MAAM,GAAG,EAAE,CACX,KAAK,CAAC,OAAO,EAAE,CACf,KAAK,MAAM;;AAGhB,eAAe,mBAAmB,SAAyB,OAAkC;AAC3F,KAAI,MAAM,SAAS,EAAG,QAAO;CAC7B,MAAM,aAAc,MAAM,UAAU,MAAM,SAAS,KAAM;CACzD,MAAM,UAAU,IAAI,IAAI,MAAM;CAC9B,IAAI,cAAc;AAElB,MAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,QAAQ,MAAM,QAAQ,mBAAmB,KAAK;AACpD,OAAK,MAAM,QAAQ,MACjB,KAAI,KAAK,cAAc,QAAQ,IAAI,KAAK,WAAW,CACjD;;CAMN,MAAM,cAAc,cAAc;AAClC,QAAO,KAAK,IAAI,GAAG,cAAc,WAAW;;AAU9C,SAAS,kBAAkB,YAA0C;AACnE,KAAI,WAAW,WAAW,EAAG,QAAO,IAAI,aAAa,EAAE;CACvD,MAAM,MAAM,WAAW,GAAG;CAC1B,MAAM,MAAM,IAAI,aAAa,IAAI;AACjC,MAAK,MAAM,OAAO,WAChB,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,IACvB,KAAI,MAAM,IAAI;CAGlB,MAAM,MAAM,IAAI,aAAa,IAAI;AACjC,MAAK,IAAI,IAAI,GAAG,IAAI,KAAK,IACvB,KAAI,KAAK,IAAI,KAAK,WAAW;AAE/B,QAAO;;;;;;AAOT,eAAsB,aACpB,SACA,MACsB;CACtB,MAAM,eAAe,KAAK,gBAAgB;CAC1C,MAAM,iBAAiB,KAAK,kBAAkB;CAC9C,MAAM,YAAY,KAAK,aAAa;CACpC,MAAM,sBAAsB,KAAK,uBAAuB;CAExD,MAAM,MAAM,KAAK,KAAK;CACtB,MAAM,OAAO,MAAM,eAAe;CAIlC,MAAM,eADc,MAAM,QAAQ,oBAAoB,KAAK,EAC3B,KAAI,OAAM;EAAE,YAAY,EAAE;EAAW,OAAO,EAAE;EAAO,YAAY,EAAE;EAAW,EAAE;CAGhH,MAAM,YAAY,MAAM,QAAQ,wBAAwB,KAAK,gBAAgB,WAAW;CAExF,MAAM,mCAAmB,IAAI,KAA6B;AAC1D,MAAK,MAAM,OAAO,WAAW;EAC3B,MAAM,MAAM,qBAAqB,IAAI,UAAU;EAC/C,MAAM,MAAM,iBAAiB,IAAI,IAAI,KAAK;AAC1C,MAAI,CAAC,IACH,kBAAiB,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC;MAErC,KAAI,KAAK,IAAI;;CAIjB,MAAM,iCAAiB,IAAI,KAA2B;AACtD,MAAK,MAAM,CAAC,MAAM,SAAS,iBACzB,gBAAe,IAAI,MAAM,kBAAkB,KAAK,CAAC;CAInD,MAAM,WAA0B,EAAE;AAClC,MAAK,MAAM,QAAQ,aAAa;EAC9B,MAAM,YAAY,eAAe,IAAI,KAAK,WAAW;AACrD,MAAI,CAAC,UAAW;AAChB,WAAS,KAAK;GACZ,OAAO,CAAC,KAAK,WAAW;GACxB,QAAQ,CAAC,KAAK,MAAM;GACpB,YAAY,CAAC,KAAK,WAAW;GAC7B,UAAU;GACX,CAAC;;CAGJ,MAAM,qBAAqB,SAAS;CAIpC,IAAI,SAAS;AACb,QAAO,UAAU,SAAS,SAAS,GAAG;AACpC,WAAS;EACT,IAAI,UAAU;EACd,IAAI,QAAQ;EACZ,IAAI,QAAQ;AAEZ,OAAK,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,IACnC,MAAK,IAAI,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;GAC5C,MAAM,MAAM,iBAAiB,SAAS,GAAG,UAAU,SAAS,GAAG,SAAS;AACxE,OAAI,MAAM,SAAS;AACjB,cAAU;AACV,YAAQ;AACR,YAAQ;;;AAKd,MAAI,UAAU,GAAI;EAGlB,MAAM,KAAK,SAAS;EACpB,MAAM,KAAK,SAAS;EACpB,MAAM,cAAc,CAAC,GAAG,GAAG,OAAO,GAAG,GAAG,MAAM;EAC9C,MAAM,eAAe,CAAC,GAAG,GAAG,QAAQ,GAAG,GAAG,OAAO;EACjD,MAAM,mBAAmB,CAAC,GAAG,GAAG,YAAY,GAAG,GAAG,WAAW;EAG7D,MAAM,mBAAmC,EAAE;AAC3C,OAAK,MAAM,KAAK,aAAa;GAC3B,MAAM,MAAM,eAAe,IAAI,EAAE;AACjC,OAAI,IAAK,kBAAiB,KAAK,IAAI;;AAGrC,WAAS,SAAS;GAChB,OAAO;GACP,QAAQ;GACR,YAAY;GACZ,UAAU,kBAAkB,iBAAiB;GAC9C;AAED,WAAS,OAAO,OAAO,EAAE;AACzB,WAAS;;CAIX,MAAM,SAAyB,EAAE;CACjC,IAAI,eAAe;AAEnB,MAAK,MAAM,WAAW,UAAU;AAC9B,MAAI,QAAQ,MAAM,SAAS,eAAgB;EAE3C,MAAM,QAAQ,cAAc,QAAQ,OAAO,IAAI,SAAS,eAAe;EACvE,MAAM,aACJ,QAAQ,WAAW,QAAQ,KAAK,MAAM,MAAM,GAAG,EAAE,GAAG,QAAQ,WAAW;EAGzE,MAAM,kBADgB,IAAI,IAAI,QAAQ,MAAM,IAAI,aAAa,CAAC,CACxB,OAAO,QAAQ,MAAM;EAE3D,MAAM,cAAc,MAAM,mBAAmB,SAAS,QAAQ,MAAM;EACpE,MAAM,mBAAmB,cAAc,MAAO,QAAQ,MAAM,UAAU;AAEtE,SAAO,KAAK;GACV,IAAI;GACJ;GACA,OAAO,QAAQ,MAAM,KAAK,MAAM,SAAS;IACvC;IACA,OAAO,QAAQ,OAAO;IACvB,EAAE;GACH,MAAM,QAAQ,MAAM;GACpB;GACA;GACA;GACA;GACD,CAAC;;AAIJ,QAAO,MACJ,GAAG,MACF,EAAE,OAAO,EAAE,mBAAmB,EAAE,aAAa,OAC7C,EAAE,OAAO,EAAE,mBAAmB,EAAE,aAAa,KAChD;AAED,QAAO;EACL,QAAQ,OAAO,MAAM,GAAG,UAAU;EAClC;EACA,YAAY;GAAE;GAAM,IAAI;GAAK;EAC9B"}
|