@velvetmonkey/flywheel-memory 2.0.155 → 2.0.157
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +279 -61
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -575,7 +575,7 @@ function getStoredEmbeddingModel() {
|
|
|
575
575
|
function diagnoseEmbeddings(vaultPath2) {
|
|
576
576
|
const db4 = getDb();
|
|
577
577
|
const checks = [];
|
|
578
|
-
const counts = { embedded: 0, vaultNotes: 0, orphaned: 0, missing: 0 };
|
|
578
|
+
const counts = { embedded: 0, vaultNotes: 0, orphaned: 0, orphanedEntities: 0, missing: 0 };
|
|
579
579
|
if (!db4) {
|
|
580
580
|
checks.push({ name: "database", status: "stale", detail: "No database available" });
|
|
581
581
|
return { healthy: false, checks, counts };
|
|
@@ -649,6 +649,25 @@ function diagnoseEmbeddings(vaultPath2) {
|
|
|
649
649
|
} catch {
|
|
650
650
|
checks.push({ name: "orphans", status: "warning", detail: "Could not check" });
|
|
651
651
|
}
|
|
652
|
+
try {
|
|
653
|
+
const embNames = new Set(
|
|
654
|
+
db4.prepare("SELECT entity_name FROM entity_embeddings").all().map((r) => r.entity_name)
|
|
655
|
+
);
|
|
656
|
+
const entityNames = new Set(
|
|
657
|
+
db4.prepare("SELECT name FROM entities").all().map((r) => r.name)
|
|
658
|
+
);
|
|
659
|
+
counts.orphanedEntities = 0;
|
|
660
|
+
for (const n of embNames) {
|
|
661
|
+
if (!entityNames.has(n)) counts.orphanedEntities++;
|
|
662
|
+
}
|
|
663
|
+
if (counts.orphanedEntities > 0) {
|
|
664
|
+
checks.push({ name: "entity_orphans", status: "warning", detail: `${counts.orphanedEntities} orphaned entity embeddings` });
|
|
665
|
+
} else {
|
|
666
|
+
checks.push({ name: "entity_orphans", status: "ok", detail: "0 orphaned" });
|
|
667
|
+
}
|
|
668
|
+
} catch {
|
|
669
|
+
checks.push({ name: "entity_orphans", status: "ok", detail: "No entity embeddings table" });
|
|
670
|
+
}
|
|
652
671
|
try {
|
|
653
672
|
const samples = db4.prepare("SELECT embedding FROM note_embeddings ORDER BY RANDOM() LIMIT 3").all();
|
|
654
673
|
let corrupt = false;
|
|
@@ -732,6 +751,7 @@ async function buildEntityEmbeddingsIndex(vaultPath2, entities, onProgress) {
|
|
|
732
751
|
const total = entities.size;
|
|
733
752
|
let done = 0;
|
|
734
753
|
let updated = 0;
|
|
754
|
+
let skipped = 0;
|
|
735
755
|
for (const [name, entity] of entities) {
|
|
736
756
|
done++;
|
|
737
757
|
try {
|
|
@@ -745,7 +765,11 @@ async function buildEntityEmbeddingsIndex(vaultPath2, entities, onProgress) {
|
|
|
745
765
|
const buf = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
746
766
|
upsert.run(name, buf, hash, activeModelConfig.id, Date.now());
|
|
747
767
|
updated++;
|
|
748
|
-
} catch {
|
|
768
|
+
} catch (err) {
|
|
769
|
+
skipped++;
|
|
770
|
+
if (skipped <= 3) {
|
|
771
|
+
console.error(`[Semantic] Failed to embed entity ${name}: ${err instanceof Error ? err.message : err}`);
|
|
772
|
+
}
|
|
749
773
|
}
|
|
750
774
|
if (onProgress) onProgress(done, total);
|
|
751
775
|
}
|
|
@@ -755,7 +779,7 @@ async function buildEntityEmbeddingsIndex(vaultPath2, entities, onProgress) {
|
|
|
755
779
|
deleteStmt.run(existingName);
|
|
756
780
|
}
|
|
757
781
|
}
|
|
758
|
-
console.error(`[Semantic] Entity embeddings: ${updated} updated, ${total - updated} unchanged`);
|
|
782
|
+
console.error(`[Semantic] Entity embeddings: ${updated} updated, ${total - updated - skipped} unchanged, ${skipped} failed`);
|
|
759
783
|
return updated;
|
|
760
784
|
}
|
|
761
785
|
async function updateEntityEmbedding(entityName, entity, vaultPath2) {
|
|
@@ -773,9 +797,34 @@ async function updateEntityEmbedding(entityName, entity, vaultPath2) {
|
|
|
773
797
|
VALUES (?, ?, ?, ?, ?)
|
|
774
798
|
`).run(entityName, buf, hash, activeModelConfig.id, Date.now());
|
|
775
799
|
entityEmbeddingsMap.set(entityName, embedding);
|
|
776
|
-
} catch {
|
|
800
|
+
} catch (err) {
|
|
801
|
+
console.error(`[Semantic] Failed to update entity embedding ${entityName}: ${err instanceof Error ? err.message : err}`);
|
|
777
802
|
}
|
|
778
803
|
}
|
|
804
|
+
function removeOrphanedNoteEmbeddings() {
|
|
805
|
+
const db4 = getDb();
|
|
806
|
+
if (!db4) return 0;
|
|
807
|
+
const result = db4.prepare(
|
|
808
|
+
"DELETE FROM note_embeddings WHERE path NOT IN (SELECT path FROM notes_fts)"
|
|
809
|
+
).run();
|
|
810
|
+
return result.changes;
|
|
811
|
+
}
|
|
812
|
+
function removeOrphanedEntityEmbeddings(currentEntityNames) {
|
|
813
|
+
const db4 = getDb();
|
|
814
|
+
if (!db4) return 0;
|
|
815
|
+
const rows = db4.prepare("SELECT entity_name FROM entity_embeddings").all();
|
|
816
|
+
const deleteStmt = db4.prepare("DELETE FROM entity_embeddings WHERE entity_name = ?");
|
|
817
|
+
const embMap = getEmbMap();
|
|
818
|
+
let removed = 0;
|
|
819
|
+
for (const row of rows) {
|
|
820
|
+
if (!currentEntityNames.has(row.entity_name)) {
|
|
821
|
+
deleteStmt.run(row.entity_name);
|
|
822
|
+
embMap.delete(row.entity_name);
|
|
823
|
+
removed++;
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
return removed;
|
|
827
|
+
}
|
|
779
828
|
function findSemanticallySimilarEntities(queryEmbedding, limit, excludeEntities) {
|
|
780
829
|
const scored = [];
|
|
781
830
|
for (const [entityName, embedding] of getEmbMap()) {
|
|
@@ -6516,6 +6565,7 @@ import {
|
|
|
6516
6565
|
loadFlywheelConfigFromDb,
|
|
6517
6566
|
saveFlywheelConfigToDb
|
|
6518
6567
|
} from "@velvetmonkey/vault-core";
|
|
6568
|
+
var DEFAULT_ENTITY_EXCLUDE_FOLDERS = ["node_modules", "templates", "attachments", "tmp"];
|
|
6519
6569
|
var DEFAULT_CONFIG = {
|
|
6520
6570
|
exclude: [],
|
|
6521
6571
|
exclude_entity_folders: [],
|
|
@@ -7238,6 +7288,7 @@ function addNoteToIndex(index, note) {
|
|
|
7238
7288
|
async function upsertNote(index, vaultPath2, notePath) {
|
|
7239
7289
|
try {
|
|
7240
7290
|
const existed = index.notes.has(notePath);
|
|
7291
|
+
const oldNote = existed ? index.notes.get(notePath) : void 0;
|
|
7241
7292
|
let releasedKeys = [];
|
|
7242
7293
|
if (existed) {
|
|
7243
7294
|
releasedKeys = removeNoteFromIndex(index, notePath);
|
|
@@ -7255,10 +7306,12 @@ async function upsertNote(index, vaultPath2, notePath) {
|
|
|
7255
7306
|
if (releasedKeys.length > 0) {
|
|
7256
7307
|
reconcileReleasedKeys(index, releasedKeys);
|
|
7257
7308
|
}
|
|
7309
|
+
const entityFieldChanged = !oldNote || (oldNote.title !== note.title || JSON.stringify(oldNote.aliases) !== JSON.stringify(note.aliases) || oldNote.frontmatter.type !== note.frontmatter.type);
|
|
7258
7310
|
return {
|
|
7259
7311
|
success: true,
|
|
7260
7312
|
action: existed ? "updated" : "added",
|
|
7261
|
-
path: notePath
|
|
7313
|
+
path: notePath,
|
|
7314
|
+
entityFieldChanged
|
|
7262
7315
|
};
|
|
7263
7316
|
} catch (error) {
|
|
7264
7317
|
return {
|
|
@@ -7300,7 +7353,8 @@ async function processBatch(index, vaultPath2, batch, options = {}) {
|
|
|
7300
7353
|
successful: 0,
|
|
7301
7354
|
failed: 0,
|
|
7302
7355
|
results: [],
|
|
7303
|
-
durationMs: 0
|
|
7356
|
+
durationMs: 0,
|
|
7357
|
+
hasEntityRelevantChanges: false
|
|
7304
7358
|
};
|
|
7305
7359
|
}
|
|
7306
7360
|
console.error(`[flywheel] Processing ${total} file events`);
|
|
@@ -7355,7 +7409,8 @@ async function processBatch(index, vaultPath2, batch, options = {}) {
|
|
|
7355
7409
|
successful,
|
|
7356
7410
|
failed,
|
|
7357
7411
|
results,
|
|
7358
|
-
durationMs
|
|
7412
|
+
durationMs,
|
|
7413
|
+
hasEntityRelevantChanges: results.some((r) => r.entityFieldChanged)
|
|
7359
7414
|
};
|
|
7360
7415
|
}
|
|
7361
7416
|
|
|
@@ -8427,6 +8482,7 @@ var PipelineRunner = class {
|
|
|
8427
8482
|
entitiesAfter = [];
|
|
8428
8483
|
entitiesBefore = [];
|
|
8429
8484
|
hubBefore = /* @__PURE__ */ new Map();
|
|
8485
|
+
hasEntityRelevantChanges = false;
|
|
8430
8486
|
forwardLinkResults = [];
|
|
8431
8487
|
linkDiffs = [];
|
|
8432
8488
|
survivedLinks = [];
|
|
@@ -8498,6 +8554,7 @@ var PipelineRunner = class {
|
|
|
8498
8554
|
if (!vaultIndex2) {
|
|
8499
8555
|
const rebuilt = await p.buildVaultIndex(p.vp);
|
|
8500
8556
|
p.updateVaultIndex(rebuilt);
|
|
8557
|
+
this.hasEntityRelevantChanges = true;
|
|
8501
8558
|
serverLog("watcher", `Index rebuilt (full): ${rebuilt.notes.size} notes, ${rebuilt.entities.size} entities`);
|
|
8502
8559
|
} else {
|
|
8503
8560
|
const absoluteBatch = {
|
|
@@ -8505,6 +8562,7 @@ var PipelineRunner = class {
|
|
|
8505
8562
|
events: p.events.map((e) => ({ ...e, path: path15.join(p.vp, e.path) }))
|
|
8506
8563
|
};
|
|
8507
8564
|
const batchResult = await processBatch(vaultIndex2, p.vp, absoluteBatch);
|
|
8565
|
+
this.hasEntityRelevantChanges = batchResult.hasEntityRelevantChanges;
|
|
8508
8566
|
serverLog("watcher", `Incremental: ${batchResult.successful}/${batchResult.total} files in ${batchResult.durationMs}ms`);
|
|
8509
8567
|
}
|
|
8510
8568
|
p.updateIndexState("ready");
|
|
@@ -8551,7 +8609,7 @@ var PipelineRunner = class {
|
|
|
8551
8609
|
for (const r of rows) this.hubBefore.set(r.name, r.hub_score);
|
|
8552
8610
|
}
|
|
8553
8611
|
const entityScanAgeMs = p.ctx.lastEntityScanAt > 0 ? Date.now() - p.ctx.lastEntityScanAt : Infinity;
|
|
8554
|
-
if (entityScanAgeMs < 5 * 60 * 1e3) {
|
|
8612
|
+
if (entityScanAgeMs < 5 * 60 * 1e3 && !this.hasEntityRelevantChanges) {
|
|
8555
8613
|
tracker.start("entity_scan", {});
|
|
8556
8614
|
tracker.skip("entity_scan", `cache valid (${Math.round(entityScanAgeMs / 1e3)}s old)`);
|
|
8557
8615
|
this.entitiesBefore = p.sd ? getAllEntitiesFromDb(p.sd) : [];
|
|
@@ -8694,8 +8752,9 @@ var PipelineRunner = class {
|
|
|
8694
8752
|
} catch {
|
|
8695
8753
|
}
|
|
8696
8754
|
}
|
|
8697
|
-
|
|
8698
|
-
|
|
8755
|
+
const orphansRemoved = removeOrphanedNoteEmbeddings();
|
|
8756
|
+
tracker.end({ updated: embUpdated, removed: embRemoved, orphans_removed: orphansRemoved });
|
|
8757
|
+
serverLog("watcher", `Note embeddings: ${embUpdated} updated, ${embRemoved} removed, ${orphansRemoved} orphans cleaned`);
|
|
8699
8758
|
} else {
|
|
8700
8759
|
tracker.skip("note_embeddings", "not built");
|
|
8701
8760
|
}
|
|
@@ -8706,6 +8765,7 @@ var PipelineRunner = class {
|
|
|
8706
8765
|
if (hasEntityEmbeddingsIndex() && p.sd) {
|
|
8707
8766
|
tracker.start("entity_embeddings", { files: p.events.length });
|
|
8708
8767
|
let entEmbUpdated = 0;
|
|
8768
|
+
let entEmbOrphansRemoved = 0;
|
|
8709
8769
|
const entEmbNames = [];
|
|
8710
8770
|
try {
|
|
8711
8771
|
const allEntities = getAllEntitiesFromDb(p.sd);
|
|
@@ -8723,10 +8783,12 @@ var PipelineRunner = class {
|
|
|
8723
8783
|
entEmbNames.push(entity.name);
|
|
8724
8784
|
}
|
|
8725
8785
|
}
|
|
8786
|
+
const currentNames = new Set(allEntities.map((e) => e.name));
|
|
8787
|
+
entEmbOrphansRemoved = removeOrphanedEntityEmbeddings(currentNames);
|
|
8726
8788
|
} catch {
|
|
8727
8789
|
}
|
|
8728
|
-
tracker.end({ updated: entEmbUpdated, updated_entities: entEmbNames.slice(0, 10) });
|
|
8729
|
-
serverLog("watcher", `Entity embeddings: ${entEmbUpdated} updated`);
|
|
8790
|
+
tracker.end({ updated: entEmbUpdated, updated_entities: entEmbNames.slice(0, 10), orphans_removed: entEmbOrphansRemoved });
|
|
8791
|
+
serverLog("watcher", `Entity embeddings: ${entEmbUpdated} updated, ${entEmbOrphansRemoved} orphans cleaned`);
|
|
8730
8792
|
} else {
|
|
8731
8793
|
tracker.skip("entity_embeddings", !p.sd ? "no sd" : "not built");
|
|
8732
8794
|
}
|
|
@@ -9274,9 +9336,10 @@ var PipelineRunner = class {
|
|
|
9274
9336
|
return { skipped: true, reason: "vacuumed recently" };
|
|
9275
9337
|
}
|
|
9276
9338
|
p.sd.db.pragma("incremental_vacuum");
|
|
9339
|
+
p.sd.db.pragma("wal_checkpoint(TRUNCATE)");
|
|
9277
9340
|
p.sd.setMetadataValue.run("last_incremental_vacuum", String(Date.now()));
|
|
9278
|
-
serverLog("watcher", "Incremental vacuum completed");
|
|
9279
|
-
return { vacuumed: true };
|
|
9341
|
+
serverLog("watcher", "Incremental vacuum + WAL checkpoint completed");
|
|
9342
|
+
return { vacuumed: true, wal_checkpointed: true };
|
|
9280
9343
|
}
|
|
9281
9344
|
};
|
|
9282
9345
|
|
|
@@ -9329,7 +9392,7 @@ async function flushLogs() {
|
|
|
9329
9392
|
|
|
9330
9393
|
// src/index.ts
|
|
9331
9394
|
init_embeddings();
|
|
9332
|
-
import { openStateDb, scanVaultEntities as scanVaultEntities4, getAllEntitiesFromDb as
|
|
9395
|
+
import { openStateDb, scanVaultEntities as scanVaultEntities4, getAllEntitiesFromDb as getAllEntitiesFromDb6, loadContentHashes, saveContentHashBatch, renameContentHash, checkDbIntegrity as checkDbIntegrity2, safeBackupAsync as safeBackupAsync2, preserveCorruptedDb, deleteStateDbFiles, attemptSalvage } from "@velvetmonkey/vault-core";
|
|
9333
9396
|
|
|
9334
9397
|
// src/core/write/memory.ts
|
|
9335
9398
|
init_wikilinkFeedback();
|
|
@@ -10723,6 +10786,20 @@ import { z } from "zod";
|
|
|
10723
10786
|
// src/core/read/constants.ts
|
|
10724
10787
|
var MAX_LIMIT = 200;
|
|
10725
10788
|
|
|
10789
|
+
// src/core/read/indexGuard.ts
|
|
10790
|
+
function requireIndex() {
|
|
10791
|
+
const state2 = getIndexState();
|
|
10792
|
+
if (state2 === "building") {
|
|
10793
|
+
const { parsed, total } = getIndexProgress();
|
|
10794
|
+
const progress = total > 0 ? ` (${parsed}/${total} files)` : "";
|
|
10795
|
+
throw new Error(`Index building${progress}... try again shortly`);
|
|
10796
|
+
}
|
|
10797
|
+
if (state2 === "error") {
|
|
10798
|
+
const error = getIndexError();
|
|
10799
|
+
throw new Error(`Index failed to build: ${error?.message || "unknown error"}`);
|
|
10800
|
+
}
|
|
10801
|
+
}
|
|
10802
|
+
|
|
10726
10803
|
// src/tools/read/query.ts
|
|
10727
10804
|
init_embeddings();
|
|
10728
10805
|
import {
|
|
@@ -11747,6 +11824,7 @@ function registerQueryTools(server2, getIndex, getVaultPath, getStateDb3) {
|
|
|
11747
11824
|
consumer: z.enum(["llm", "human"]).default("llm").describe('Output format: "llm" applies sandwich ordering and strips scoring fields for context efficiency. "human" preserves score order and all scoring metadata for UI display.')
|
|
11748
11825
|
},
|
|
11749
11826
|
async ({ query, where, has_tag, has_any_tag, has_all_tags, include_children, folder, title_contains, modified_after, modified_before, sort_by, order, prefix, limit: requestedLimit, detail_count: requestedDetailCount, context_note, consumer }) => {
|
|
11827
|
+
requireIndex();
|
|
11750
11828
|
const limit = Math.min(requestedLimit ?? 10, MAX_LIMIT);
|
|
11751
11829
|
const detailN = requestedDetailCount ?? 5;
|
|
11752
11830
|
const index = getIndex();
|
|
@@ -12466,20 +12544,6 @@ function detectCycles(index, maxLength = 10, limit = 20) {
|
|
|
12466
12544
|
return cycles;
|
|
12467
12545
|
}
|
|
12468
12546
|
|
|
12469
|
-
// src/core/read/indexGuard.ts
|
|
12470
|
-
function requireIndex() {
|
|
12471
|
-
const state2 = getIndexState();
|
|
12472
|
-
if (state2 === "building") {
|
|
12473
|
-
const { parsed, total } = getIndexProgress();
|
|
12474
|
-
const progress = total > 0 ? ` (${parsed}/${total} files)` : "";
|
|
12475
|
-
throw new Error(`Index building${progress}... try again shortly`);
|
|
12476
|
-
}
|
|
12477
|
-
if (state2 === "error") {
|
|
12478
|
-
const error = getIndexError();
|
|
12479
|
-
throw new Error(`Index failed to build: ${error?.message || "unknown error"}`);
|
|
12480
|
-
}
|
|
12481
|
-
}
|
|
12482
|
-
|
|
12483
12547
|
// src/tools/read/graph.ts
|
|
12484
12548
|
async function getContext(vaultPath2, sourcePath, line, contextLines = 1) {
|
|
12485
12549
|
try {
|
|
@@ -14656,7 +14720,7 @@ function registerHealthTools(server2, getIndex, getVaultPath, getConfig2 = () =>
|
|
|
14656
14720
|
import * as fs15 from "fs";
|
|
14657
14721
|
import * as path18 from "path";
|
|
14658
14722
|
import { z as z6 } from "zod";
|
|
14659
|
-
import { scanVaultEntities as scanVaultEntities2, getEntityIndexFromDb as getEntityIndexFromDb2 } from "@velvetmonkey/vault-core";
|
|
14723
|
+
import { scanVaultEntities as scanVaultEntities2, getEntityIndexFromDb as getEntityIndexFromDb2, getAllEntitiesFromDb as getAllEntitiesFromDb3 } from "@velvetmonkey/vault-core";
|
|
14660
14724
|
|
|
14661
14725
|
// src/core/read/aliasSuggestions.ts
|
|
14662
14726
|
import { STOPWORDS_EN as STOPWORDS_EN3 } from "@velvetmonkey/vault-core";
|
|
@@ -14726,6 +14790,10 @@ function suggestEntityAliases(stateDb2, folder) {
|
|
|
14726
14790
|
// src/tools/read/system.ts
|
|
14727
14791
|
init_edgeWeights();
|
|
14728
14792
|
init_embeddings();
|
|
14793
|
+
init_wikilinks();
|
|
14794
|
+
init_wikilinkFeedback();
|
|
14795
|
+
init_recency();
|
|
14796
|
+
init_cooccurrence();
|
|
14729
14797
|
function registerSystemTools(server2, getIndex, setIndex, getVaultPath, setConfig, getStateDb3) {
|
|
14730
14798
|
const RefreshIndexOutputSchema = {
|
|
14731
14799
|
success: z6.boolean().describe("Whether the refresh succeeded"),
|
|
@@ -14733,7 +14801,15 @@ function registerSystemTools(server2, getIndex, setIndex, getVaultPath, setConfi
|
|
|
14733
14801
|
entities_count: z6.number().describe("Number of entities (titles + aliases)"),
|
|
14734
14802
|
fts5_notes: z6.number().describe("Number of notes in FTS5 search index"),
|
|
14735
14803
|
edges_recomputed: z6.number().optional().describe("Number of edges with recomputed weights"),
|
|
14804
|
+
hub_scores: z6.number().optional().describe("Number of hub scores exported"),
|
|
14805
|
+
graph_snapshot: z6.boolean().optional().describe("Whether graph topology snapshot was recorded"),
|
|
14806
|
+
suppression_list: z6.boolean().optional().describe("Whether wikilink suppression list was updated"),
|
|
14807
|
+
task_cache: z6.boolean().optional().describe("Whether task cache was refreshed"),
|
|
14736
14808
|
embeddings_refreshed: z6.number().optional().describe("Number of note embeddings updated"),
|
|
14809
|
+
entity_embeddings_refreshed: z6.number().optional().describe("Number of entity embeddings updated"),
|
|
14810
|
+
recency_rebuilt: z6.boolean().optional().describe("Whether recency index was rebuilt"),
|
|
14811
|
+
cooccurrence_associations: z6.number().optional().describe("Number of co-occurrence associations rebuilt"),
|
|
14812
|
+
index_cached: z6.boolean().optional().describe("Whether vault index cache was saved"),
|
|
14737
14813
|
duration_ms: z6.number().describe("Time taken to rebuild index")
|
|
14738
14814
|
};
|
|
14739
14815
|
server2.registerTool(
|
|
@@ -14747,90 +14823,224 @@ function registerSystemTools(server2, getIndex, setIndex, getVaultPath, setConfi
|
|
|
14747
14823
|
async () => {
|
|
14748
14824
|
const vaultPath2 = getVaultPath();
|
|
14749
14825
|
const startTime = Date.now();
|
|
14826
|
+
const tracker = createStepTracker();
|
|
14750
14827
|
setIndexState("building");
|
|
14751
14828
|
setIndexError(null);
|
|
14752
14829
|
try {
|
|
14830
|
+
tracker.start("vault_index", {});
|
|
14753
14831
|
const newIndex = await buildVaultIndex(vaultPath2);
|
|
14754
14832
|
setIndex(newIndex);
|
|
14755
14833
|
setIndexState("ready");
|
|
14834
|
+
tracker.end({ notes: newIndex.notes.size, entities: newIndex.entities.size });
|
|
14756
14835
|
const stateDb2 = getStateDb3?.();
|
|
14757
14836
|
if (stateDb2) {
|
|
14837
|
+
tracker.start("entity_sync", {});
|
|
14758
14838
|
try {
|
|
14759
14839
|
const config = loadConfig(stateDb2);
|
|
14840
|
+
const excludeFolders = config.exclude_entity_folders?.length ? config.exclude_entity_folders : DEFAULT_ENTITY_EXCLUDE_FOLDERS;
|
|
14760
14841
|
const entityIndex2 = await scanVaultEntities2(vaultPath2, {
|
|
14761
|
-
excludeFolders
|
|
14762
|
-
"daily-notes",
|
|
14763
|
-
"daily",
|
|
14764
|
-
"weekly",
|
|
14765
|
-
"weekly-notes",
|
|
14766
|
-
"monthly",
|
|
14767
|
-
"monthly-notes",
|
|
14768
|
-
"quarterly",
|
|
14769
|
-
"yearly-notes",
|
|
14770
|
-
"periodic",
|
|
14771
|
-
"journal",
|
|
14772
|
-
"inbox",
|
|
14773
|
-
"templates",
|
|
14774
|
-
"attachments",
|
|
14775
|
-
"tmp",
|
|
14776
|
-
"clippings",
|
|
14777
|
-
"readwise",
|
|
14778
|
-
"articles",
|
|
14779
|
-
"bookmarks",
|
|
14780
|
-
"web-clips"
|
|
14781
|
-
],
|
|
14842
|
+
excludeFolders,
|
|
14782
14843
|
customCategories: config.custom_categories
|
|
14783
14844
|
});
|
|
14784
14845
|
stateDb2.replaceAllEntities(entityIndex2);
|
|
14846
|
+
tracker.end({ entities: entityIndex2._metadata.total_entities });
|
|
14785
14847
|
console.error(`[Flywheel] Updated ${entityIndex2._metadata.total_entities} entities in StateDb`);
|
|
14786
14848
|
} catch (e) {
|
|
14849
|
+
tracker.end({ error: String(e) });
|
|
14787
14850
|
console.error("[Flywheel] Failed to update entities:", e);
|
|
14788
14851
|
}
|
|
14789
14852
|
}
|
|
14853
|
+
let flywheelConfig2;
|
|
14790
14854
|
if (setConfig) {
|
|
14855
|
+
tracker.start("config_merge", {});
|
|
14791
14856
|
const existing = loadConfig(stateDb2);
|
|
14792
14857
|
const inferred = inferConfig(newIndex, vaultPath2);
|
|
14793
14858
|
if (stateDb2) {
|
|
14794
14859
|
saveConfig(stateDb2, inferred, existing);
|
|
14795
14860
|
}
|
|
14796
|
-
|
|
14861
|
+
flywheelConfig2 = loadConfig(stateDb2);
|
|
14862
|
+
setConfig(flywheelConfig2);
|
|
14863
|
+
tracker.end({});
|
|
14797
14864
|
}
|
|
14798
14865
|
let fts5Notes = 0;
|
|
14866
|
+
tracker.start("fts5_rebuild", {});
|
|
14799
14867
|
try {
|
|
14800
14868
|
const ftsState = await buildFTS5Index(vaultPath2);
|
|
14801
14869
|
fts5Notes = ftsState.noteCount;
|
|
14870
|
+
tracker.end({ notes: fts5Notes });
|
|
14802
14871
|
console.error(`[Flywheel] FTS5 index rebuilt: ${fts5Notes} notes`);
|
|
14803
14872
|
} catch (err) {
|
|
14873
|
+
tracker.end({ error: String(err) });
|
|
14804
14874
|
console.error("[Flywheel] FTS5 rebuild failed:", err);
|
|
14805
14875
|
}
|
|
14806
14876
|
let edgesRecomputed = 0;
|
|
14807
14877
|
if (stateDb2) {
|
|
14878
|
+
tracker.start("edge_weights", {});
|
|
14808
14879
|
try {
|
|
14809
14880
|
const edgeResult = recomputeEdgeWeights(stateDb2);
|
|
14810
14881
|
edgesRecomputed = edgeResult.edges_updated;
|
|
14882
|
+
tracker.end({ edges: edgeResult.edges_updated, duration_ms: edgeResult.duration_ms });
|
|
14811
14883
|
console.error(`[Flywheel] Edge weights: ${edgeResult.edges_updated} edges in ${edgeResult.duration_ms}ms`);
|
|
14812
14884
|
} catch (err) {
|
|
14885
|
+
tracker.end({ error: String(err) });
|
|
14813
14886
|
console.error("[Flywheel] Edge weight recompute failed:", err);
|
|
14814
14887
|
}
|
|
14815
14888
|
}
|
|
14889
|
+
tracker.start("entity_index_init", {});
|
|
14890
|
+
try {
|
|
14891
|
+
await initializeEntityIndex(vaultPath2);
|
|
14892
|
+
tracker.end({});
|
|
14893
|
+
console.error("[Flywheel] Entity index initialized");
|
|
14894
|
+
} catch (err) {
|
|
14895
|
+
tracker.end({ error: String(err) });
|
|
14896
|
+
console.error("[Flywheel] Entity index init failed:", err);
|
|
14897
|
+
}
|
|
14898
|
+
let hubScoresExported = 0;
|
|
14899
|
+
tracker.start("hub_scores", {});
|
|
14900
|
+
try {
|
|
14901
|
+
hubScoresExported = await exportHubScores(newIndex, stateDb2);
|
|
14902
|
+
tracker.end({ exported: hubScoresExported });
|
|
14903
|
+
if (hubScoresExported > 0) {
|
|
14904
|
+
console.error(`[Flywheel] Hub scores: ${hubScoresExported} entities`);
|
|
14905
|
+
}
|
|
14906
|
+
} catch (err) {
|
|
14907
|
+
tracker.end({ error: String(err) });
|
|
14908
|
+
console.error("[Flywheel] Hub score export failed:", err);
|
|
14909
|
+
}
|
|
14910
|
+
let graphSnapshotRecorded = false;
|
|
14911
|
+
if (stateDb2) {
|
|
14912
|
+
tracker.start("graph_snapshot", {});
|
|
14913
|
+
try {
|
|
14914
|
+
const graphMetrics = computeGraphMetrics(newIndex);
|
|
14915
|
+
recordGraphSnapshot(stateDb2, graphMetrics);
|
|
14916
|
+
graphSnapshotRecorded = true;
|
|
14917
|
+
tracker.end({ recorded: true });
|
|
14918
|
+
} catch (err) {
|
|
14919
|
+
tracker.end({ error: String(err) });
|
|
14920
|
+
console.error("[Flywheel] Graph snapshot failed:", err);
|
|
14921
|
+
}
|
|
14922
|
+
}
|
|
14923
|
+
let suppressionUpdated = false;
|
|
14924
|
+
if (stateDb2) {
|
|
14925
|
+
tracker.start("suppression_list", {});
|
|
14926
|
+
try {
|
|
14927
|
+
updateSuppressionList(stateDb2);
|
|
14928
|
+
suppressionUpdated = true;
|
|
14929
|
+
tracker.end({ updated: true });
|
|
14930
|
+
} catch (err) {
|
|
14931
|
+
tracker.end({ error: String(err) });
|
|
14932
|
+
console.error("[Flywheel] Suppression list update failed:", err);
|
|
14933
|
+
}
|
|
14934
|
+
}
|
|
14935
|
+
let recencyRebuilt = false;
|
|
14936
|
+
if (stateDb2) {
|
|
14937
|
+
tracker.start("recency", {});
|
|
14938
|
+
try {
|
|
14939
|
+
const entities = getAllEntitiesFromDb3(stateDb2).map((e) => ({
|
|
14940
|
+
name: e.name,
|
|
14941
|
+
path: e.path,
|
|
14942
|
+
aliases: e.aliases
|
|
14943
|
+
}));
|
|
14944
|
+
const recencyIndex2 = await buildRecencyIndex(vaultPath2, entities);
|
|
14945
|
+
saveRecencyToStateDb(recencyIndex2, stateDb2);
|
|
14946
|
+
recencyRebuilt = true;
|
|
14947
|
+
tracker.end({ entities: recencyIndex2.lastMentioned.size });
|
|
14948
|
+
console.error(`[Flywheel] Recency: rebuilt ${recencyIndex2.lastMentioned.size} entities`);
|
|
14949
|
+
} catch (err) {
|
|
14950
|
+
tracker.end({ error: String(err) });
|
|
14951
|
+
console.error("[Flywheel] Recency rebuild failed:", err);
|
|
14952
|
+
}
|
|
14953
|
+
}
|
|
14954
|
+
let cooccurrenceAssociations;
|
|
14955
|
+
if (stateDb2) {
|
|
14956
|
+
tracker.start("cooccurrence", {});
|
|
14957
|
+
try {
|
|
14958
|
+
const entityNames = getAllEntitiesFromDb3(stateDb2).map((e) => e.name);
|
|
14959
|
+
const cooccurrenceIdx = await mineCooccurrences(vaultPath2, entityNames);
|
|
14960
|
+
setCooccurrenceIndex(cooccurrenceIdx);
|
|
14961
|
+
saveCooccurrenceToStateDb(stateDb2, cooccurrenceIdx);
|
|
14962
|
+
cooccurrenceAssociations = cooccurrenceIdx._metadata.total_associations;
|
|
14963
|
+
tracker.end({ associations: cooccurrenceAssociations });
|
|
14964
|
+
console.error(`[Flywheel] Co-occurrence: rebuilt ${cooccurrenceAssociations} associations`);
|
|
14965
|
+
} catch (err) {
|
|
14966
|
+
tracker.end({ error: String(err) });
|
|
14967
|
+
console.error("[Flywheel] Co-occurrence rebuild failed:", err);
|
|
14968
|
+
}
|
|
14969
|
+
}
|
|
14970
|
+
let taskCacheRefreshed = false;
|
|
14971
|
+
tracker.start("task_cache", {});
|
|
14972
|
+
try {
|
|
14973
|
+
if (!flywheelConfig2) {
|
|
14974
|
+
flywheelConfig2 = loadConfig(stateDb2);
|
|
14975
|
+
}
|
|
14976
|
+
await buildTaskCache(vaultPath2, newIndex, getExcludeTags(flywheelConfig2));
|
|
14977
|
+
taskCacheRefreshed = true;
|
|
14978
|
+
tracker.end({ rebuilt: true });
|
|
14979
|
+
console.error("[Flywheel] Task cache rebuilt");
|
|
14980
|
+
} catch (err) {
|
|
14981
|
+
tracker.end({ error: String(err) });
|
|
14982
|
+
console.error("[Flywheel] Task cache rebuild failed:", err);
|
|
14983
|
+
}
|
|
14816
14984
|
let embeddingsRefreshed = 0;
|
|
14817
14985
|
if (hasEmbeddingsIndex()) {
|
|
14986
|
+
tracker.start("embeddings_sync", {});
|
|
14818
14987
|
try {
|
|
14819
14988
|
const progress = await buildEmbeddingsIndex(vaultPath2);
|
|
14820
14989
|
embeddingsRefreshed = progress.total - progress.skipped;
|
|
14990
|
+
tracker.end({ refreshed: embeddingsRefreshed });
|
|
14821
14991
|
if (embeddingsRefreshed > 0) {
|
|
14822
14992
|
console.error(`[Flywheel] Embeddings: ${embeddingsRefreshed} notes updated`);
|
|
14823
14993
|
}
|
|
14824
14994
|
} catch (err) {
|
|
14995
|
+
tracker.end({ error: String(err) });
|
|
14825
14996
|
console.error("[Flywheel] Embedding sync failed:", err);
|
|
14826
14997
|
}
|
|
14827
14998
|
}
|
|
14999
|
+
let entityEmbeddingsRefreshed = 0;
|
|
15000
|
+
if (stateDb2 && hasEntityEmbeddingsIndex()) {
|
|
15001
|
+
tracker.start("entity_embeddings", {});
|
|
15002
|
+
try {
|
|
15003
|
+
const entities = getAllEntitiesFromDb3(stateDb2);
|
|
15004
|
+
if (entities.length > 0) {
|
|
15005
|
+
const entityMap = new Map(entities.map((e) => [e.name, {
|
|
15006
|
+
name: e.name,
|
|
15007
|
+
path: e.path,
|
|
15008
|
+
category: e.category,
|
|
15009
|
+
aliases: e.aliases
|
|
15010
|
+
}]));
|
|
15011
|
+
entityEmbeddingsRefreshed = await buildEntityEmbeddingsIndex(vaultPath2, entityMap);
|
|
15012
|
+
loadEntityEmbeddingsToMemory();
|
|
15013
|
+
tracker.end({ refreshed: entityEmbeddingsRefreshed });
|
|
15014
|
+
if (entityEmbeddingsRefreshed > 0) {
|
|
15015
|
+
console.error(`[Flywheel] Entity embeddings: ${entityEmbeddingsRefreshed} updated`);
|
|
15016
|
+
}
|
|
15017
|
+
} else {
|
|
15018
|
+
tracker.end({ refreshed: 0 });
|
|
15019
|
+
}
|
|
15020
|
+
} catch (err) {
|
|
15021
|
+
tracker.end({ error: String(err) });
|
|
15022
|
+
console.error("[Flywheel] Entity embedding sync failed:", err);
|
|
15023
|
+
}
|
|
15024
|
+
}
|
|
15025
|
+
let indexCached = false;
|
|
15026
|
+
if (stateDb2) {
|
|
15027
|
+
tracker.start("index_cache", {});
|
|
15028
|
+
try {
|
|
15029
|
+
saveVaultIndexToCache(stateDb2, newIndex);
|
|
15030
|
+
indexCached = true;
|
|
15031
|
+
tracker.end({ cached: true });
|
|
15032
|
+
} catch (err) {
|
|
15033
|
+
tracker.end({ error: String(err) });
|
|
15034
|
+
console.error("[Flywheel] Index cache save failed:", err);
|
|
15035
|
+
}
|
|
15036
|
+
}
|
|
14828
15037
|
const duration = Date.now() - startTime;
|
|
14829
15038
|
if (stateDb2) {
|
|
14830
15039
|
recordIndexEvent(stateDb2, {
|
|
14831
15040
|
trigger: "manual_refresh",
|
|
14832
15041
|
duration_ms: duration,
|
|
14833
|
-
note_count: newIndex.notes.size
|
|
15042
|
+
note_count: newIndex.notes.size,
|
|
15043
|
+
steps: tracker.steps
|
|
14834
15044
|
});
|
|
14835
15045
|
}
|
|
14836
15046
|
const output = {
|
|
@@ -14839,7 +15049,15 @@ function registerSystemTools(server2, getIndex, setIndex, getVaultPath, setConfi
|
|
|
14839
15049
|
entities_count: newIndex.entities.size,
|
|
14840
15050
|
fts5_notes: fts5Notes,
|
|
14841
15051
|
edges_recomputed: edgesRecomputed,
|
|
15052
|
+
hub_scores: hubScoresExported || void 0,
|
|
15053
|
+
graph_snapshot: graphSnapshotRecorded || void 0,
|
|
15054
|
+
suppression_list: suppressionUpdated || void 0,
|
|
15055
|
+
task_cache: taskCacheRefreshed || void 0,
|
|
14842
15056
|
embeddings_refreshed: embeddingsRefreshed || void 0,
|
|
15057
|
+
entity_embeddings_refreshed: entityEmbeddingsRefreshed || void 0,
|
|
15058
|
+
recency_rebuilt: recencyRebuilt || void 0,
|
|
15059
|
+
cooccurrence_associations: cooccurrenceAssociations,
|
|
15060
|
+
index_cached: indexCached || void 0,
|
|
14843
15061
|
duration_ms: duration
|
|
14844
15062
|
};
|
|
14845
15063
|
return {
|
|
@@ -22728,7 +22946,7 @@ function registerSimilarityTools(server2, getIndex, getVaultPath, getStateDb3) {
|
|
|
22728
22946
|
// src/tools/read/semantic.ts
|
|
22729
22947
|
init_embeddings();
|
|
22730
22948
|
import { z as z32 } from "zod";
|
|
22731
|
-
import { getAllEntitiesFromDb as
|
|
22949
|
+
import { getAllEntitiesFromDb as getAllEntitiesFromDb4 } from "@velvetmonkey/vault-core";
|
|
22732
22950
|
function registerSemanticTools(server2, getVaultPath, getStateDb3) {
|
|
22733
22951
|
server2.registerTool(
|
|
22734
22952
|
"init_semantic",
|
|
@@ -22792,7 +23010,7 @@ function registerSemanticTools(server2, getVaultPath, getStateDb3) {
|
|
|
22792
23010
|
const embedded = progress.total - progress.skipped;
|
|
22793
23011
|
let entityEmbedded = 0;
|
|
22794
23012
|
try {
|
|
22795
|
-
const allEntities =
|
|
23013
|
+
const allEntities = getAllEntitiesFromDb4(stateDb2);
|
|
22796
23014
|
const entityMap = /* @__PURE__ */ new Map();
|
|
22797
23015
|
for (const e of allEntities) {
|
|
22798
23016
|
entityMap.set(e.name, {
|
|
@@ -22844,7 +23062,7 @@ function registerSemanticTools(server2, getVaultPath, getStateDb3) {
|
|
|
22844
23062
|
// src/tools/read/merges.ts
|
|
22845
23063
|
init_levenshtein();
|
|
22846
23064
|
import { z as z33 } from "zod";
|
|
22847
|
-
import { getAllEntitiesFromDb as
|
|
23065
|
+
import { getAllEntitiesFromDb as getAllEntitiesFromDb5, getDismissedMergePairs, recordMergeDismissal } from "@velvetmonkey/vault-core";
|
|
22848
23066
|
function normalizeName(name) {
|
|
22849
23067
|
return name.toLowerCase().replace(/[.\-_]/g, "").replace(/js$/, "").replace(/ts$/, "");
|
|
22850
23068
|
}
|
|
@@ -22862,7 +23080,7 @@ function registerMergeTools2(server2, getStateDb3) {
|
|
|
22862
23080
|
content: [{ type: "text", text: JSON.stringify({ suggestions: [], error: "StateDb not available" }) }]
|
|
22863
23081
|
};
|
|
22864
23082
|
}
|
|
22865
|
-
const entities =
|
|
23083
|
+
const entities = getAllEntitiesFromDb5(stateDb2);
|
|
22866
23084
|
if (entities.length === 0) {
|
|
22867
23085
|
return {
|
|
22868
23086
|
content: [{ type: "text", text: JSON.stringify({ suggestions: [] }) }]
|
|
@@ -24768,7 +24986,8 @@ function registerAllTools(targetServer, ctx) {
|
|
|
24768
24986
|
var __filename = fileURLToPath2(import.meta.url);
|
|
24769
24987
|
var __dirname = dirname7(__filename);
|
|
24770
24988
|
var pkg = JSON.parse(readFileSync6(join21(__dirname, "../package.json"), "utf-8"));
|
|
24771
|
-
var
|
|
24989
|
+
var _earlyVaultConfigs = parseVaultConfig();
|
|
24990
|
+
var vaultPath = _earlyVaultConfigs ? _earlyVaultConfigs[0].path : process.env.PROJECT_PATH || process.env.VAULT_PATH || findVaultRoot();
|
|
24772
24991
|
var resolvedVaultPath;
|
|
24773
24992
|
try {
|
|
24774
24993
|
resolvedVaultPath = realpathSync(vaultPath).replace(/\\/g, "/");
|
|
@@ -25046,7 +25265,7 @@ async function main() {
|
|
|
25046
25265
|
serverLog("server", `Starting Flywheel Memory v${pkg.version}...`);
|
|
25047
25266
|
serverLog("server", `Vault: ${vaultPath}`);
|
|
25048
25267
|
const startTime = Date.now();
|
|
25049
|
-
const vaultConfigs =
|
|
25268
|
+
const vaultConfigs = _earlyVaultConfigs;
|
|
25050
25269
|
if (vaultConfigs) {
|
|
25051
25270
|
vaultRegistry = new VaultRegistry(vaultConfigs[0].name);
|
|
25052
25271
|
serverLog("server", `Multi-vault mode: ${vaultConfigs.map((v) => v.name).join(", ")}`);
|
|
@@ -25122,7 +25341,6 @@ async function main() {
|
|
|
25122
25341
|
})();
|
|
25123
25342
|
}
|
|
25124
25343
|
}
|
|
25125
|
-
var DEFAULT_ENTITY_EXCLUDE_FOLDERS = ["node_modules", "templates", "attachments", "tmp"];
|
|
25126
25344
|
async function updateEntitiesInStateDb(vp, sd) {
|
|
25127
25345
|
const db4 = sd ?? stateDb;
|
|
25128
25346
|
const vault = vp ?? vaultPath;
|
|
@@ -25293,7 +25511,7 @@ async function runPostIndexWork(ctx) {
|
|
|
25293
25511
|
}
|
|
25294
25512
|
});
|
|
25295
25513
|
if (sd) {
|
|
25296
|
-
const entities =
|
|
25514
|
+
const entities = getAllEntitiesFromDb6(sd);
|
|
25297
25515
|
if (entities.length > 0) {
|
|
25298
25516
|
const entityMap = new Map(entities.map((e) => [e.name, {
|
|
25299
25517
|
name: e.name,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@velvetmonkey/flywheel-memory",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.157",
|
|
4
4
|
"description": "MCP tools that search, write, and auto-link your Obsidian vault — and learn from your edits.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -54,7 +54,7 @@
|
|
|
54
54
|
"dependencies": {
|
|
55
55
|
"@huggingface/transformers": "^3.8.1",
|
|
56
56
|
"@modelcontextprotocol/sdk": "^1.25.1",
|
|
57
|
-
"@velvetmonkey/vault-core": "^2.0.
|
|
57
|
+
"@velvetmonkey/vault-core": "^2.0.157",
|
|
58
58
|
"better-sqlite3": "^12.0.0",
|
|
59
59
|
"chokidar": "^4.0.0",
|
|
60
60
|
"gray-matter": "^4.0.3",
|