@velvetmonkey/flywheel-memory 2.0.48 → 2.0.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +448 -152
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -3493,23 +3493,39 @@ import {
|
|
|
3493
3493
|
} from "@velvetmonkey/vault-core";
|
|
3494
3494
|
|
|
3495
3495
|
// src/core/write/wikilinkFeedback.ts
|
|
3496
|
-
var
|
|
3497
|
-
var
|
|
3498
|
-
var
|
|
3496
|
+
var PRIOR_ALPHA = 8;
|
|
3497
|
+
var PRIOR_BETA = 1;
|
|
3498
|
+
var SUPPRESSION_POSTERIOR_THRESHOLD = 0.35;
|
|
3499
|
+
var SUPPRESSION_MIN_OBSERVATIONS = 20;
|
|
3500
|
+
var MAX_SUPPRESSION_PENALTY = -15;
|
|
3501
|
+
var FEEDBACK_BOOST_MIN_SAMPLES = 3;
|
|
3499
3502
|
var FOLDER_SUPPRESSION_MIN_COUNT = 5;
|
|
3503
|
+
var SUPPRESSION_TTL_DAYS = 30;
|
|
3504
|
+
var FEEDBACK_DECAY_HALF_LIFE_DAYS = 30;
|
|
3505
|
+
var FEEDBACK_DECAY_LAMBDA = Math.LN2 / FEEDBACK_DECAY_HALF_LIFE_DAYS;
|
|
3506
|
+
function computePosteriorMean(weightedCorrect, weightedFp) {
|
|
3507
|
+
const alpha = PRIOR_ALPHA + weightedCorrect;
|
|
3508
|
+
const beta_ = PRIOR_BETA + weightedFp;
|
|
3509
|
+
return alpha / (alpha + beta_);
|
|
3510
|
+
}
|
|
3500
3511
|
var FEEDBACK_BOOST_TIERS = [
|
|
3501
|
-
{ minAccuracy: 0.
|
|
3502
|
-
|
|
3503
|
-
{ minAccuracy: 0.
|
|
3504
|
-
|
|
3505
|
-
{ minAccuracy: 0, minSamples:
|
|
3512
|
+
{ minAccuracy: 0.85, minSamples: 5, boost: 10 },
|
|
3513
|
+
// Strong reward for high accuracy
|
|
3514
|
+
{ minAccuracy: 0.7, minSamples: 3, boost: 6 },
|
|
3515
|
+
// Most TPs land here
|
|
3516
|
+
{ minAccuracy: 0.5, minSamples: 3, boost: 0 },
|
|
3517
|
+
// Neutral — Layer 0 suppression handles demotion
|
|
3518
|
+
{ minAccuracy: 0.3, minSamples: 3, boost: 0 },
|
|
3519
|
+
// Neutral — no double-penalty with Layer 0
|
|
3520
|
+
{ minAccuracy: 0, minSamples: 3, boost: 0 }
|
|
3521
|
+
// Neutral — no double-penalty with Layer 0
|
|
3506
3522
|
];
|
|
3507
|
-
function recordFeedback(stateDb2, entity, context, notePath, correct) {
|
|
3523
|
+
function recordFeedback(stateDb2, entity, context, notePath, correct, confidence = 1) {
|
|
3508
3524
|
try {
|
|
3509
3525
|
console.error(`[Flywheel] recordFeedback: entity="${entity}" context="${context}" notePath="${notePath}" correct=${correct}`);
|
|
3510
3526
|
const result = stateDb2.db.prepare(
|
|
3511
|
-
"INSERT INTO wikilink_feedback (entity, context, note_path, correct) VALUES (?, ?, ?, ?)"
|
|
3512
|
-
).run(entity, context, notePath, correct ? 1 : 0);
|
|
3527
|
+
"INSERT INTO wikilink_feedback (entity, context, note_path, correct, confidence) VALUES (?, ?, ?, ?, ?)"
|
|
3528
|
+
).run(entity, context, notePath, correct ? 1 : 0, confidence);
|
|
3513
3529
|
console.error(`[Flywheel] recordFeedback: inserted id=${result.lastInsertRowid}`);
|
|
3514
3530
|
} catch (e) {
|
|
3515
3531
|
console.error(`[Flywheel] recordFeedback failed for entity="${entity}": ${e}`);
|
|
@@ -3559,16 +3575,91 @@ function getEntityStats(stateDb2) {
|
|
|
3559
3575
|
};
|
|
3560
3576
|
});
|
|
3561
3577
|
}
|
|
3562
|
-
function
|
|
3563
|
-
const
|
|
3564
|
-
|
|
3578
|
+
function computeFeedbackWeight(createdAt, now) {
|
|
3579
|
+
const ref = now ?? /* @__PURE__ */ new Date();
|
|
3580
|
+
const normalized = createdAt.includes("T") ? createdAt : createdAt.replace(" ", "T") + "Z";
|
|
3581
|
+
const ageDays = (ref.getTime() - new Date(normalized).getTime()) / (24 * 60 * 60 * 1e3);
|
|
3582
|
+
if (ageDays < 1 / 1440) return 1;
|
|
3583
|
+
return Math.exp(-FEEDBACK_DECAY_LAMBDA * ageDays);
|
|
3584
|
+
}
|
|
3585
|
+
function getWeightedEntityStats(stateDb2, now) {
|
|
3586
|
+
const rows = stateDb2.db.prepare(`
|
|
3587
|
+
SELECT entity, correct, confidence, created_at
|
|
3588
|
+
FROM wikilink_feedback
|
|
3589
|
+
ORDER BY entity COLLATE NOCASE
|
|
3590
|
+
`).all();
|
|
3591
|
+
const acc = /* @__PURE__ */ new Map();
|
|
3592
|
+
for (const row of rows) {
|
|
3593
|
+
const key = row.entity.toLowerCase();
|
|
3594
|
+
if (!acc.has(key)) {
|
|
3595
|
+
acc.set(key, { weightedTotal: 0, weightedCorrect: 0, weightedFp: 0, rawTotal: 0 });
|
|
3596
|
+
}
|
|
3597
|
+
const stats = acc.get(key);
|
|
3598
|
+
const recencyWeight = computeFeedbackWeight(row.created_at, now);
|
|
3599
|
+
const weight = recencyWeight * (row.confidence ?? 1);
|
|
3600
|
+
stats.weightedTotal += weight;
|
|
3601
|
+
stats.rawTotal++;
|
|
3602
|
+
if (row.correct === 1) {
|
|
3603
|
+
stats.weightedCorrect += weight;
|
|
3604
|
+
} else {
|
|
3605
|
+
stats.weightedFp += weight;
|
|
3606
|
+
}
|
|
3607
|
+
if (stats.rawTotal === 1 || !acc.has(row.entity)) {
|
|
3608
|
+
}
|
|
3609
|
+
}
|
|
3610
|
+
const entityNames = /* @__PURE__ */ new Map();
|
|
3611
|
+
for (const row of rows) {
|
|
3612
|
+
const key = row.entity.toLowerCase();
|
|
3613
|
+
if (!entityNames.has(key)) entityNames.set(key, row.entity);
|
|
3614
|
+
}
|
|
3615
|
+
const result = [];
|
|
3616
|
+
for (const [key, stats] of acc) {
|
|
3617
|
+
const entity = entityNames.get(key) ?? key;
|
|
3618
|
+
result.push({
|
|
3565
3619
|
entity,
|
|
3566
|
-
|
|
3567
|
-
|
|
3620
|
+
weightedTotal: stats.weightedTotal,
|
|
3621
|
+
weightedCorrect: stats.weightedCorrect,
|
|
3622
|
+
weightedFp: stats.weightedFp,
|
|
3623
|
+
rawTotal: stats.rawTotal,
|
|
3624
|
+
weightedAccuracy: stats.weightedTotal > 0 ? stats.weightedCorrect / stats.weightedTotal : 0,
|
|
3625
|
+
weightedFpRate: stats.weightedTotal > 0 ? stats.weightedFp / stats.weightedTotal : 0
|
|
3626
|
+
});
|
|
3627
|
+
}
|
|
3628
|
+
return result;
|
|
3629
|
+
}
|
|
3630
|
+
function getWeightedFolderStats(stateDb2, entity, folder, now) {
|
|
3631
|
+
const rows = stateDb2.db.prepare(`
|
|
3632
|
+
SELECT correct, confidence, created_at
|
|
3568
3633
|
FROM wikilink_feedback
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3634
|
+
WHERE entity = ? COLLATE NOCASE AND (
|
|
3635
|
+
CASE WHEN ? = '' THEN note_path NOT LIKE '%/%'
|
|
3636
|
+
ELSE note_path LIKE ? || '/%'
|
|
3637
|
+
END
|
|
3638
|
+
)
|
|
3639
|
+
`).all(entity, folder, folder);
|
|
3640
|
+
let weightedTotal = 0;
|
|
3641
|
+
let weightedCorrect = 0;
|
|
3642
|
+
let weightedFp = 0;
|
|
3643
|
+
for (const row of rows) {
|
|
3644
|
+
const recencyWeight = computeFeedbackWeight(row.created_at, now);
|
|
3645
|
+
const weight = recencyWeight * (row.confidence ?? 1);
|
|
3646
|
+
weightedTotal += weight;
|
|
3647
|
+
if (row.correct === 1) {
|
|
3648
|
+
weightedCorrect += weight;
|
|
3649
|
+
} else {
|
|
3650
|
+
weightedFp += weight;
|
|
3651
|
+
}
|
|
3652
|
+
}
|
|
3653
|
+
return {
|
|
3654
|
+
weightedTotal,
|
|
3655
|
+
weightedFp,
|
|
3656
|
+
weightedAccuracy: weightedTotal > 0 ? weightedCorrect / weightedTotal : 0,
|
|
3657
|
+
weightedFpRate: weightedTotal > 0 ? weightedFp / weightedTotal : 0,
|
|
3658
|
+
rawTotal: rows.length
|
|
3659
|
+
};
|
|
3660
|
+
}
|
|
3661
|
+
function updateSuppressionList(stateDb2, now) {
|
|
3662
|
+
const weightedStats = getWeightedEntityStats(stateDb2, now);
|
|
3572
3663
|
let updated = 0;
|
|
3573
3664
|
const upsert = stateDb2.db.prepare(`
|
|
3574
3665
|
INSERT INTO wikilink_suppressions (entity, false_positive_rate, updated_at)
|
|
@@ -3581,10 +3672,18 @@ function updateSuppressionList(stateDb2) {
|
|
|
3581
3672
|
"DELETE FROM wikilink_suppressions WHERE entity = ?"
|
|
3582
3673
|
);
|
|
3583
3674
|
const transaction = stateDb2.db.transaction(() => {
|
|
3584
|
-
|
|
3585
|
-
|
|
3586
|
-
|
|
3587
|
-
|
|
3675
|
+
stateDb2.db.prepare(
|
|
3676
|
+
`DELETE FROM wikilink_suppressions
|
|
3677
|
+
WHERE datetime(updated_at, '+' || ? || ' days') <= datetime('now')`
|
|
3678
|
+
).run(SUPPRESSION_TTL_DAYS);
|
|
3679
|
+
for (const stat4 of weightedStats) {
|
|
3680
|
+
const posteriorMean = computePosteriorMean(stat4.weightedCorrect, stat4.weightedFp);
|
|
3681
|
+
const totalObs = PRIOR_ALPHA + stat4.weightedCorrect + PRIOR_BETA + stat4.weightedFp;
|
|
3682
|
+
if (totalObs < SUPPRESSION_MIN_OBSERVATIONS) {
|
|
3683
|
+
continue;
|
|
3684
|
+
}
|
|
3685
|
+
if (posteriorMean < SUPPRESSION_POSTERIOR_THRESHOLD) {
|
|
3686
|
+
upsert.run(stat4.entity, 1 - posteriorMean);
|
|
3588
3687
|
updated++;
|
|
3589
3688
|
} else {
|
|
3590
3689
|
remove.run(stat4.entity);
|
|
@@ -3607,26 +3706,20 @@ function unsuppressEntity(stateDb2, entity) {
|
|
|
3607
3706
|
).run(entity);
|
|
3608
3707
|
return result.changes > 0;
|
|
3609
3708
|
}
|
|
3610
|
-
function isSuppressed(stateDb2, entity, folder) {
|
|
3709
|
+
function isSuppressed(stateDb2, entity, folder, now) {
|
|
3611
3710
|
const row = stateDb2.db.prepare(
|
|
3612
|
-
|
|
3613
|
-
|
|
3711
|
+
`SELECT entity, updated_at FROM wikilink_suppressions
|
|
3712
|
+
WHERE entity = ? COLLATE NOCASE
|
|
3713
|
+
AND datetime(updated_at, '+' || ? || ' days') > datetime('now')`
|
|
3714
|
+
).get(entity, SUPPRESSION_TTL_DAYS);
|
|
3614
3715
|
if (row) return true;
|
|
3615
3716
|
if (folder !== void 0) {
|
|
3616
|
-
const
|
|
3617
|
-
|
|
3618
|
-
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
3622
|
-
CASE WHEN ? = '' THEN note_path NOT LIKE '%/%'
|
|
3623
|
-
ELSE note_path LIKE ? || '/%'
|
|
3624
|
-
END
|
|
3625
|
-
)
|
|
3626
|
-
`).get(entity, folder, folder);
|
|
3627
|
-
if (folderStats && folderStats.total >= FOLDER_SUPPRESSION_MIN_COUNT) {
|
|
3628
|
-
const fpRate = folderStats.false_positives / folderStats.total;
|
|
3629
|
-
if (fpRate >= SUPPRESSION_THRESHOLD) {
|
|
3717
|
+
const stats = getWeightedFolderStats(stateDb2, entity, folder, now);
|
|
3718
|
+
if (stats.rawTotal >= FOLDER_SUPPRESSION_MIN_COUNT) {
|
|
3719
|
+
const folderCorrect = stats.weightedTotal - stats.weightedFp;
|
|
3720
|
+
const posteriorMean = computePosteriorMean(folderCorrect, stats.weightedFp);
|
|
3721
|
+
const totalObs = PRIOR_ALPHA + folderCorrect + PRIOR_BETA + stats.weightedFp;
|
|
3722
|
+
if (totalObs >= SUPPRESSION_MIN_OBSERVATIONS && posteriorMean < SUPPRESSION_POSTERIOR_THRESHOLD) {
|
|
3630
3723
|
return true;
|
|
3631
3724
|
}
|
|
3632
3725
|
}
|
|
@@ -3656,59 +3749,65 @@ function computeBoostFromAccuracy(accuracy, sampleCount) {
|
|
|
3656
3749
|
}
|
|
3657
3750
|
return 0;
|
|
3658
3751
|
}
|
|
3659
|
-
function getAllFeedbackBoosts(stateDb2, folder) {
|
|
3660
|
-
const
|
|
3661
|
-
|
|
3662
|
-
entity,
|
|
3663
|
-
COUNT(*) as total,
|
|
3664
|
-
SUM(CASE WHEN correct = 1 THEN 1 ELSE 0 END) as correct_count
|
|
3665
|
-
FROM wikilink_feedback
|
|
3666
|
-
GROUP BY entity
|
|
3667
|
-
HAVING total >= ?
|
|
3668
|
-
`).all(FEEDBACK_BOOST_MIN_SAMPLES);
|
|
3669
|
-
let folderStats = null;
|
|
3752
|
+
function getAllFeedbackBoosts(stateDb2, folder, now) {
|
|
3753
|
+
const globalStats = getWeightedEntityStats(stateDb2, now);
|
|
3754
|
+
let folderStatsMap = null;
|
|
3670
3755
|
if (folder !== void 0) {
|
|
3671
|
-
|
|
3672
|
-
|
|
3673
|
-
|
|
3674
|
-
|
|
3675
|
-
|
|
3676
|
-
|
|
3677
|
-
|
|
3678
|
-
|
|
3679
|
-
|
|
3680
|
-
END
|
|
3681
|
-
)
|
|
3682
|
-
GROUP BY entity
|
|
3683
|
-
HAVING total >= ?
|
|
3684
|
-
`).all(folder, folder, FEEDBACK_BOOST_MIN_SAMPLES);
|
|
3685
|
-
folderStats = /* @__PURE__ */ new Map();
|
|
3686
|
-
for (const row of folderRows) {
|
|
3687
|
-
folderStats.set(row.entity, {
|
|
3688
|
-
accuracy: row.correct_count / row.total,
|
|
3689
|
-
count: row.total
|
|
3690
|
-
});
|
|
3756
|
+
folderStatsMap = /* @__PURE__ */ new Map();
|
|
3757
|
+
for (const gs of globalStats) {
|
|
3758
|
+
const fs32 = getWeightedFolderStats(stateDb2, gs.entity, folder, now);
|
|
3759
|
+
if (fs32.rawTotal >= FEEDBACK_BOOST_MIN_SAMPLES) {
|
|
3760
|
+
folderStatsMap.set(gs.entity, {
|
|
3761
|
+
weightedAccuracy: fs32.weightedAccuracy,
|
|
3762
|
+
rawCount: fs32.rawTotal
|
|
3763
|
+
});
|
|
3764
|
+
}
|
|
3691
3765
|
}
|
|
3692
3766
|
}
|
|
3693
3767
|
const boosts = /* @__PURE__ */ new Map();
|
|
3694
|
-
for (const
|
|
3768
|
+
for (const stat4 of globalStats) {
|
|
3769
|
+
if (stat4.rawTotal < FEEDBACK_BOOST_MIN_SAMPLES) continue;
|
|
3695
3770
|
let accuracy;
|
|
3696
3771
|
let sampleCount;
|
|
3697
|
-
const fs32 =
|
|
3698
|
-
if (fs32 && fs32.
|
|
3699
|
-
accuracy = fs32.
|
|
3700
|
-
sampleCount = fs32.
|
|
3772
|
+
const fs32 = folderStatsMap?.get(stat4.entity);
|
|
3773
|
+
if (fs32 && fs32.rawCount >= FEEDBACK_BOOST_MIN_SAMPLES) {
|
|
3774
|
+
accuracy = fs32.weightedAccuracy;
|
|
3775
|
+
sampleCount = fs32.rawCount;
|
|
3701
3776
|
} else {
|
|
3702
|
-
accuracy =
|
|
3703
|
-
sampleCount =
|
|
3777
|
+
accuracy = stat4.weightedAccuracy;
|
|
3778
|
+
sampleCount = stat4.rawTotal;
|
|
3704
3779
|
}
|
|
3705
3780
|
const boost = computeBoostFromAccuracy(accuracy, sampleCount);
|
|
3706
3781
|
if (boost !== 0) {
|
|
3707
|
-
boosts.set(
|
|
3782
|
+
boosts.set(stat4.entity, boost);
|
|
3708
3783
|
}
|
|
3709
3784
|
}
|
|
3710
3785
|
return boosts;
|
|
3711
3786
|
}
|
|
3787
|
+
function getAllSuppressionPenalties(stateDb2, now) {
|
|
3788
|
+
const penalties = /* @__PURE__ */ new Map();
|
|
3789
|
+
const weightedStats = getWeightedEntityStats(stateDb2, now);
|
|
3790
|
+
for (const stat4 of weightedStats) {
|
|
3791
|
+
const posteriorMean = computePosteriorMean(stat4.weightedCorrect, stat4.weightedFp);
|
|
3792
|
+
const totalObs = PRIOR_ALPHA + stat4.weightedCorrect + PRIOR_BETA + stat4.weightedFp;
|
|
3793
|
+
if (totalObs >= SUPPRESSION_MIN_OBSERVATIONS && posteriorMean < SUPPRESSION_POSTERIOR_THRESHOLD) {
|
|
3794
|
+
const penalty = Math.round(MAX_SUPPRESSION_PENALTY * (1 - posteriorMean / SUPPRESSION_POSTERIOR_THRESHOLD));
|
|
3795
|
+
if (penalty < 0) {
|
|
3796
|
+
penalties.set(stat4.entity, penalty);
|
|
3797
|
+
}
|
|
3798
|
+
}
|
|
3799
|
+
}
|
|
3800
|
+
const rows = stateDb2.db.prepare(
|
|
3801
|
+
`SELECT entity, updated_at FROM wikilink_suppressions
|
|
3802
|
+
WHERE datetime(updated_at, '+' || ? || ' days') > datetime('now')`
|
|
3803
|
+
).all(SUPPRESSION_TTL_DAYS);
|
|
3804
|
+
for (const row of rows) {
|
|
3805
|
+
if (!penalties.has(row.entity)) {
|
|
3806
|
+
penalties.set(row.entity, MAX_SUPPRESSION_PENALTY);
|
|
3807
|
+
}
|
|
3808
|
+
}
|
|
3809
|
+
return penalties;
|
|
3810
|
+
}
|
|
3712
3811
|
function trackWikilinkApplications(stateDb2, notePath, entities) {
|
|
3713
3812
|
const upsert = stateDb2.db.prepare(`
|
|
3714
3813
|
INSERT INTO wikilink_applications (entity, note_path, applied_at, status)
|
|
@@ -3730,6 +3829,18 @@ function getTrackedApplications(stateDb2, notePath) {
|
|
|
3730
3829
|
).all(notePath);
|
|
3731
3830
|
return rows.map((r) => r.entity);
|
|
3732
3831
|
}
|
|
3832
|
+
function getTrackedApplicationsWithTime(stateDb2, notePath) {
|
|
3833
|
+
return stateDb2.db.prepare(
|
|
3834
|
+
`SELECT entity, applied_at FROM wikilink_applications WHERE note_path = ? AND status = 'applied'`
|
|
3835
|
+
).all(notePath);
|
|
3836
|
+
}
|
|
3837
|
+
function computeImplicitRemovalConfidence(appliedAt) {
|
|
3838
|
+
const normalized = appliedAt.includes("T") ? appliedAt : appliedAt.replace(" ", "T") + "Z";
|
|
3839
|
+
const ageHours = (Date.now() - new Date(normalized).getTime()) / (60 * 60 * 1e3);
|
|
3840
|
+
if (ageHours <= 1) return 1;
|
|
3841
|
+
if (ageHours <= 24) return 0.85;
|
|
3842
|
+
return 0.7;
|
|
3843
|
+
}
|
|
3733
3844
|
function getStoredNoteLinks(stateDb2, notePath) {
|
|
3734
3845
|
const rows = stateDb2.db.prepare(
|
|
3735
3846
|
"SELECT target FROM note_links WHERE note_path = ?"
|
|
@@ -3783,17 +3894,18 @@ function updateStoredNoteTags(stateDb2, notePath, currentTags) {
|
|
|
3783
3894
|
tx();
|
|
3784
3895
|
}
|
|
3785
3896
|
function processImplicitFeedback(stateDb2, notePath, currentContent) {
|
|
3786
|
-
const
|
|
3787
|
-
if (
|
|
3897
|
+
const trackedWithTime = getTrackedApplicationsWithTime(stateDb2, notePath);
|
|
3898
|
+
if (trackedWithTime.length === 0) return [];
|
|
3788
3899
|
const currentLinks = extractLinkedEntities(currentContent);
|
|
3789
3900
|
const removed = [];
|
|
3790
3901
|
const markRemoved = stateDb2.db.prepare(
|
|
3791
3902
|
`UPDATE wikilink_applications SET status = 'removed' WHERE entity = ? AND note_path = ?`
|
|
3792
3903
|
);
|
|
3793
3904
|
const transaction = stateDb2.db.transaction(() => {
|
|
3794
|
-
for (const entity of
|
|
3905
|
+
for (const { entity, applied_at } of trackedWithTime) {
|
|
3795
3906
|
if (!currentLinks.has(entity.toLowerCase())) {
|
|
3796
|
-
|
|
3907
|
+
const confidence = computeImplicitRemovalConfidence(applied_at);
|
|
3908
|
+
recordFeedback(stateDb2, entity, "implicit:removed", notePath, false, confidence);
|
|
3797
3909
|
markRemoved.run(entity, notePath);
|
|
3798
3910
|
removed.push(entity);
|
|
3799
3911
|
}
|
|
@@ -3806,11 +3918,11 @@ function processImplicitFeedback(stateDb2, notePath, currentContent) {
|
|
|
3806
3918
|
return removed;
|
|
3807
3919
|
}
|
|
3808
3920
|
var TIER_LABELS = [
|
|
3809
|
-
{ label: "Champion (+
|
|
3810
|
-
{ label: "Strong (+
|
|
3811
|
-
{ label: "Neutral (0)", boost: 0, minAccuracy: 0.
|
|
3812
|
-
{ label: "
|
|
3813
|
-
{ label: "
|
|
3921
|
+
{ label: "Champion (+10)", boost: 10, minAccuracy: 0.85, minSamples: 5 },
|
|
3922
|
+
{ label: "Strong (+6)", boost: 6, minAccuracy: 0.7, minSamples: 3 },
|
|
3923
|
+
{ label: "Neutral (0)", boost: 0, minAccuracy: 0.5, minSamples: 3 },
|
|
3924
|
+
{ label: "Neutral (0)", boost: 0, minAccuracy: 0.3, minSamples: 3 },
|
|
3925
|
+
{ label: "Neutral (0)", boost: 0, minAccuracy: 0, minSamples: 3 }
|
|
3814
3926
|
];
|
|
3815
3927
|
function getDashboardData(stateDb2) {
|
|
3816
3928
|
const entityStats = getEntityStats(stateDb2);
|
|
@@ -5271,14 +5383,14 @@ function stem(word) {
|
|
|
5271
5383
|
}
|
|
5272
5384
|
function tokenize(text) {
|
|
5273
5385
|
const cleanText = text.replace(/\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g, "$1").replace(/[*_`#\[\]()]/g, " ").toLowerCase();
|
|
5274
|
-
const words = cleanText.match(/\b[a-z]{
|
|
5386
|
+
const words = cleanText.match(/\b[a-z][a-z0-9]{2,}\b/g) || [];
|
|
5275
5387
|
return words.filter((word) => !STOPWORDS.has(word));
|
|
5276
5388
|
}
|
|
5277
5389
|
|
|
5278
5390
|
// src/core/shared/cooccurrence.ts
|
|
5279
5391
|
import { readdir as readdir2, readFile as readFile2 } from "fs/promises";
|
|
5280
5392
|
import path10 from "path";
|
|
5281
|
-
var DEFAULT_MIN_COOCCURRENCE =
|
|
5393
|
+
var DEFAULT_MIN_COOCCURRENCE = 0.5;
|
|
5282
5394
|
var EXCLUDED_FOLDERS2 = /* @__PURE__ */ new Set([
|
|
5283
5395
|
"templates",
|
|
5284
5396
|
".obsidian",
|
|
@@ -5300,12 +5412,12 @@ function noteContainsEntity(content, entityName) {
|
|
|
5300
5412
|
}
|
|
5301
5413
|
return matchCount / entityTokens.length >= 0.5;
|
|
5302
5414
|
}
|
|
5303
|
-
function incrementCooccurrence(associations, entityA, entityB) {
|
|
5415
|
+
function incrementCooccurrence(associations, entityA, entityB, weight = 1) {
|
|
5304
5416
|
if (!associations[entityA]) {
|
|
5305
5417
|
associations[entityA] = /* @__PURE__ */ new Map();
|
|
5306
5418
|
}
|
|
5307
5419
|
const current = associations[entityA].get(entityB) || 0;
|
|
5308
|
-
associations[entityA].set(entityB, current +
|
|
5420
|
+
associations[entityA].set(entityB, current + weight);
|
|
5309
5421
|
}
|
|
5310
5422
|
async function* walkMarkdownFiles2(dir, baseDir) {
|
|
5311
5423
|
try {
|
|
@@ -5329,6 +5441,7 @@ async function* walkMarkdownFiles2(dir, baseDir) {
|
|
|
5329
5441
|
async function mineCooccurrences(vaultPath2, entities, options = {}) {
|
|
5330
5442
|
const { minCount = DEFAULT_MIN_COOCCURRENCE } = options;
|
|
5331
5443
|
const associations = {};
|
|
5444
|
+
const documentFrequency = /* @__PURE__ */ new Map();
|
|
5332
5445
|
let notesScanned = 0;
|
|
5333
5446
|
const validEntities = entities.filter((e) => e.length <= 30);
|
|
5334
5447
|
for await (const file of walkMarkdownFiles2(vaultPath2, vaultPath2)) {
|
|
@@ -5341,10 +5454,15 @@ async function mineCooccurrences(vaultPath2, entities, options = {}) {
|
|
|
5341
5454
|
mentionedEntities.push(entity);
|
|
5342
5455
|
}
|
|
5343
5456
|
}
|
|
5457
|
+
for (const entity of mentionedEntities) {
|
|
5458
|
+
documentFrequency.set(entity, (documentFrequency.get(entity) || 0) + 1);
|
|
5459
|
+
}
|
|
5460
|
+
const degree = mentionedEntities.length;
|
|
5461
|
+
const adamicAdarWeight = degree >= 3 ? 1 / Math.log(degree) : 1;
|
|
5344
5462
|
for (const entityA of mentionedEntities) {
|
|
5345
5463
|
for (const entityB of mentionedEntities) {
|
|
5346
5464
|
if (entityA !== entityB) {
|
|
5347
|
-
incrementCooccurrence(associations, entityA, entityB);
|
|
5465
|
+
incrementCooccurrence(associations, entityA, entityB, adamicAdarWeight);
|
|
5348
5466
|
}
|
|
5349
5467
|
}
|
|
5350
5468
|
}
|
|
@@ -5362,6 +5480,8 @@ async function mineCooccurrences(vaultPath2, entities, options = {}) {
|
|
|
5362
5480
|
return {
|
|
5363
5481
|
associations,
|
|
5364
5482
|
minCount,
|
|
5483
|
+
documentFrequency,
|
|
5484
|
+
totalNotesScanned: notesScanned,
|
|
5365
5485
|
_metadata: {
|
|
5366
5486
|
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5367
5487
|
total_associations: totalAssociations,
|
|
@@ -5369,26 +5489,50 @@ async function mineCooccurrences(vaultPath2, entities, options = {}) {
|
|
|
5369
5489
|
}
|
|
5370
5490
|
};
|
|
5371
5491
|
}
|
|
5372
|
-
var MAX_COOCCURRENCE_BOOST =
|
|
5492
|
+
var MAX_COOCCURRENCE_BOOST = 12;
|
|
5493
|
+
var PMI_SCALE = 12;
|
|
5494
|
+
function computeNpmi(coocCount, dfEntity, dfSeed, totalNotes) {
|
|
5495
|
+
if (coocCount === 0 || dfEntity === 0 || dfSeed === 0 || totalNotes === 0) return 0;
|
|
5496
|
+
const pxy = coocCount / totalNotes;
|
|
5497
|
+
const px = dfEntity / totalNotes;
|
|
5498
|
+
const py = dfSeed / totalNotes;
|
|
5499
|
+
const pmi = Math.log(pxy / (px * py));
|
|
5500
|
+
const negLogPxy = -Math.log(pxy);
|
|
5501
|
+
if (negLogPxy === 0) return 1;
|
|
5502
|
+
const npmi = pmi / negLogPxy;
|
|
5503
|
+
return Math.max(0, Math.min(1, npmi));
|
|
5504
|
+
}
|
|
5373
5505
|
function getCooccurrenceBoost(entityName, matchedEntities, cooccurrenceIndex2, recencyIndex2) {
|
|
5374
5506
|
if (!cooccurrenceIndex2) return 0;
|
|
5375
|
-
|
|
5376
|
-
const
|
|
5507
|
+
const { associations, minCount, documentFrequency, totalNotesScanned } = cooccurrenceIndex2;
|
|
5508
|
+
const dfEntity = documentFrequency.get(entityName) || 0;
|
|
5509
|
+
if (dfEntity === 0 || totalNotesScanned === 0) return 0;
|
|
5510
|
+
let bestNpmi = 0;
|
|
5377
5511
|
for (const matched of matchedEntities) {
|
|
5378
5512
|
const entityAssocs = associations[matched];
|
|
5379
|
-
if (entityAssocs)
|
|
5380
|
-
|
|
5381
|
-
|
|
5382
|
-
|
|
5383
|
-
|
|
5384
|
-
|
|
5385
|
-
}
|
|
5386
|
-
if (
|
|
5513
|
+
if (!entityAssocs) continue;
|
|
5514
|
+
const coocCount = entityAssocs.get(entityName) || 0;
|
|
5515
|
+
if (coocCount < minCount) continue;
|
|
5516
|
+
const dfSeed = documentFrequency.get(matched) || 0;
|
|
5517
|
+
const npmi = computeNpmi(coocCount, dfEntity, dfSeed, totalNotesScanned);
|
|
5518
|
+
bestNpmi = Math.max(bestNpmi, npmi);
|
|
5519
|
+
}
|
|
5520
|
+
if (bestNpmi === 0) return 0;
|
|
5521
|
+
let boost = bestNpmi * PMI_SCALE;
|
|
5522
|
+
if (recencyIndex2) {
|
|
5387
5523
|
const recencyBoostVal = getRecencyBoost(entityName, recencyIndex2);
|
|
5388
5524
|
const recencyMultiplier = recencyBoostVal > 0 ? 1.5 : 0.5;
|
|
5389
|
-
boost =
|
|
5525
|
+
boost = boost * recencyMultiplier;
|
|
5390
5526
|
}
|
|
5391
|
-
return Math.min(boost, MAX_COOCCURRENCE_BOOST);
|
|
5527
|
+
return Math.min(Math.round(boost), MAX_COOCCURRENCE_BOOST);
|
|
5528
|
+
}
|
|
5529
|
+
function tokenIdf(token, coocIndex) {
|
|
5530
|
+
if (!coocIndex || coocIndex.totalNotesScanned === 0) return 1;
|
|
5531
|
+
const df = coocIndex.documentFrequency.get(token);
|
|
5532
|
+
if (df === void 0) return 1;
|
|
5533
|
+
const N = coocIndex.totalNotesScanned;
|
|
5534
|
+
const rawIdf = Math.log((N + 1) / (df + 1));
|
|
5535
|
+
return Math.max(0.5, Math.min(2.5, rawIdf));
|
|
5392
5536
|
}
|
|
5393
5537
|
|
|
5394
5538
|
// src/core/write/edgeWeights.ts
|
|
@@ -5964,8 +6108,8 @@ function isLikelyArticleTitle(name) {
|
|
|
5964
6108
|
var STRICTNESS_CONFIGS = {
|
|
5965
6109
|
conservative: {
|
|
5966
6110
|
minWordLength: 3,
|
|
5967
|
-
minSuggestionScore:
|
|
5968
|
-
// Requires exact match (10) + at least one
|
|
6111
|
+
minSuggestionScore: 18,
|
|
6112
|
+
// Requires exact match (10) + stem (5) + at least one boost
|
|
5969
6113
|
minMatchRatio: 0.6,
|
|
5970
6114
|
// 60% of multi-word entity must match
|
|
5971
6115
|
requireMultipleMatches: true,
|
|
@@ -5977,8 +6121,8 @@ var STRICTNESS_CONFIGS = {
|
|
|
5977
6121
|
},
|
|
5978
6122
|
balanced: {
|
|
5979
6123
|
minWordLength: 3,
|
|
5980
|
-
minSuggestionScore:
|
|
5981
|
-
//
|
|
6124
|
+
minSuggestionScore: 10,
|
|
6125
|
+
// Exact match (10) or two stem matches
|
|
5982
6126
|
minMatchRatio: 0.4,
|
|
5983
6127
|
// 40% of multi-word entity must match
|
|
5984
6128
|
requireMultipleMatches: false,
|
|
@@ -6125,7 +6269,7 @@ function getAdaptiveMinScore(contentLength, baseScore) {
|
|
|
6125
6269
|
var MIN_SUGGESTION_SCORE = STRICTNESS_CONFIGS.balanced.minSuggestionScore;
|
|
6126
6270
|
var MIN_MATCH_RATIO = STRICTNESS_CONFIGS.balanced.minMatchRatio;
|
|
6127
6271
|
var FULL_ALIAS_MATCH_BONUS = 8;
|
|
6128
|
-
function scoreNameAgainstContent(name, contentTokens, contentStems, config) {
|
|
6272
|
+
function scoreNameAgainstContent(name, contentTokens, contentStems, config, coocIndex) {
|
|
6129
6273
|
const nameTokens = tokenize(name);
|
|
6130
6274
|
if (nameTokens.length === 0) {
|
|
6131
6275
|
return { score: 0, matchedWords: 0, exactMatches: 0, totalTokens: 0 };
|
|
@@ -6137,24 +6281,26 @@ function scoreNameAgainstContent(name, contentTokens, contentStems, config) {
|
|
|
6137
6281
|
for (let i = 0; i < nameTokens.length; i++) {
|
|
6138
6282
|
const token = nameTokens[i];
|
|
6139
6283
|
const nameStem = nameStems[i];
|
|
6284
|
+
const idfWeight = coocIndex ? tokenIdf(token, coocIndex) : 1;
|
|
6140
6285
|
if (contentTokens.has(token)) {
|
|
6141
|
-
score += config.exactMatchBonus;
|
|
6286
|
+
score += config.exactMatchBonus * idfWeight;
|
|
6142
6287
|
matchedWords++;
|
|
6143
6288
|
exactMatches++;
|
|
6144
6289
|
} else if (contentStems.has(nameStem)) {
|
|
6145
|
-
score += config.stemMatchBonus;
|
|
6290
|
+
score += config.stemMatchBonus * idfWeight;
|
|
6146
6291
|
matchedWords++;
|
|
6147
6292
|
}
|
|
6148
6293
|
}
|
|
6294
|
+
score = Math.round(score * 10) / 10;
|
|
6149
6295
|
return { score, matchedWords, exactMatches, totalTokens: nameTokens.length };
|
|
6150
6296
|
}
|
|
6151
|
-
function scoreEntity(entity, contentTokens, contentStems, config) {
|
|
6297
|
+
function scoreEntity(entity, contentTokens, contentStems, config, coocIndex) {
|
|
6152
6298
|
const entityName = getEntityName2(entity);
|
|
6153
6299
|
const aliases = getEntityAliases(entity);
|
|
6154
|
-
const nameResult = scoreNameAgainstContent(entityName, contentTokens, contentStems, config);
|
|
6300
|
+
const nameResult = scoreNameAgainstContent(entityName, contentTokens, contentStems, config, coocIndex);
|
|
6155
6301
|
let bestAliasResult = { score: 0, matchedWords: 0, exactMatches: 0, totalTokens: 0 };
|
|
6156
6302
|
for (const alias of aliases) {
|
|
6157
|
-
const aliasResult = scoreNameAgainstContent(alias, contentTokens, contentStems, config);
|
|
6303
|
+
const aliasResult = scoreNameAgainstContent(alias, contentTokens, contentStems, config, coocIndex);
|
|
6158
6304
|
if (aliasResult.score > bestAliasResult.score) {
|
|
6159
6305
|
bestAliasResult = aliasResult;
|
|
6160
6306
|
}
|
|
@@ -6164,7 +6310,7 @@ function scoreEntity(entity, contentTokens, contentStems, config) {
|
|
|
6164
6310
|
if (totalTokens === 0) return 0;
|
|
6165
6311
|
for (const alias of aliases) {
|
|
6166
6312
|
const aliasLower = alias.toLowerCase();
|
|
6167
|
-
if (aliasLower.length >=
|
|
6313
|
+
if (aliasLower.length >= 3 && !/\s/.test(aliasLower) && contentTokens.has(aliasLower)) {
|
|
6168
6314
|
score += FULL_ALIAS_MATCH_BONUS;
|
|
6169
6315
|
break;
|
|
6170
6316
|
}
|
|
@@ -6189,7 +6335,7 @@ function getEdgeWeightBoostScore(entityName, map) {
|
|
|
6189
6335
|
}
|
|
6190
6336
|
async function suggestRelatedLinks(content, options = {}) {
|
|
6191
6337
|
const {
|
|
6192
|
-
maxSuggestions =
|
|
6338
|
+
maxSuggestions = 8,
|
|
6193
6339
|
excludeLinked = true,
|
|
6194
6340
|
strictness = getEffectiveStrictness(options.notePath),
|
|
6195
6341
|
notePath,
|
|
@@ -6231,6 +6377,7 @@ async function suggestRelatedLinks(content, options = {}) {
|
|
|
6231
6377
|
const linkedEntities = excludeLinked ? extractLinkedEntities(content) : /* @__PURE__ */ new Set();
|
|
6232
6378
|
const noteFolder = notePath ? notePath.split("/")[0] : void 0;
|
|
6233
6379
|
const feedbackBoosts = moduleStateDb5 ? getAllFeedbackBoosts(moduleStateDb5, noteFolder) : /* @__PURE__ */ new Map();
|
|
6380
|
+
const suppressionPenalties = moduleStateDb5 ? getAllSuppressionPenalties(moduleStateDb5) : /* @__PURE__ */ new Map();
|
|
6234
6381
|
const edgeWeightMap = moduleStateDb5 ? getEntityEdgeWeightMap(moduleStateDb5) : /* @__PURE__ */ new Map();
|
|
6235
6382
|
const scoredEntities = [];
|
|
6236
6383
|
const directlyMatchedEntities = /* @__PURE__ */ new Set();
|
|
@@ -6247,13 +6394,7 @@ async function suggestRelatedLinks(content, options = {}) {
|
|
|
6247
6394
|
if (linkedEntities.has(entityName.toLowerCase())) {
|
|
6248
6395
|
continue;
|
|
6249
6396
|
}
|
|
6250
|
-
|
|
6251
|
-
const noteFolder2 = notePath ? notePath.split("/").slice(0, -1).join("/") : void 0;
|
|
6252
|
-
if (isSuppressed(moduleStateDb5, entityName, noteFolder2)) {
|
|
6253
|
-
continue;
|
|
6254
|
-
}
|
|
6255
|
-
}
|
|
6256
|
-
const contentScore = disabled.has("exact_match") && disabled.has("stem_match") ? 0 : scoreEntity(entity, contentTokens, contentStems, config);
|
|
6397
|
+
const contentScore = disabled.has("exact_match") && disabled.has("stem_match") ? 0 : scoreEntity(entity, contentTokens, contentStems, config, cooccurrenceIndex);
|
|
6257
6398
|
let score = contentScore;
|
|
6258
6399
|
if (contentScore > 0) {
|
|
6259
6400
|
entitiesWithContentMatch.add(entityName);
|
|
@@ -6272,10 +6413,12 @@ async function suggestRelatedLinks(content, options = {}) {
|
|
|
6272
6413
|
score += layerFeedbackAdj;
|
|
6273
6414
|
const layerEdgeWeightBoost = disabled.has("edge_weight") ? 0 : getEdgeWeightBoostScore(entityName, edgeWeightMap);
|
|
6274
6415
|
score += layerEdgeWeightBoost;
|
|
6275
|
-
if (
|
|
6416
|
+
if (contentScore > 0) {
|
|
6276
6417
|
directlyMatchedEntities.add(entityName);
|
|
6277
6418
|
}
|
|
6278
|
-
|
|
6419
|
+
const layerSuppressionPenalty = disabled.has("feedback") ? 0 : suppressionPenalties.get(entityName) ?? 0;
|
|
6420
|
+
score += layerSuppressionPenalty;
|
|
6421
|
+
if (contentScore > 0 && score >= adaptiveMinScore) {
|
|
6279
6422
|
scoredEntities.push({
|
|
6280
6423
|
name: entityName,
|
|
6281
6424
|
path: entity.path || "",
|
|
@@ -6290,23 +6433,31 @@ async function suggestRelatedLinks(content, options = {}) {
|
|
|
6290
6433
|
crossFolderBoost: layerCrossFolderBoost,
|
|
6291
6434
|
hubBoost: layerHubBoost,
|
|
6292
6435
|
feedbackAdjustment: layerFeedbackAdj,
|
|
6436
|
+
suppressionPenalty: layerSuppressionPenalty,
|
|
6293
6437
|
edgeWeightBoost: layerEdgeWeightBoost
|
|
6294
6438
|
}
|
|
6295
6439
|
});
|
|
6296
6440
|
}
|
|
6297
6441
|
}
|
|
6298
|
-
|
|
6442
|
+
const cooccurrenceSeeds = new Set(directlyMatchedEntities);
|
|
6443
|
+
if (linkedEntities.size > 0) {
|
|
6444
|
+
const lowerToDisplay = /* @__PURE__ */ new Map();
|
|
6445
|
+
for (const { entity } of entitiesWithTypes) {
|
|
6446
|
+
if (entity.name) lowerToDisplay.set(entity.name.toLowerCase(), entity.name);
|
|
6447
|
+
}
|
|
6448
|
+
for (const linked of linkedEntities) {
|
|
6449
|
+
const displayName = lowerToDisplay.get(linked);
|
|
6450
|
+
if (displayName) cooccurrenceSeeds.add(displayName);
|
|
6451
|
+
}
|
|
6452
|
+
}
|
|
6453
|
+
if (!disabled.has("cooccurrence") && cooccurrenceIndex && cooccurrenceSeeds.size > 0) {
|
|
6299
6454
|
for (const { entity, category } of entitiesWithTypes) {
|
|
6300
6455
|
const entityName = entity.name;
|
|
6301
6456
|
if (!entityName) continue;
|
|
6302
6457
|
if (!disabled.has("length_filter") && entityName.length > MAX_ENTITY_LENGTH) continue;
|
|
6303
6458
|
if (!disabled.has("article_filter") && isLikelyArticleTitle(entityName)) continue;
|
|
6304
6459
|
if (linkedEntities.has(entityName.toLowerCase())) continue;
|
|
6305
|
-
|
|
6306
|
-
const noteFolder2 = notePath ? notePath.split("/").slice(0, -1).join("/") : void 0;
|
|
6307
|
-
if (isSuppressed(moduleStateDb5, entityName, noteFolder2)) continue;
|
|
6308
|
-
}
|
|
6309
|
-
const boost = getCooccurrenceBoost(entityName, directlyMatchedEntities, cooccurrenceIndex, recencyIndex);
|
|
6460
|
+
const boost = getCooccurrenceBoost(entityName, cooccurrenceSeeds, cooccurrenceIndex, recencyIndex);
|
|
6310
6461
|
if (boost > 0) {
|
|
6311
6462
|
const existing = scoredEntities.find((e) => e.name === entityName);
|
|
6312
6463
|
if (existing) {
|
|
@@ -6317,19 +6468,24 @@ async function suggestRelatedLinks(content, options = {}) {
|
|
|
6317
6468
|
const hasContentOverlap = entityTokens.some(
|
|
6318
6469
|
(token) => contentTokens.has(token) || contentStems.has(stem(token))
|
|
6319
6470
|
);
|
|
6320
|
-
|
|
6471
|
+
const strongCooccurrence = boost >= 4;
|
|
6472
|
+
if (!hasContentOverlap && !strongCooccurrence) {
|
|
6321
6473
|
continue;
|
|
6322
6474
|
}
|
|
6323
|
-
|
|
6475
|
+
if (hasContentOverlap || strongCooccurrence) {
|
|
6476
|
+
entitiesWithContentMatch.add(entityName);
|
|
6477
|
+
}
|
|
6324
6478
|
const typeBoost = disabled.has("type_boost") ? 0 : TYPE_BOOST[category] || 0;
|
|
6325
6479
|
const contextBoost = disabled.has("context_boost") ? 0 : contextBoosts[category] || 0;
|
|
6326
|
-
const recencyBoostVal = disabled.has("recency") ?
|
|
6480
|
+
const recencyBoostVal = hasContentOverlap && !disabled.has("recency") ? recencyIndex ? getRecencyBoost(entityName, recencyIndex) : 0 : 0;
|
|
6327
6481
|
const crossFolderBoost = disabled.has("cross_folder") ? 0 : notePath && entity.path ? getCrossFolderBoost(entity.path, notePath) : 0;
|
|
6328
6482
|
const hubBoost = disabled.has("hub_boost") ? 0 : getHubBoost(entity);
|
|
6329
6483
|
const feedbackAdj = disabled.has("feedback") ? 0 : feedbackBoosts.get(entityName) ?? 0;
|
|
6330
6484
|
const edgeWeightBoost = disabled.has("edge_weight") ? 0 : getEdgeWeightBoostScore(entityName, edgeWeightMap);
|
|
6331
|
-
const
|
|
6332
|
-
|
|
6485
|
+
const suppPenalty = disabled.has("feedback") ? 0 : suppressionPenalties.get(entityName) ?? 0;
|
|
6486
|
+
const totalBoost = boost + typeBoost + contextBoost + recencyBoostVal + crossFolderBoost + hubBoost + feedbackAdj + edgeWeightBoost + suppPenalty;
|
|
6487
|
+
const effectiveMinScore = !hasContentOverlap ? Math.max(adaptiveMinScore, 10) : adaptiveMinScore;
|
|
6488
|
+
if (totalBoost >= effectiveMinScore) {
|
|
6333
6489
|
scoredEntities.push({
|
|
6334
6490
|
name: entityName,
|
|
6335
6491
|
path: entity.path || "",
|
|
@@ -6344,6 +6500,7 @@ async function suggestRelatedLinks(content, options = {}) {
|
|
|
6344
6500
|
crossFolderBoost,
|
|
6345
6501
|
hubBoost,
|
|
6346
6502
|
feedbackAdjustment: feedbackAdj,
|
|
6503
|
+
suppressionPenalty: suppPenalty,
|
|
6347
6504
|
edgeWeightBoost
|
|
6348
6505
|
}
|
|
6349
6506
|
});
|
|
@@ -6370,10 +6527,6 @@ async function suggestRelatedLinks(content, options = {}) {
|
|
|
6370
6527
|
existing.score += boost;
|
|
6371
6528
|
existing.breakdown.semanticBoost = boost;
|
|
6372
6529
|
} else if (!linkedEntities.has(match.entityName.toLowerCase())) {
|
|
6373
|
-
if (moduleStateDb5 && !disabled.has("feedback")) {
|
|
6374
|
-
const noteFolder2 = notePath ? notePath.split("/").slice(0, -1).join("/") : void 0;
|
|
6375
|
-
if (isSuppressed(moduleStateDb5, match.entityName, noteFolder2)) continue;
|
|
6376
|
-
}
|
|
6377
6530
|
const entityWithType = entitiesWithTypes.find(
|
|
6378
6531
|
(et) => et.entity.name === match.entityName
|
|
6379
6532
|
);
|
|
@@ -6387,7 +6540,8 @@ async function suggestRelatedLinks(content, options = {}) {
|
|
|
6387
6540
|
const layerCrossFolderBoost = disabled.has("cross_folder") ? 0 : notePath && entity.path ? getCrossFolderBoost(entity.path, notePath) : 0;
|
|
6388
6541
|
const layerFeedbackAdj = disabled.has("feedback") ? 0 : feedbackBoosts.get(match.entityName) ?? 0;
|
|
6389
6542
|
const layerEdgeWeightBoost = disabled.has("edge_weight") ? 0 : getEdgeWeightBoostScore(match.entityName, edgeWeightMap);
|
|
6390
|
-
const
|
|
6543
|
+
const layerSuppPenalty = disabled.has("feedback") ? 0 : suppressionPenalties.get(match.entityName) ?? 0;
|
|
6544
|
+
const totalScore = boost + layerTypeBoost + layerContextBoost + layerHubBoost + layerCrossFolderBoost + layerFeedbackAdj + layerEdgeWeightBoost + layerSuppPenalty;
|
|
6391
6545
|
if (totalScore >= adaptiveMinScore) {
|
|
6392
6546
|
scoredEntities.push({
|
|
6393
6547
|
name: match.entityName,
|
|
@@ -6403,6 +6557,7 @@ async function suggestRelatedLinks(content, options = {}) {
|
|
|
6403
6557
|
crossFolderBoost: layerCrossFolderBoost,
|
|
6404
6558
|
hubBoost: layerHubBoost,
|
|
6405
6559
|
feedbackAdjustment: layerFeedbackAdj,
|
|
6560
|
+
suppressionPenalty: layerSuppPenalty,
|
|
6406
6561
|
semanticBoost: boost,
|
|
6407
6562
|
edgeWeightBoost: layerEdgeWeightBoost
|
|
6408
6563
|
}
|
|
@@ -7226,7 +7381,7 @@ function refreshIfStale(vaultPath2, index, excludeTags) {
|
|
|
7226
7381
|
}
|
|
7227
7382
|
|
|
7228
7383
|
// src/index.ts
|
|
7229
|
-
import { openStateDb, scanVaultEntities as scanVaultEntities3, getSessionId, getAllEntitiesFromDb as getAllEntitiesFromDb3, findEntityMatches as findEntityMatches2, getProtectedZones as getProtectedZones2, rangeOverlapsProtectedZone } from "@velvetmonkey/vault-core";
|
|
7384
|
+
import { openStateDb, scanVaultEntities as scanVaultEntities3, getSessionId, getAllEntitiesFromDb as getAllEntitiesFromDb3, findEntityMatches as findEntityMatches2, getProtectedZones as getProtectedZones2, rangeOverlapsProtectedZone, detectImplicitEntities as detectImplicitEntities3 } from "@velvetmonkey/vault-core";
|
|
7230
7385
|
|
|
7231
7386
|
// src/tools/read/graph.ts
|
|
7232
7387
|
import * as fs9 from "fs";
|
|
@@ -7754,6 +7909,7 @@ function registerGraphTools(server2, getIndex, getVaultPath, getStateDb) {
|
|
|
7754
7909
|
|
|
7755
7910
|
// src/tools/read/wikilinks.ts
|
|
7756
7911
|
import { z as z2 } from "zod";
|
|
7912
|
+
import { detectImplicitEntities as detectImplicitEntities2 } from "@velvetmonkey/vault-core";
|
|
7757
7913
|
function findEntityMatches(text, entities) {
|
|
7758
7914
|
const matches = [];
|
|
7759
7915
|
const sortedEntities = Array.from(entities.entries()).filter(([name]) => name.length >= 2).sort((a, b) => b[0].length - a[0].length);
|
|
@@ -7868,12 +8024,69 @@ function registerWikilinkTools(server2, getIndex, getVaultPath) {
|
|
|
7868
8024
|
const index = getIndex();
|
|
7869
8025
|
const allMatches = findEntityMatches(text, index.entities);
|
|
7870
8026
|
const matches = allMatches.slice(offset, offset + limit);
|
|
8027
|
+
const linkedSet = new Set(allMatches.map((m) => m.entity.toLowerCase()));
|
|
8028
|
+
const prospects = [];
|
|
8029
|
+
const prospectSeen = /* @__PURE__ */ new Set();
|
|
8030
|
+
for (const [target, links] of index.backlinks) {
|
|
8031
|
+
if (links.length < 2) continue;
|
|
8032
|
+
if (index.entities.has(target.toLowerCase())) continue;
|
|
8033
|
+
if (linkedSet.has(target.toLowerCase())) continue;
|
|
8034
|
+
const targetLower = target.toLowerCase();
|
|
8035
|
+
const textLower = text.toLowerCase();
|
|
8036
|
+
let searchPos = 0;
|
|
8037
|
+
while (searchPos < textLower.length) {
|
|
8038
|
+
const pos = textLower.indexOf(targetLower, searchPos);
|
|
8039
|
+
if (pos === -1) break;
|
|
8040
|
+
const end = pos + target.length;
|
|
8041
|
+
const before = pos > 0 ? text[pos - 1] : " ";
|
|
8042
|
+
const after = end < text.length ? text[end] : " ";
|
|
8043
|
+
if (/[\s\n\r.,;:!?()[\]{}'"<>-]/.test(before) && /[\s\n\r.,;:!?()[\]{}'"<>-]/.test(after)) {
|
|
8044
|
+
if (!prospectSeen.has(targetLower)) {
|
|
8045
|
+
prospectSeen.add(targetLower);
|
|
8046
|
+
prospects.push({
|
|
8047
|
+
entity: text.substring(pos, end),
|
|
8048
|
+
start: pos,
|
|
8049
|
+
end,
|
|
8050
|
+
source: "dead_link",
|
|
8051
|
+
confidence: links.length >= 3 ? "high" : "medium",
|
|
8052
|
+
backlink_count: links.length
|
|
8053
|
+
});
|
|
8054
|
+
}
|
|
8055
|
+
break;
|
|
8056
|
+
}
|
|
8057
|
+
searchPos = pos + 1;
|
|
8058
|
+
}
|
|
8059
|
+
}
|
|
8060
|
+
const implicit = detectImplicitEntities2(text);
|
|
8061
|
+
for (const imp of implicit) {
|
|
8062
|
+
const impLower = imp.text.toLowerCase();
|
|
8063
|
+
if (linkedSet.has(impLower)) continue;
|
|
8064
|
+
if (prospectSeen.has(impLower)) {
|
|
8065
|
+
const existing = prospects.find((p) => p.entity.toLowerCase() === impLower);
|
|
8066
|
+
if (existing) {
|
|
8067
|
+
existing.source = "both";
|
|
8068
|
+
existing.confidence = "high";
|
|
8069
|
+
}
|
|
8070
|
+
continue;
|
|
8071
|
+
}
|
|
8072
|
+
prospectSeen.add(impLower);
|
|
8073
|
+
prospects.push({
|
|
8074
|
+
entity: imp.text,
|
|
8075
|
+
start: imp.start,
|
|
8076
|
+
end: imp.end,
|
|
8077
|
+
source: "implicit",
|
|
8078
|
+
confidence: "low"
|
|
8079
|
+
});
|
|
8080
|
+
}
|
|
7871
8081
|
const output = {
|
|
7872
8082
|
input_length: text.length,
|
|
7873
8083
|
suggestion_count: allMatches.length,
|
|
7874
8084
|
returned_count: matches.length,
|
|
7875
8085
|
suggestions: matches
|
|
7876
8086
|
};
|
|
8087
|
+
if (prospects.length > 0) {
|
|
8088
|
+
output.prospects = prospects;
|
|
8089
|
+
}
|
|
7877
8090
|
if (detail) {
|
|
7878
8091
|
const scored = await suggestRelatedLinks(text, {
|
|
7879
8092
|
detail: true,
|
|
@@ -12901,7 +13114,7 @@ Example: vault_add_to_section({ path: "daily/2026-02-15.md", section: "Log", con
|
|
|
12901
13114
|
preserveListNesting: z11.boolean().default(true).describe("Detect and preserve the indentation level of surrounding list items. Set false to disable."),
|
|
12902
13115
|
bumpHeadings: z11.boolean().default(true).describe("Auto-bump heading levels in inserted content so they nest under the target section (e.g., ## in a ## section becomes ###). Set false to disable."),
|
|
12903
13116
|
suggestOutgoingLinks: z11.boolean().default(true).describe('Append suggested outgoing wikilinks based on content (e.g., "\u2192 [[AI]], [[Philosophy]]"). Set false to disable.'),
|
|
12904
|
-
maxSuggestions: z11.number().min(1).max(10).default(
|
|
13117
|
+
maxSuggestions: z11.number().min(1).max(10).default(5).describe("Maximum number of suggested wikilinks to append (1-10, default: 5)"),
|
|
12905
13118
|
validate: z11.boolean().default(true).describe("Check input for common issues (double timestamps, non-markdown bullets, etc.)"),
|
|
12906
13119
|
normalize: z11.boolean().default(true).describe("Auto-fix common issues before formatting (replace \u2022 with -, trim excessive whitespace, etc.)"),
|
|
12907
13120
|
guardrails: z11.enum(["warn", "strict", "off"]).default("warn").describe('Output validation mode: "warn" returns issues but proceeds, "strict" blocks on errors, "off" disables'),
|
|
@@ -13047,7 +13260,7 @@ Example: vault_add_to_section({ path: "daily/2026-02-15.md", section: "Log", con
|
|
|
13047
13260
|
commit: z11.boolean().default(false).describe("If true, commit this change to git (creates undo point)"),
|
|
13048
13261
|
skipWikilinks: z11.boolean().default(false).describe("If true, skip auto-wikilink application on replacement text"),
|
|
13049
13262
|
suggestOutgoingLinks: z11.boolean().default(true).describe('Append suggested outgoing wikilinks based on content (e.g., "\u2192 [[AI]], [[Philosophy]]"). Set false to disable.'),
|
|
13050
|
-
maxSuggestions: z11.number().min(1).max(10).default(
|
|
13263
|
+
maxSuggestions: z11.number().min(1).max(10).default(5).describe("Maximum number of suggested wikilinks to append (1-10, default: 5)"),
|
|
13051
13264
|
validate: z11.boolean().default(true).describe("Check input for common issues (double timestamps, non-markdown bullets, etc.)"),
|
|
13052
13265
|
normalize: z11.boolean().default(true).describe("Auto-fix common issues before formatting (replace \u2022 with -, trim excessive whitespace, etc.)"),
|
|
13053
13266
|
guardrails: z11.enum(["warn", "strict", "off"]).default("warn").describe('Output validation mode: "warn" returns issues but proceeds, "strict" blocks on errors, "off" disables'),
|
|
@@ -13245,7 +13458,7 @@ function registerTaskTools(server2, vaultPath2) {
|
|
|
13245
13458
|
commit: z12.boolean().default(false).describe("If true, commit this change to git (creates undo point)"),
|
|
13246
13459
|
skipWikilinks: z12.boolean().default(false).describe("If true, skip auto-wikilink application (wikilinks are applied by default)"),
|
|
13247
13460
|
suggestOutgoingLinks: z12.boolean().default(true).describe('Append suggested outgoing wikilinks based on content (e.g., "\u2192 [[AI]], [[Philosophy]]"). Set false to disable.'),
|
|
13248
|
-
maxSuggestions: z12.number().min(1).max(10).default(
|
|
13461
|
+
maxSuggestions: z12.number().min(1).max(10).default(5).describe("Maximum number of suggested wikilinks to append (1-10, default: 5)"),
|
|
13249
13462
|
preserveListNesting: z12.boolean().default(true).describe("Preserve indentation when inserting into nested lists. Default: true"),
|
|
13250
13463
|
validate: z12.boolean().default(true).describe("Check input for common issues"),
|
|
13251
13464
|
normalize: z12.boolean().default(true).describe("Auto-fix common issues before formatting"),
|
|
@@ -13386,7 +13599,7 @@ function registerNoteTools(server2, vaultPath2, getIndex) {
|
|
|
13386
13599
|
commit: z14.boolean().default(false).describe("If true, commit this change to git (creates undo point)"),
|
|
13387
13600
|
skipWikilinks: z14.boolean().default(false).describe("If true, skip auto-wikilink application (wikilinks are applied by default)"),
|
|
13388
13601
|
suggestOutgoingLinks: z14.boolean().default(true).describe('Append suggested outgoing wikilinks based on content (e.g., "\u2192 [[AI]], [[Philosophy]]").'),
|
|
13389
|
-
maxSuggestions: z14.number().min(1).max(10).default(
|
|
13602
|
+
maxSuggestions: z14.number().min(1).max(10).default(5).describe("Maximum number of suggested wikilinks to append (1-10, default: 5)"),
|
|
13390
13603
|
agent_id: z14.string().optional().describe("Agent identifier for multi-agent scoping"),
|
|
13391
13604
|
session_id: z14.string().optional().describe("Session identifier for conversation scoping")
|
|
13392
13605
|
},
|
|
@@ -18442,7 +18655,7 @@ async function runPostIndexWork(index) {
|
|
|
18442
18655
|
(e) => e.nameLower === link || (e.aliases ?? []).some((a) => a.toLowerCase() === link)
|
|
18443
18656
|
);
|
|
18444
18657
|
if (entity) {
|
|
18445
|
-
recordFeedback(stateDb, entity.name, "implicit:kept", entry.file, true);
|
|
18658
|
+
recordFeedback(stateDb, entity.name, "implicit:kept", entry.file, true, 0.8);
|
|
18446
18659
|
markPositive.run(entry.file, link);
|
|
18447
18660
|
}
|
|
18448
18661
|
}
|
|
@@ -18468,14 +18681,22 @@ async function runPostIndexWork(index) {
|
|
|
18468
18681
|
}
|
|
18469
18682
|
}
|
|
18470
18683
|
}
|
|
18684
|
+
const newDeadLinks = [];
|
|
18685
|
+
for (const diff of linkDiffs) {
|
|
18686
|
+
const newDead = diff.added.filter((target) => !vaultIndex.entities.has(target.toLowerCase()));
|
|
18687
|
+
if (newDead.length > 0) {
|
|
18688
|
+
newDeadLinks.push({ file: diff.file, targets: newDead });
|
|
18689
|
+
}
|
|
18690
|
+
}
|
|
18471
18691
|
tracker.end({
|
|
18472
18692
|
total_resolved: totalResolved,
|
|
18473
18693
|
total_dead: totalDead,
|
|
18474
18694
|
links: forwardLinkResults,
|
|
18475
18695
|
link_diffs: linkDiffs,
|
|
18476
|
-
survived: survivedLinks
|
|
18696
|
+
survived: survivedLinks,
|
|
18697
|
+
new_dead_links: newDeadLinks
|
|
18477
18698
|
});
|
|
18478
|
-
serverLog("watcher", `Forward links: ${totalResolved} resolved, ${totalDead} dead`);
|
|
18699
|
+
serverLog("watcher", `Forward links: ${totalResolved} resolved, ${totalDead} dead${newDeadLinks.length > 0 ? `, ${newDeadLinks.reduce((s, d) => s + d.targets.length, 0)} new dead` : ""}`);
|
|
18479
18700
|
tracker.start("wikilink_check", { files: filteredEvents.length });
|
|
18480
18701
|
const trackedLinks = [];
|
|
18481
18702
|
if (stateDb) {
|
|
@@ -18539,6 +18760,7 @@ async function runPostIndexWork(index) {
|
|
|
18539
18760
|
tracker.end({ tracked: trackedLinks, mentions: mentionResults });
|
|
18540
18761
|
serverLog("watcher", `Wikilink check: ${trackedLinks.reduce((s, t) => s + t.entities.length, 0)} tracked links in ${trackedLinks.length} files, ${mentionResults.reduce((s, m) => s + m.entities.length, 0)} unwikified mentions`);
|
|
18541
18762
|
tracker.start("implicit_feedback", { files: filteredEvents.length });
|
|
18763
|
+
const preSuppressed = stateDb ? new Set(getAllSuppressionPenalties(stateDb).keys()) : /* @__PURE__ */ new Set();
|
|
18542
18764
|
const feedbackResults = [];
|
|
18543
18765
|
if (stateDb) {
|
|
18544
18766
|
for (const event of filteredEvents) {
|
|
@@ -18583,10 +18805,84 @@ async function runPostIndexWork(index) {
|
|
|
18583
18805
|
}
|
|
18584
18806
|
}
|
|
18585
18807
|
}
|
|
18586
|
-
|
|
18808
|
+
const newlySuppressed = [];
|
|
18809
|
+
if (stateDb) {
|
|
18810
|
+
const postSuppressed = getAllSuppressionPenalties(stateDb);
|
|
18811
|
+
for (const entity of postSuppressed.keys()) {
|
|
18812
|
+
if (!preSuppressed.has(entity)) {
|
|
18813
|
+
newlySuppressed.push(entity);
|
|
18814
|
+
}
|
|
18815
|
+
}
|
|
18816
|
+
}
|
|
18817
|
+
tracker.end({ removals: feedbackResults, additions: additionResults, newly_suppressed: newlySuppressed });
|
|
18587
18818
|
if (feedbackResults.length > 0 || additionResults.length > 0) {
|
|
18588
18819
|
serverLog("watcher", `Implicit feedback: ${feedbackResults.length} removals, ${additionResults.length} manual additions detected`);
|
|
18589
18820
|
}
|
|
18821
|
+
if (newlySuppressed.length > 0) {
|
|
18822
|
+
serverLog("watcher", `Suppression: ${newlySuppressed.length} entities newly suppressed: ${newlySuppressed.join(", ")}`);
|
|
18823
|
+
}
|
|
18824
|
+
tracker.start("prospect_scan", { files: filteredEvents.length });
|
|
18825
|
+
const prospectResults = [];
|
|
18826
|
+
for (const event of filteredEvents) {
|
|
18827
|
+
if (event.type === "delete" || !event.path.endsWith(".md")) continue;
|
|
18828
|
+
try {
|
|
18829
|
+
const content = await fs31.readFile(path32.join(vaultPath, event.path), "utf-8");
|
|
18830
|
+
const zones = getProtectedZones2(content);
|
|
18831
|
+
const linkedSet = new Set(
|
|
18832
|
+
(forwardLinkResults.find((r) => r.file === event.path)?.resolved ?? []).concat(forwardLinkResults.find((r) => r.file === event.path)?.dead ?? []).map((n) => n.toLowerCase())
|
|
18833
|
+
);
|
|
18834
|
+
const knownEntitySet = new Set(entitiesAfter.map((e) => e.nameLower));
|
|
18835
|
+
const implicitMatches = detectImplicitEntities3(content);
|
|
18836
|
+
const implicitNames = implicitMatches.filter((imp) => !linkedSet.has(imp.text.toLowerCase()) && !knownEntitySet.has(imp.text.toLowerCase())).map((imp) => imp.text);
|
|
18837
|
+
const deadLinkMatches = [];
|
|
18838
|
+
for (const [key, links] of vaultIndex.backlinks) {
|
|
18839
|
+
if (links.length < 2 || vaultIndex.entities.has(key) || linkedSet.has(key)) continue;
|
|
18840
|
+
const matches = findEntityMatches2(content, key, true);
|
|
18841
|
+
if (matches.some((m) => !rangeOverlapsProtectedZone(m.start, m.end, zones))) {
|
|
18842
|
+
deadLinkMatches.push(key);
|
|
18843
|
+
}
|
|
18844
|
+
}
|
|
18845
|
+
if (implicitNames.length > 0 || deadLinkMatches.length > 0) {
|
|
18846
|
+
prospectResults.push({ file: event.path, implicit: implicitNames, deadLinkMatches });
|
|
18847
|
+
}
|
|
18848
|
+
} catch {
|
|
18849
|
+
}
|
|
18850
|
+
}
|
|
18851
|
+
tracker.end({ prospects: prospectResults });
|
|
18852
|
+
if (prospectResults.length > 0) {
|
|
18853
|
+
const implicitCount = prospectResults.reduce((s, p) => s + p.implicit.length, 0);
|
|
18854
|
+
const deadCount = prospectResults.reduce((s, p) => s + p.deadLinkMatches.length, 0);
|
|
18855
|
+
serverLog("watcher", `Prospect scan: ${implicitCount} implicit entities, ${deadCount} dead link matches across ${prospectResults.length} files`);
|
|
18856
|
+
}
|
|
18857
|
+
tracker.start("suggestion_scoring", { files: filteredEvents.length });
|
|
18858
|
+
const suggestionResults = [];
|
|
18859
|
+
for (const event of filteredEvents) {
|
|
18860
|
+
if (event.type === "delete" || !event.path.endsWith(".md")) continue;
|
|
18861
|
+
try {
|
|
18862
|
+
const content = await fs31.readFile(path32.join(vaultPath, event.path), "utf-8");
|
|
18863
|
+
const result = await suggestRelatedLinks(content, {
|
|
18864
|
+
maxSuggestions: 5,
|
|
18865
|
+
strictness: "balanced",
|
|
18866
|
+
notePath: event.path,
|
|
18867
|
+
detail: true
|
|
18868
|
+
});
|
|
18869
|
+
if (result.detailed && result.detailed.length > 0) {
|
|
18870
|
+
suggestionResults.push({
|
|
18871
|
+
file: event.path,
|
|
18872
|
+
top: result.detailed.slice(0, 5).map((s) => ({
|
|
18873
|
+
entity: s.entity,
|
|
18874
|
+
score: s.totalScore,
|
|
18875
|
+
confidence: s.confidence
|
|
18876
|
+
}))
|
|
18877
|
+
});
|
|
18878
|
+
}
|
|
18879
|
+
} catch {
|
|
18880
|
+
}
|
|
18881
|
+
}
|
|
18882
|
+
tracker.end({ scored_files: suggestionResults.length, suggestions: suggestionResults });
|
|
18883
|
+
if (suggestionResults.length > 0) {
|
|
18884
|
+
serverLog("watcher", `Suggestion scoring: ${suggestionResults.length} files scored`);
|
|
18885
|
+
}
|
|
18590
18886
|
tracker.start("tag_scan", { files: filteredEvents.length });
|
|
18591
18887
|
const tagDiffs = [];
|
|
18592
18888
|
if (stateDb) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@velvetmonkey/flywheel-memory",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.50",
|
|
4
4
|
"description": "MCP server that gives Claude full read/write access to your Obsidian vault. Select from 42 tools for search, backlinks, graph queries, mutations, and hybrid semantic search.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
},
|
|
53
53
|
"dependencies": {
|
|
54
54
|
"@modelcontextprotocol/sdk": "^1.25.1",
|
|
55
|
-
"@velvetmonkey/vault-core": "^2.0.
|
|
55
|
+
"@velvetmonkey/vault-core": "^2.0.50",
|
|
56
56
|
"better-sqlite3": "^11.0.0",
|
|
57
57
|
"chokidar": "^4.0.0",
|
|
58
58
|
"gray-matter": "^4.0.3",
|