@velvetmonkey/flywheel-crank 0.5.1 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +623 -122
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -492,23 +492,525 @@ import {
|
|
|
492
492
|
loadEntityCache,
|
|
493
493
|
saveEntityCache
|
|
494
494
|
} from "@velvetmonkey/vault-core";
|
|
495
|
+
import path4 from "path";
|
|
496
|
+
|
|
497
|
+
// src/core/stemmer.ts
|
|
498
|
+
var STOPWORDS = /* @__PURE__ */ new Set([
|
|
499
|
+
"the",
|
|
500
|
+
"a",
|
|
501
|
+
"an",
|
|
502
|
+
"and",
|
|
503
|
+
"or",
|
|
504
|
+
"but",
|
|
505
|
+
"in",
|
|
506
|
+
"on",
|
|
507
|
+
"at",
|
|
508
|
+
"to",
|
|
509
|
+
"for",
|
|
510
|
+
"of",
|
|
511
|
+
"with",
|
|
512
|
+
"by",
|
|
513
|
+
"from",
|
|
514
|
+
"as",
|
|
515
|
+
"is",
|
|
516
|
+
"was",
|
|
517
|
+
"are",
|
|
518
|
+
"were",
|
|
519
|
+
"been",
|
|
520
|
+
"be",
|
|
521
|
+
"have",
|
|
522
|
+
"has",
|
|
523
|
+
"had",
|
|
524
|
+
"do",
|
|
525
|
+
"does",
|
|
526
|
+
"did",
|
|
527
|
+
"will",
|
|
528
|
+
"would",
|
|
529
|
+
"could",
|
|
530
|
+
"should",
|
|
531
|
+
"may",
|
|
532
|
+
"might",
|
|
533
|
+
"must",
|
|
534
|
+
"shall",
|
|
535
|
+
"can",
|
|
536
|
+
"need",
|
|
537
|
+
"this",
|
|
538
|
+
"that",
|
|
539
|
+
"these",
|
|
540
|
+
"those",
|
|
541
|
+
"i",
|
|
542
|
+
"you",
|
|
543
|
+
"he",
|
|
544
|
+
"she",
|
|
545
|
+
"it",
|
|
546
|
+
"we",
|
|
547
|
+
"they",
|
|
548
|
+
"what",
|
|
549
|
+
"which",
|
|
550
|
+
"who",
|
|
551
|
+
"whom",
|
|
552
|
+
"when",
|
|
553
|
+
"where",
|
|
554
|
+
"why",
|
|
555
|
+
"how",
|
|
556
|
+
"all",
|
|
557
|
+
"each",
|
|
558
|
+
"every",
|
|
559
|
+
"both",
|
|
560
|
+
"few",
|
|
561
|
+
"more",
|
|
562
|
+
"most",
|
|
563
|
+
"other",
|
|
564
|
+
"some",
|
|
565
|
+
"such",
|
|
566
|
+
"no",
|
|
567
|
+
"not",
|
|
568
|
+
"only",
|
|
569
|
+
"own",
|
|
570
|
+
"same",
|
|
571
|
+
"so",
|
|
572
|
+
"than",
|
|
573
|
+
"too",
|
|
574
|
+
"very",
|
|
575
|
+
"just",
|
|
576
|
+
"also",
|
|
577
|
+
"about",
|
|
578
|
+
"after",
|
|
579
|
+
"before",
|
|
580
|
+
"being",
|
|
581
|
+
"between",
|
|
582
|
+
"into",
|
|
583
|
+
"through",
|
|
584
|
+
"during",
|
|
585
|
+
"above",
|
|
586
|
+
"below",
|
|
587
|
+
"out",
|
|
588
|
+
"off",
|
|
589
|
+
"over",
|
|
590
|
+
"under",
|
|
591
|
+
"again",
|
|
592
|
+
"further",
|
|
593
|
+
"then",
|
|
594
|
+
"once",
|
|
595
|
+
"here",
|
|
596
|
+
"there",
|
|
597
|
+
"any",
|
|
598
|
+
"now",
|
|
599
|
+
"new",
|
|
600
|
+
"even",
|
|
601
|
+
"much",
|
|
602
|
+
"back",
|
|
603
|
+
"going"
|
|
604
|
+
]);
|
|
605
|
+
function isConsonant(word, i) {
|
|
606
|
+
const c = word[i];
|
|
607
|
+
if (c === "a" || c === "e" || c === "i" || c === "o" || c === "u") {
|
|
608
|
+
return false;
|
|
609
|
+
}
|
|
610
|
+
if (c === "y") {
|
|
611
|
+
return i === 0 || !isConsonant(word, i - 1);
|
|
612
|
+
}
|
|
613
|
+
return true;
|
|
614
|
+
}
|
|
615
|
+
function measure(word, end) {
|
|
616
|
+
let n = 0;
|
|
617
|
+
let i = 0;
|
|
618
|
+
while (i <= end) {
|
|
619
|
+
if (!isConsonant(word, i))
|
|
620
|
+
break;
|
|
621
|
+
i++;
|
|
622
|
+
}
|
|
623
|
+
if (i > end)
|
|
624
|
+
return n;
|
|
625
|
+
i++;
|
|
626
|
+
while (true) {
|
|
627
|
+
while (i <= end) {
|
|
628
|
+
if (isConsonant(word, i))
|
|
629
|
+
break;
|
|
630
|
+
i++;
|
|
631
|
+
}
|
|
632
|
+
if (i > end)
|
|
633
|
+
return n;
|
|
634
|
+
n++;
|
|
635
|
+
i++;
|
|
636
|
+
while (i <= end) {
|
|
637
|
+
if (!isConsonant(word, i))
|
|
638
|
+
break;
|
|
639
|
+
i++;
|
|
640
|
+
}
|
|
641
|
+
if (i > end)
|
|
642
|
+
return n;
|
|
643
|
+
i++;
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
function hasVowel(word, end) {
|
|
647
|
+
for (let i = 0; i <= end; i++) {
|
|
648
|
+
if (!isConsonant(word, i))
|
|
649
|
+
return true;
|
|
650
|
+
}
|
|
651
|
+
return false;
|
|
652
|
+
}
|
|
653
|
+
function endsWithDoubleConsonant(word, end) {
|
|
654
|
+
if (end < 1)
|
|
655
|
+
return false;
|
|
656
|
+
if (word[end] !== word[end - 1])
|
|
657
|
+
return false;
|
|
658
|
+
return isConsonant(word, end);
|
|
659
|
+
}
|
|
660
|
+
function cvcPattern(word, i) {
|
|
661
|
+
if (i < 2)
|
|
662
|
+
return false;
|
|
663
|
+
if (!isConsonant(word, i) || isConsonant(word, i - 1) || !isConsonant(word, i - 2)) {
|
|
664
|
+
return false;
|
|
665
|
+
}
|
|
666
|
+
const c = word[i];
|
|
667
|
+
return c !== "w" && c !== "x" && c !== "y";
|
|
668
|
+
}
|
|
669
|
+
function replaceSuffix(word, suffix, replacement, minMeasure) {
|
|
670
|
+
if (!word.endsWith(suffix))
|
|
671
|
+
return word;
|
|
672
|
+
const stem2 = word.slice(0, word.length - suffix.length);
|
|
673
|
+
if (measure(stem2, stem2.length - 1) > minMeasure) {
|
|
674
|
+
return stem2 + replacement;
|
|
675
|
+
}
|
|
676
|
+
return word;
|
|
677
|
+
}
|
|
678
|
+
function step1a(word) {
|
|
679
|
+
if (word.endsWith("sses")) {
|
|
680
|
+
return word.slice(0, -2);
|
|
681
|
+
}
|
|
682
|
+
if (word.endsWith("ies")) {
|
|
683
|
+
return word.slice(0, -2);
|
|
684
|
+
}
|
|
685
|
+
if (word.endsWith("ss")) {
|
|
686
|
+
return word;
|
|
687
|
+
}
|
|
688
|
+
if (word.endsWith("s")) {
|
|
689
|
+
return word.slice(0, -1);
|
|
690
|
+
}
|
|
691
|
+
return word;
|
|
692
|
+
}
|
|
693
|
+
function step1b(word) {
|
|
694
|
+
if (word.endsWith("eed")) {
|
|
695
|
+
const stem3 = word.slice(0, -3);
|
|
696
|
+
if (measure(stem3, stem3.length - 1) > 0) {
|
|
697
|
+
return stem3 + "ee";
|
|
698
|
+
}
|
|
699
|
+
return word;
|
|
700
|
+
}
|
|
701
|
+
let stem2 = "";
|
|
702
|
+
let didRemove = false;
|
|
703
|
+
if (word.endsWith("ed")) {
|
|
704
|
+
stem2 = word.slice(0, -2);
|
|
705
|
+
if (hasVowel(stem2, stem2.length - 1)) {
|
|
706
|
+
word = stem2;
|
|
707
|
+
didRemove = true;
|
|
708
|
+
}
|
|
709
|
+
} else if (word.endsWith("ing")) {
|
|
710
|
+
stem2 = word.slice(0, -3);
|
|
711
|
+
if (hasVowel(stem2, stem2.length - 1)) {
|
|
712
|
+
word = stem2;
|
|
713
|
+
didRemove = true;
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
if (didRemove) {
|
|
717
|
+
if (word.endsWith("at") || word.endsWith("bl") || word.endsWith("iz")) {
|
|
718
|
+
return word + "e";
|
|
719
|
+
}
|
|
720
|
+
if (endsWithDoubleConsonant(word, word.length - 1)) {
|
|
721
|
+
const c = word[word.length - 1];
|
|
722
|
+
if (c !== "l" && c !== "s" && c !== "z") {
|
|
723
|
+
return word.slice(0, -1);
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
if (measure(word, word.length - 1) === 1 && cvcPattern(word, word.length - 1)) {
|
|
727
|
+
return word + "e";
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
return word;
|
|
731
|
+
}
|
|
732
|
+
function step1c(word) {
|
|
733
|
+
if (word.endsWith("y")) {
|
|
734
|
+
const stem2 = word.slice(0, -1);
|
|
735
|
+
if (hasVowel(stem2, stem2.length - 1)) {
|
|
736
|
+
return stem2 + "i";
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
return word;
|
|
740
|
+
}
|
|
741
|
+
function step2(word) {
|
|
742
|
+
const suffixes = [
|
|
743
|
+
["ational", "ate"],
|
|
744
|
+
["tional", "tion"],
|
|
745
|
+
["enci", "ence"],
|
|
746
|
+
["anci", "ance"],
|
|
747
|
+
["izer", "ize"],
|
|
748
|
+
["abli", "able"],
|
|
749
|
+
["alli", "al"],
|
|
750
|
+
["entli", "ent"],
|
|
751
|
+
["eli", "e"],
|
|
752
|
+
["ousli", "ous"],
|
|
753
|
+
["ization", "ize"],
|
|
754
|
+
["ation", "ate"],
|
|
755
|
+
["ator", "ate"],
|
|
756
|
+
["alism", "al"],
|
|
757
|
+
["iveness", "ive"],
|
|
758
|
+
["fulness", "ful"],
|
|
759
|
+
["ousness", "ous"],
|
|
760
|
+
["aliti", "al"],
|
|
761
|
+
["iviti", "ive"],
|
|
762
|
+
["biliti", "ble"]
|
|
763
|
+
];
|
|
764
|
+
for (const [suffix, replacement] of suffixes) {
|
|
765
|
+
if (word.endsWith(suffix)) {
|
|
766
|
+
return replaceSuffix(word, suffix, replacement, 0);
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
return word;
|
|
770
|
+
}
|
|
771
|
+
function step3(word) {
|
|
772
|
+
const suffixes = [
|
|
773
|
+
["icate", "ic"],
|
|
774
|
+
["ative", ""],
|
|
775
|
+
["alize", "al"],
|
|
776
|
+
["iciti", "ic"],
|
|
777
|
+
["ical", "ic"],
|
|
778
|
+
["ful", ""],
|
|
779
|
+
["ness", ""]
|
|
780
|
+
];
|
|
781
|
+
for (const [suffix, replacement] of suffixes) {
|
|
782
|
+
if (word.endsWith(suffix)) {
|
|
783
|
+
return replaceSuffix(word, suffix, replacement, 0);
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
return word;
|
|
787
|
+
}
|
|
788
|
+
function step4(word) {
|
|
789
|
+
const suffixes = [
|
|
790
|
+
"al",
|
|
791
|
+
"ance",
|
|
792
|
+
"ence",
|
|
793
|
+
"er",
|
|
794
|
+
"ic",
|
|
795
|
+
"able",
|
|
796
|
+
"ible",
|
|
797
|
+
"ant",
|
|
798
|
+
"ement",
|
|
799
|
+
"ment",
|
|
800
|
+
"ent",
|
|
801
|
+
"ion",
|
|
802
|
+
"ou",
|
|
803
|
+
"ism",
|
|
804
|
+
"ate",
|
|
805
|
+
"iti",
|
|
806
|
+
"ous",
|
|
807
|
+
"ive",
|
|
808
|
+
"ize"
|
|
809
|
+
];
|
|
810
|
+
for (const suffix of suffixes) {
|
|
811
|
+
if (word.endsWith(suffix)) {
|
|
812
|
+
const stem2 = word.slice(0, word.length - suffix.length);
|
|
813
|
+
if (measure(stem2, stem2.length - 1) > 1) {
|
|
814
|
+
if (suffix === "ion") {
|
|
815
|
+
const lastChar = stem2[stem2.length - 1];
|
|
816
|
+
if (lastChar === "s" || lastChar === "t") {
|
|
817
|
+
return stem2;
|
|
818
|
+
}
|
|
819
|
+
} else {
|
|
820
|
+
return stem2;
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
return word;
|
|
826
|
+
}
|
|
827
|
+
function step5a(word) {
|
|
828
|
+
if (word.endsWith("e")) {
|
|
829
|
+
const stem2 = word.slice(0, -1);
|
|
830
|
+
const m = measure(stem2, stem2.length - 1);
|
|
831
|
+
if (m > 1) {
|
|
832
|
+
return stem2;
|
|
833
|
+
}
|
|
834
|
+
if (m === 1 && !cvcPattern(stem2, stem2.length - 1)) {
|
|
835
|
+
return stem2;
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
return word;
|
|
839
|
+
}
|
|
840
|
+
function step5b(word) {
|
|
841
|
+
if (word.endsWith("ll")) {
|
|
842
|
+
const stem2 = word.slice(0, -1);
|
|
843
|
+
if (measure(stem2, stem2.length - 1) > 1) {
|
|
844
|
+
return stem2;
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
return word;
|
|
848
|
+
}
|
|
849
|
+
function stem(word) {
|
|
850
|
+
word = word.toLowerCase();
|
|
851
|
+
if (word.length < 3) {
|
|
852
|
+
return word;
|
|
853
|
+
}
|
|
854
|
+
word = step1a(word);
|
|
855
|
+
word = step1b(word);
|
|
856
|
+
word = step1c(word);
|
|
857
|
+
word = step2(word);
|
|
858
|
+
word = step3(word);
|
|
859
|
+
word = step4(word);
|
|
860
|
+
word = step5a(word);
|
|
861
|
+
word = step5b(word);
|
|
862
|
+
return word;
|
|
863
|
+
}
|
|
864
|
+
function tokenize(text) {
|
|
865
|
+
const cleanText = text.replace(/\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g, "$1").replace(/[*_`#\[\]()]/g, " ").toLowerCase();
|
|
866
|
+
const words = cleanText.match(/\b[a-z]{4,}\b/g) || [];
|
|
867
|
+
return words.filter((word) => !STOPWORDS.has(word));
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
// src/core/cooccurrence.ts
|
|
871
|
+
import { readdir, readFile } from "fs/promises";
|
|
495
872
|
import path3 from "path";
|
|
873
|
+
var DEFAULT_MIN_COOCCURRENCE = 2;
|
|
874
|
+
var EXCLUDED_FOLDERS = /* @__PURE__ */ new Set([
|
|
875
|
+
"templates",
|
|
876
|
+
".obsidian",
|
|
877
|
+
".claude",
|
|
878
|
+
".git"
|
|
879
|
+
]);
|
|
880
|
+
function noteContainsEntity(content, entityName) {
|
|
881
|
+
const entityTokens = tokenize(entityName);
|
|
882
|
+
if (entityTokens.length === 0)
|
|
883
|
+
return false;
|
|
884
|
+
const contentTokens = new Set(tokenize(content));
|
|
885
|
+
let matchCount = 0;
|
|
886
|
+
for (const token of entityTokens) {
|
|
887
|
+
if (contentTokens.has(token)) {
|
|
888
|
+
matchCount++;
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
if (entityTokens.length === 1) {
|
|
892
|
+
return matchCount === 1;
|
|
893
|
+
}
|
|
894
|
+
return matchCount / entityTokens.length >= 0.5;
|
|
895
|
+
}
|
|
896
|
+
function incrementCooccurrence(associations, entityA, entityB) {
|
|
897
|
+
if (!associations[entityA]) {
|
|
898
|
+
associations[entityA] = /* @__PURE__ */ new Map();
|
|
899
|
+
}
|
|
900
|
+
const current = associations[entityA].get(entityB) || 0;
|
|
901
|
+
associations[entityA].set(entityB, current + 1);
|
|
902
|
+
}
|
|
903
|
+
async function* walkMarkdownFiles(dir, baseDir) {
|
|
904
|
+
try {
|
|
905
|
+
const entries = await readdir(dir, { withFileTypes: true });
|
|
906
|
+
for (const entry of entries) {
|
|
907
|
+
const fullPath = path3.join(dir, entry.name);
|
|
908
|
+
const relativePath = path3.relative(baseDir, fullPath);
|
|
909
|
+
const topFolder = relativePath.split(path3.sep)[0];
|
|
910
|
+
if (EXCLUDED_FOLDERS.has(topFolder)) {
|
|
911
|
+
continue;
|
|
912
|
+
}
|
|
913
|
+
if (entry.isDirectory()) {
|
|
914
|
+
yield* walkMarkdownFiles(fullPath, baseDir);
|
|
915
|
+
} else if (entry.isFile() && entry.name.endsWith(".md")) {
|
|
916
|
+
yield { path: fullPath, relativePath };
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
} catch {
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
async function mineCooccurrences(vaultPath2, entities, options = {}) {
|
|
923
|
+
const { minCount = DEFAULT_MIN_COOCCURRENCE } = options;
|
|
924
|
+
const associations = {};
|
|
925
|
+
let notesScanned = 0;
|
|
926
|
+
const validEntities = entities.filter((e) => e.length <= 30);
|
|
927
|
+
for await (const file of walkMarkdownFiles(vaultPath2, vaultPath2)) {
|
|
928
|
+
try {
|
|
929
|
+
const content = await readFile(file.path, "utf-8");
|
|
930
|
+
notesScanned++;
|
|
931
|
+
const mentionedEntities = [];
|
|
932
|
+
for (const entity of validEntities) {
|
|
933
|
+
if (noteContainsEntity(content, entity)) {
|
|
934
|
+
mentionedEntities.push(entity);
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
for (const entityA of mentionedEntities) {
|
|
938
|
+
for (const entityB of mentionedEntities) {
|
|
939
|
+
if (entityA !== entityB) {
|
|
940
|
+
incrementCooccurrence(associations, entityA, entityB);
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
}
|
|
944
|
+
} catch {
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
let totalAssociations = 0;
|
|
948
|
+
for (const entityAssocs of Object.values(associations)) {
|
|
949
|
+
for (const count of entityAssocs.values()) {
|
|
950
|
+
if (count >= minCount) {
|
|
951
|
+
totalAssociations++;
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
return {
|
|
956
|
+
associations,
|
|
957
|
+
minCount,
|
|
958
|
+
_metadata: {
|
|
959
|
+
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
960
|
+
total_associations: totalAssociations,
|
|
961
|
+
notes_scanned: notesScanned
|
|
962
|
+
}
|
|
963
|
+
};
|
|
964
|
+
}
|
|
965
|
+
function getCooccurrenceBoost(entityName, matchedEntities, cooccurrenceIndex2) {
|
|
966
|
+
if (!cooccurrenceIndex2)
|
|
967
|
+
return 0;
|
|
968
|
+
let boost = 0;
|
|
969
|
+
const { associations, minCount } = cooccurrenceIndex2;
|
|
970
|
+
for (const matched of matchedEntities) {
|
|
971
|
+
const entityAssocs = associations[matched];
|
|
972
|
+
if (entityAssocs) {
|
|
973
|
+
const count = entityAssocs.get(entityName) || 0;
|
|
974
|
+
if (count >= minCount) {
|
|
975
|
+
boost += 3;
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
return boost;
|
|
980
|
+
}
|
|
981
|
+
|
|
982
|
+
// src/core/wikilinks.ts
|
|
496
983
|
var entityIndex = null;
|
|
497
984
|
var indexReady = false;
|
|
498
985
|
var indexError = null;
|
|
986
|
+
var cooccurrenceIndex = null;
|
|
499
987
|
var DEFAULT_EXCLUDE_FOLDERS = [
|
|
988
|
+
// Periodic notes
|
|
500
989
|
"daily-notes",
|
|
501
990
|
"daily",
|
|
502
991
|
"weekly",
|
|
992
|
+
"weekly-notes",
|
|
503
993
|
"monthly",
|
|
994
|
+
"monthly-notes",
|
|
504
995
|
"quarterly",
|
|
996
|
+
"yearly-notes",
|
|
505
997
|
"periodic",
|
|
506
998
|
"journal",
|
|
999
|
+
// Working folders
|
|
507
1000
|
"inbox",
|
|
508
|
-
"templates"
|
|
1001
|
+
"templates",
|
|
1002
|
+
"attachments",
|
|
1003
|
+
"tmp",
|
|
1004
|
+
"new",
|
|
1005
|
+
// Clippings & external content (article titles are not concepts)
|
|
1006
|
+
"clippings",
|
|
1007
|
+
"readwise",
|
|
1008
|
+
"articles",
|
|
1009
|
+
"bookmarks",
|
|
1010
|
+
"web-clips"
|
|
509
1011
|
];
|
|
510
1012
|
async function initializeEntityIndex(vaultPath2) {
|
|
511
|
-
const cacheFile =
|
|
1013
|
+
const cacheFile = path4.join(vaultPath2, ".claude", "wikilink-entities.json");
|
|
512
1014
|
try {
|
|
513
1015
|
const cached = await loadEntityCache(cacheFile);
|
|
514
1016
|
if (cached) {
|
|
@@ -534,8 +1036,17 @@ async function rebuildIndex(vaultPath2, cacheFile) {
|
|
|
534
1036
|
excludeFolders: DEFAULT_EXCLUDE_FOLDERS
|
|
535
1037
|
});
|
|
536
1038
|
indexReady = true;
|
|
537
|
-
const
|
|
538
|
-
console.error(`[Crank] Entity index built: ${entityIndex._metadata.total_entities} entities in ${
|
|
1039
|
+
const entityDuration = Date.now() - startTime;
|
|
1040
|
+
console.error(`[Crank] Entity index built: ${entityIndex._metadata.total_entities} entities in ${entityDuration}ms`);
|
|
1041
|
+
try {
|
|
1042
|
+
const cooccurrenceStart = Date.now();
|
|
1043
|
+
const entities = getAllEntities(entityIndex);
|
|
1044
|
+
cooccurrenceIndex = await mineCooccurrences(vaultPath2, entities);
|
|
1045
|
+
const cooccurrenceDuration = Date.now() - cooccurrenceStart;
|
|
1046
|
+
console.error(`[Crank] Co-occurrence index built: ${cooccurrenceIndex._metadata.total_associations} associations in ${cooccurrenceDuration}ms`);
|
|
1047
|
+
} catch (e) {
|
|
1048
|
+
console.error(`[Crank] Failed to build co-occurrence index: ${e}`);
|
|
1049
|
+
}
|
|
539
1050
|
try {
|
|
540
1051
|
await saveEntityCache(cacheFile, entityIndex);
|
|
541
1052
|
console.error(`[Crank] Entity cache saved`);
|
|
@@ -579,86 +1090,6 @@ function maybeApplyWikilinks(content, skipWikilinks) {
|
|
|
579
1090
|
return { content: result.content };
|
|
580
1091
|
}
|
|
581
1092
|
var SUGGESTION_PATTERN = /→\s*\[\[.+$/;
|
|
582
|
-
var STOPWORDS = /* @__PURE__ */ new Set([
|
|
583
|
-
"the",
|
|
584
|
-
"a",
|
|
585
|
-
"an",
|
|
586
|
-
"and",
|
|
587
|
-
"or",
|
|
588
|
-
"but",
|
|
589
|
-
"in",
|
|
590
|
-
"on",
|
|
591
|
-
"at",
|
|
592
|
-
"to",
|
|
593
|
-
"for",
|
|
594
|
-
"of",
|
|
595
|
-
"with",
|
|
596
|
-
"by",
|
|
597
|
-
"from",
|
|
598
|
-
"as",
|
|
599
|
-
"is",
|
|
600
|
-
"was",
|
|
601
|
-
"are",
|
|
602
|
-
"were",
|
|
603
|
-
"been",
|
|
604
|
-
"be",
|
|
605
|
-
"have",
|
|
606
|
-
"has",
|
|
607
|
-
"had",
|
|
608
|
-
"do",
|
|
609
|
-
"does",
|
|
610
|
-
"did",
|
|
611
|
-
"will",
|
|
612
|
-
"would",
|
|
613
|
-
"could",
|
|
614
|
-
"should",
|
|
615
|
-
"may",
|
|
616
|
-
"might",
|
|
617
|
-
"must",
|
|
618
|
-
"shall",
|
|
619
|
-
"can",
|
|
620
|
-
"need",
|
|
621
|
-
"this",
|
|
622
|
-
"that",
|
|
623
|
-
"these",
|
|
624
|
-
"those",
|
|
625
|
-
"i",
|
|
626
|
-
"you",
|
|
627
|
-
"he",
|
|
628
|
-
"she",
|
|
629
|
-
"it",
|
|
630
|
-
"we",
|
|
631
|
-
"they",
|
|
632
|
-
"what",
|
|
633
|
-
"which",
|
|
634
|
-
"who",
|
|
635
|
-
"whom",
|
|
636
|
-
"when",
|
|
637
|
-
"where",
|
|
638
|
-
"why",
|
|
639
|
-
"how",
|
|
640
|
-
"all",
|
|
641
|
-
"each",
|
|
642
|
-
"every",
|
|
643
|
-
"both",
|
|
644
|
-
"few",
|
|
645
|
-
"more",
|
|
646
|
-
"most",
|
|
647
|
-
"other",
|
|
648
|
-
"some",
|
|
649
|
-
"such",
|
|
650
|
-
"no",
|
|
651
|
-
"not",
|
|
652
|
-
"only",
|
|
653
|
-
"own",
|
|
654
|
-
"same",
|
|
655
|
-
"so",
|
|
656
|
-
"than",
|
|
657
|
-
"too",
|
|
658
|
-
"very",
|
|
659
|
-
"just",
|
|
660
|
-
"also"
|
|
661
|
-
]);
|
|
662
1093
|
function extractLinkedEntities(content) {
|
|
663
1094
|
const linked = /* @__PURE__ */ new Set();
|
|
664
1095
|
const wikilinkRegex = /\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g;
|
|
@@ -668,23 +1099,61 @@ function extractLinkedEntities(content) {
|
|
|
668
1099
|
}
|
|
669
1100
|
return linked;
|
|
670
1101
|
}
|
|
671
|
-
function
|
|
672
|
-
const
|
|
673
|
-
const
|
|
674
|
-
|
|
1102
|
+
function tokenizeForMatching(content) {
|
|
1103
|
+
const tokens = tokenize(content);
|
|
1104
|
+
const tokenSet = new Set(tokens);
|
|
1105
|
+
const stems = new Set(tokens.map((t) => stem(t)));
|
|
1106
|
+
return { tokens: tokenSet, stems };
|
|
1107
|
+
}
|
|
1108
|
+
var MAX_ENTITY_LENGTH = 25;
|
|
1109
|
+
var MAX_ENTITY_WORDS = 3;
|
|
1110
|
+
var ARTICLE_PATTERNS = [
|
|
1111
|
+
/\bguide\s+to\b/i,
|
|
1112
|
+
/\bhow\s+to\b/i,
|
|
1113
|
+
/\bcomplete\s+/i,
|
|
1114
|
+
/\bultimate\s+/i,
|
|
1115
|
+
/\bchecklist\b/i,
|
|
1116
|
+
/\bcheatsheet\b/i,
|
|
1117
|
+
/\bcheat\s+sheet\b/i,
|
|
1118
|
+
/\bbest\s+practices\b/i,
|
|
1119
|
+
/\bintroduction\s+to\b/i,
|
|
1120
|
+
/\btutorial\b/i,
|
|
1121
|
+
/\bworksheet\b/i
|
|
1122
|
+
];
|
|
1123
|
+
function isLikelyArticleTitle(name) {
|
|
1124
|
+
if (ARTICLE_PATTERNS.some((pattern) => pattern.test(name))) {
|
|
1125
|
+
return true;
|
|
1126
|
+
}
|
|
1127
|
+
const words = name.split(/\s+/).filter((w) => w.length > 0);
|
|
1128
|
+
if (words.length > MAX_ENTITY_WORDS) {
|
|
1129
|
+
return true;
|
|
1130
|
+
}
|
|
1131
|
+
return false;
|
|
675
1132
|
}
|
|
676
|
-
|
|
677
|
-
|
|
1133
|
+
var MIN_SUGGESTION_SCORE = 5;
|
|
1134
|
+
var MIN_MATCH_RATIO = 0.4;
|
|
1135
|
+
function scoreEntity(entityName, contentTokens, contentStems) {
|
|
1136
|
+
const entityTokens = tokenize(entityName);
|
|
1137
|
+
if (entityTokens.length === 0)
|
|
1138
|
+
return 0;
|
|
1139
|
+
const entityStems = entityTokens.map((t) => stem(t));
|
|
678
1140
|
let score = 0;
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
1141
|
+
let matchedWords = 0;
|
|
1142
|
+
for (let i = 0; i < entityTokens.length; i++) {
|
|
1143
|
+
const token = entityTokens[i];
|
|
1144
|
+
const entityStem = entityStems[i];
|
|
1145
|
+
if (contentTokens.has(token)) {
|
|
1146
|
+
score += 10;
|
|
1147
|
+
matchedWords++;
|
|
1148
|
+
} else if (contentStems.has(entityStem)) {
|
|
1149
|
+
score += 5;
|
|
1150
|
+
matchedWords++;
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
if (entityTokens.length > 1) {
|
|
1154
|
+
const matchRatio = matchedWords / entityTokens.length;
|
|
1155
|
+
if (matchRatio < MIN_MATCH_RATIO) {
|
|
1156
|
+
return 0;
|
|
688
1157
|
}
|
|
689
1158
|
}
|
|
690
1159
|
return score;
|
|
@@ -702,24 +1171,56 @@ function suggestRelatedLinks(content, options = {}) {
|
|
|
702
1171
|
if (entities.length === 0) {
|
|
703
1172
|
return emptyResult;
|
|
704
1173
|
}
|
|
705
|
-
const contentTokens =
|
|
706
|
-
if (contentTokens.
|
|
1174
|
+
const { tokens: contentTokens, stems: contentStems } = tokenizeForMatching(content);
|
|
1175
|
+
if (contentTokens.size === 0) {
|
|
707
1176
|
return emptyResult;
|
|
708
1177
|
}
|
|
709
1178
|
const linkedEntities = excludeLinked ? extractLinkedEntities(content) : /* @__PURE__ */ new Set();
|
|
710
1179
|
const scoredEntities = [];
|
|
1180
|
+
const directlyMatchedEntities = /* @__PURE__ */ new Set();
|
|
711
1181
|
for (const entity of entities) {
|
|
712
1182
|
const entityName = typeof entity === "string" ? entity : entity.name;
|
|
713
1183
|
if (!entityName)
|
|
714
1184
|
continue;
|
|
1185
|
+
if (entityName.length > MAX_ENTITY_LENGTH) {
|
|
1186
|
+
continue;
|
|
1187
|
+
}
|
|
1188
|
+
if (isLikelyArticleTitle(entityName)) {
|
|
1189
|
+
continue;
|
|
1190
|
+
}
|
|
715
1191
|
if (linkedEntities.has(entityName.toLowerCase())) {
|
|
716
1192
|
continue;
|
|
717
1193
|
}
|
|
718
|
-
const score = scoreEntity(entityName, contentTokens);
|
|
1194
|
+
const score = scoreEntity(entityName, contentTokens, contentStems);
|
|
719
1195
|
if (score > 0) {
|
|
1196
|
+
directlyMatchedEntities.add(entityName);
|
|
1197
|
+
}
|
|
1198
|
+
if (score >= MIN_SUGGESTION_SCORE) {
|
|
720
1199
|
scoredEntities.push({ name: entityName, score });
|
|
721
1200
|
}
|
|
722
1201
|
}
|
|
1202
|
+
if (cooccurrenceIndex && directlyMatchedEntities.size > 0) {
|
|
1203
|
+
for (const entity of entities) {
|
|
1204
|
+
const entityName = typeof entity === "string" ? entity : entity.name;
|
|
1205
|
+
if (!entityName)
|
|
1206
|
+
continue;
|
|
1207
|
+
if (entityName.length > MAX_ENTITY_LENGTH)
|
|
1208
|
+
continue;
|
|
1209
|
+
if (isLikelyArticleTitle(entityName))
|
|
1210
|
+
continue;
|
|
1211
|
+
if (linkedEntities.has(entityName.toLowerCase()))
|
|
1212
|
+
continue;
|
|
1213
|
+
const boost = getCooccurrenceBoost(entityName, directlyMatchedEntities, cooccurrenceIndex);
|
|
1214
|
+
if (boost > 0) {
|
|
1215
|
+
const existing = scoredEntities.find((e) => e.name === entityName);
|
|
1216
|
+
if (existing) {
|
|
1217
|
+
existing.score += boost;
|
|
1218
|
+
} else if (boost >= MIN_SUGGESTION_SCORE) {
|
|
1219
|
+
scoredEntities.push({ name: entityName, score: boost });
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
723
1224
|
scoredEntities.sort((a, b) => b.score - a.score);
|
|
724
1225
|
const topSuggestions = scoredEntities.slice(0, maxSuggestions).map((e) => e.name);
|
|
725
1226
|
if (topSuggestions.length === 0) {
|
|
@@ -734,7 +1235,7 @@ function suggestRelatedLinks(content, options = {}) {
|
|
|
734
1235
|
|
|
735
1236
|
// src/tools/mutations.ts
|
|
736
1237
|
import fs2 from "fs/promises";
|
|
737
|
-
import
|
|
1238
|
+
import path5 from "path";
|
|
738
1239
|
function registerMutationTools(server2, vaultPath2) {
|
|
739
1240
|
server2.tool(
|
|
740
1241
|
"vault_add_to_section",
|
|
@@ -752,7 +1253,7 @@ function registerMutationTools(server2, vaultPath2) {
|
|
|
752
1253
|
},
|
|
753
1254
|
async ({ path: notePath, section, content, position, format, commit, skipWikilinks, preserveListNesting, suggestOutgoingLinks }) => {
|
|
754
1255
|
try {
|
|
755
|
-
const fullPath =
|
|
1256
|
+
const fullPath = path5.join(vaultPath2, notePath);
|
|
756
1257
|
try {
|
|
757
1258
|
await fs2.access(fullPath);
|
|
758
1259
|
} catch {
|
|
@@ -836,7 +1337,7 @@ function registerMutationTools(server2, vaultPath2) {
|
|
|
836
1337
|
},
|
|
837
1338
|
async ({ path: notePath, section, pattern, mode, useRegex, commit }) => {
|
|
838
1339
|
try {
|
|
839
|
-
const fullPath =
|
|
1340
|
+
const fullPath = path5.join(vaultPath2, notePath);
|
|
840
1341
|
try {
|
|
841
1342
|
await fs2.access(fullPath);
|
|
842
1343
|
} catch {
|
|
@@ -918,7 +1419,7 @@ function registerMutationTools(server2, vaultPath2) {
|
|
|
918
1419
|
},
|
|
919
1420
|
async ({ path: notePath, section, search, replacement, mode, useRegex, commit, skipWikilinks, suggestOutgoingLinks }) => {
|
|
920
1421
|
try {
|
|
921
|
-
const fullPath =
|
|
1422
|
+
const fullPath = path5.join(vaultPath2, notePath);
|
|
922
1423
|
try {
|
|
923
1424
|
await fs2.access(fullPath);
|
|
924
1425
|
} catch {
|
|
@@ -1004,7 +1505,7 @@ function registerMutationTools(server2, vaultPath2) {
|
|
|
1004
1505
|
// src/tools/tasks.ts
|
|
1005
1506
|
import { z as z2 } from "zod";
|
|
1006
1507
|
import fs3 from "fs/promises";
|
|
1007
|
-
import
|
|
1508
|
+
import path6 from "path";
|
|
1008
1509
|
var TASK_REGEX = /^(\s*)-\s*\[([ xX])\]\s*(.*)$/;
|
|
1009
1510
|
function findTasks(content, section) {
|
|
1010
1511
|
const lines = content.split("\n");
|
|
@@ -1060,7 +1561,7 @@ function registerTaskTools(server2, vaultPath2) {
|
|
|
1060
1561
|
},
|
|
1061
1562
|
async ({ path: notePath, task, section, commit }) => {
|
|
1062
1563
|
try {
|
|
1063
|
-
const fullPath =
|
|
1564
|
+
const fullPath = path6.join(vaultPath2, notePath);
|
|
1064
1565
|
try {
|
|
1065
1566
|
await fs3.access(fullPath);
|
|
1066
1567
|
} catch {
|
|
@@ -1155,7 +1656,7 @@ function registerTaskTools(server2, vaultPath2) {
|
|
|
1155
1656
|
},
|
|
1156
1657
|
async ({ path: notePath, section, task, position, completed, commit, skipWikilinks, suggestOutgoingLinks, preserveListNesting }) => {
|
|
1157
1658
|
try {
|
|
1158
|
-
const fullPath =
|
|
1659
|
+
const fullPath = path6.join(vaultPath2, notePath);
|
|
1159
1660
|
try {
|
|
1160
1661
|
await fs3.access(fullPath);
|
|
1161
1662
|
} catch {
|
|
@@ -1232,7 +1733,7 @@ function registerTaskTools(server2, vaultPath2) {
|
|
|
1232
1733
|
// src/tools/frontmatter.ts
|
|
1233
1734
|
import { z as z3 } from "zod";
|
|
1234
1735
|
import fs4 from "fs/promises";
|
|
1235
|
-
import
|
|
1736
|
+
import path7 from "path";
|
|
1236
1737
|
function registerFrontmatterTools(server2, vaultPath2) {
|
|
1237
1738
|
server2.tool(
|
|
1238
1739
|
"vault_update_frontmatter",
|
|
@@ -1244,7 +1745,7 @@ function registerFrontmatterTools(server2, vaultPath2) {
|
|
|
1244
1745
|
},
|
|
1245
1746
|
async ({ path: notePath, frontmatter: updates, commit }) => {
|
|
1246
1747
|
try {
|
|
1247
|
-
const fullPath =
|
|
1748
|
+
const fullPath = path7.join(vaultPath2, notePath);
|
|
1248
1749
|
try {
|
|
1249
1750
|
await fs4.access(fullPath);
|
|
1250
1751
|
} catch {
|
|
@@ -1300,7 +1801,7 @@ function registerFrontmatterTools(server2, vaultPath2) {
|
|
|
1300
1801
|
},
|
|
1301
1802
|
async ({ path: notePath, key, value, commit }) => {
|
|
1302
1803
|
try {
|
|
1303
|
-
const fullPath =
|
|
1804
|
+
const fullPath = path7.join(vaultPath2, notePath);
|
|
1304
1805
|
try {
|
|
1305
1806
|
await fs4.access(fullPath);
|
|
1306
1807
|
} catch {
|
|
@@ -1357,7 +1858,7 @@ function registerFrontmatterTools(server2, vaultPath2) {
|
|
|
1357
1858
|
// src/tools/notes.ts
|
|
1358
1859
|
import { z as z4 } from "zod";
|
|
1359
1860
|
import fs5 from "fs/promises";
|
|
1360
|
-
import
|
|
1861
|
+
import path8 from "path";
|
|
1361
1862
|
function registerNoteTools(server2, vaultPath2) {
|
|
1362
1863
|
server2.tool(
|
|
1363
1864
|
"vault_create_note",
|
|
@@ -1379,7 +1880,7 @@ function registerNoteTools(server2, vaultPath2) {
|
|
|
1379
1880
|
};
|
|
1380
1881
|
return { content: [{ type: "text", text: JSON.stringify(result2, null, 2) }] };
|
|
1381
1882
|
}
|
|
1382
|
-
const fullPath =
|
|
1883
|
+
const fullPath = path8.join(vaultPath2, notePath);
|
|
1383
1884
|
try {
|
|
1384
1885
|
await fs5.access(fullPath);
|
|
1385
1886
|
if (!overwrite) {
|
|
@@ -1392,7 +1893,7 @@ function registerNoteTools(server2, vaultPath2) {
|
|
|
1392
1893
|
}
|
|
1393
1894
|
} catch {
|
|
1394
1895
|
}
|
|
1395
|
-
const dir =
|
|
1896
|
+
const dir = path8.dirname(fullPath);
|
|
1396
1897
|
await fs5.mkdir(dir, { recursive: true });
|
|
1397
1898
|
await writeVaultFile(vaultPath2, notePath, content, frontmatter);
|
|
1398
1899
|
let gitCommit;
|
|
@@ -1451,7 +1952,7 @@ Content length: ${content.length} chars`,
|
|
|
1451
1952
|
};
|
|
1452
1953
|
return { content: [{ type: "text", text: JSON.stringify(result2, null, 2) }] };
|
|
1453
1954
|
}
|
|
1454
|
-
const fullPath =
|
|
1955
|
+
const fullPath = path8.join(vaultPath2, notePath);
|
|
1455
1956
|
try {
|
|
1456
1957
|
await fs5.access(fullPath);
|
|
1457
1958
|
} catch {
|
|
@@ -1497,7 +1998,7 @@ Content length: ${content.length} chars`,
|
|
|
1497
1998
|
// src/tools/system.ts
|
|
1498
1999
|
import { z as z5 } from "zod";
|
|
1499
2000
|
import fs6 from "fs/promises";
|
|
1500
|
-
import
|
|
2001
|
+
import path9 from "path";
|
|
1501
2002
|
function registerSystemTools(server2, vaultPath2) {
|
|
1502
2003
|
server2.tool(
|
|
1503
2004
|
"vault_list_sections",
|
|
@@ -1517,7 +2018,7 @@ function registerSystemTools(server2, vaultPath2) {
|
|
|
1517
2018
|
};
|
|
1518
2019
|
return { content: [{ type: "text", text: JSON.stringify(result2, null, 2) }] };
|
|
1519
2020
|
}
|
|
1520
|
-
const fullPath =
|
|
2021
|
+
const fullPath = path9.join(vaultPath2, notePath);
|
|
1521
2022
|
try {
|
|
1522
2023
|
await fs6.access(fullPath);
|
|
1523
2024
|
} catch {
|
|
@@ -1625,18 +2126,18 @@ Message: ${undoResult.undoneCommit.message}` : void 0
|
|
|
1625
2126
|
|
|
1626
2127
|
// src/core/vaultRoot.ts
|
|
1627
2128
|
import * as fs7 from "fs";
|
|
1628
|
-
import * as
|
|
2129
|
+
import * as path10 from "path";
|
|
1629
2130
|
var VAULT_MARKERS = [".obsidian", ".claude"];
|
|
1630
2131
|
function findVaultRoot(startPath) {
|
|
1631
|
-
let current =
|
|
2132
|
+
let current = path10.resolve(startPath || process.cwd());
|
|
1632
2133
|
while (true) {
|
|
1633
2134
|
for (const marker of VAULT_MARKERS) {
|
|
1634
|
-
const markerPath =
|
|
2135
|
+
const markerPath = path10.join(current, marker);
|
|
1635
2136
|
if (fs7.existsSync(markerPath) && fs7.statSync(markerPath).isDirectory()) {
|
|
1636
2137
|
return current;
|
|
1637
2138
|
}
|
|
1638
2139
|
}
|
|
1639
|
-
const parent =
|
|
2140
|
+
const parent = path10.dirname(current);
|
|
1640
2141
|
if (parent === current) {
|
|
1641
2142
|
return startPath || process.cwd();
|
|
1642
2143
|
}
|