@cad0p/napkin 0.8.1

package/dist/core/overview.js

@@ -0,0 +1,384 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { loadConfig } from "../utils/config.js";
+import { listFiles } from "../utils/files.js";
+import { parseFrontmatter } from "../utils/frontmatter.js";
+import { extractHeadings, extractTags } from "../utils/markdown.js";
+const CODE_BLOCK_RE = /```[\s\S]*?```/g;
+const INLINE_CODE_RE = /`[^`]+`/g;
+const URL_RE = /https?:\/\/[^\s)>\]]+/g;
+const EMAIL_RE = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g;
+const HEX_HASH_RE = /\b[a-f0-9]{8,}\b/g;
+const TOKEN_RE = /[a-z]{3,}/g;
+const FRONTMATTER_RE = /^---[\s\S]*?---\n?/;
+const ATX_HEADING_LINE_RE = /^#{1,6}\s+.+$/gm;
+const WIKILINK_ONLY_RE = /^\[\[[^\]]+(?:\|[^\]]+)?\]\]$/;
+const ISO_DATE_PREFIX_RE = /^\d{4}-\d{2}-\d{2}/;
+const STOP_WORDS = new Set([
+    "the",
+    "a",
+    "an",
+    "and",
+    "or",
+    "but",
+    "in",
+    "on",
+    "at",
+    "to",
+    "for",
+    "of",
+    "with",
+    "by",
+    "from",
+    "is",
+    "it",
+    "as",
+    "be",
+    "was",
+    "are",
+    "this",
+    "that",
+    "not",
+    "has",
+    "have",
+    "had",
+    "will",
+    "can",
+    "may",
+    "do",
+    "does",
+    "did",
+    "been",
+    "being",
+    "would",
+    "could",
+    "should",
+    "its",
+    "my",
+    "your",
+    "our",
+    "their",
+    "his",
+    "her",
+    "we",
+    "they",
+    "you",
+    "he",
+    "she",
+    "all",
+    "each",
+    "every",
+    "both",
+    "few",
+    "more",
+    "most",
+    "other",
+    "some",
+    "such",
+    "than",
+    "too",
+    "very",
+    "just",
+    "about",
+    "above",
+    "after",
+    "again",
+    "also",
+    "any",
+    "because",
+    "before",
+    "between",
+    "down",
+    "during",
+    "even",
+    "first",
+    "get",
+    "how",
+    "if",
+    "into",
+    "like",
+    "made",
+    "make",
+    "many",
+    "much",
+    "new",
+    "no",
+    "now",
+    "off",
+    "old",
+    "only",
+    "one",
+    "out",
+    "over",
+    "own",
+    "same",
+    "so",
+    "still",
+    "then",
+    "there",
+    "these",
+    "those",
+    "through",
+    "under",
+    "up",
+    "use",
+    "used",
+    "using",
+    "want",
+    "way",
+    "well",
+    "what",
+    "when",
+    "where",
+    "which",
+    "while",
+    "who",
+    "why",
+    "work",
+    "see",
+    "here",
+    "need",
+    "etc",
+    "two",
+    "next",
+    "per",
+    "via",
+    "vs",
+    "yet",
+    "ago",
+    "due",
+    "tbd",
+]); // prettier-ignore
+function stripNoise(text) {
+    return text
+        .replace(CODE_BLOCK_RE, "")
+        .replace(INLINE_CODE_RE, "")
+        .replace(URL_RE, "")
+        .replace(EMAIL_RE, "")
+        .replace(HEX_HASH_RE, "");
+}
+function tokenize(text) {
+    const cleaned = stripNoise(text);
+    return (cleaned.toLowerCase().match(TOKEN_RE) || []).filter((w) => !STOP_WORDS.has(w));
+}
+function extractBigrams(text) {
+    const words = tokenize(text);
+    const bigrams = [];
+    for (let i = 0; i < words.length - 1; i++) {
+        bigrams.push(`${words[i]} ${words[i + 1]}`);
+    }
+    return bigrams;
+}
+function terms(text) {
+    return [...tokenize(text), ...extractBigrams(text)];
+}
+function addWeightedTerms(target, sourceTerms, weight) {
+    for (const term of sourceTerms) {
+        target.set(term, (target.get(term) || 0) + weight);
+    }
+}
+function buildTF(sources) {
+    const freq = new Map();
+    for (const { text, weight } of sources) {
+        addWeightedTerms(freq, terms(text), weight);
+    }
+    return freq;
+}
+function folderKeywordTokens(folderPath) {
+    const tokens = new Set();
+    for (const segment of folderPath.split("/")) {
+        for (const token of tokenize(segment)) {
+            tokens.add(token);
+            tokens.add(token.endsWith("s") ? token.slice(0, -1) : `${token}s`);
+        }
+    }
+    return tokens;
+}
+function shouldSkipOverviewFile(file, folder, templatesFolder) {
+    const basename = path.basename(file);
+    const topLevelFolder = folder === "/" ? "" : folder.split("/")[0];
+    return (topLevelFolder === templatesFolder ||
+        (folder === "/" && basename === "NAPKIN.md") ||
+        basename === "_about.md");
+}
+function frontmatterText(properties) {
+    const values = [];
+    const visit = (value) => {
+        if (typeof value === "string")
+            values.push(value);
+        else if (Array.isArray(value))
+            value.forEach(visit);
+    };
+    for (const [key, value] of Object.entries(properties)) {
+        if (key === "title" || key === "tags")
+            continue;
+        visit(value);
+    }
+    return values.filter((value) => {
+        const trimmed = value.trim();
+        return (trimmed.length > 0 &&
+            !WIKILINK_ONLY_RE.test(trimmed) &&
+            !ISO_DATE_PREFIX_RE.test(trimmed));
+    });
+}
+function markdownBodyText(content) {
+    return content.replace(FRONTMATTER_RE, "").replace(ATX_HEADING_LINE_RE, "");
+}
+function buildHeadingSignals(headings) {
+    const lineCount = new Map();
+    const weightedTerms = new Map();
+    const uniqueTerms = new Set();
+    for (const heading of headings) {
+        const seenInHeading = new Set(terms(heading));
+        for (const term of seenInHeading) {
+            uniqueTerms.add(term);
+            lineCount.set(term, (lineCount.get(term) || 0) + 1);
+        }
+    }
+    addWeightedTerms(weightedTerms, uniqueTerms, 3);
+    return { lineCount, weightedTerms };
+}
+function isCandidateKeyword(term, tf, folderTokens, headingLineCount, hasNonHeading) {
+    if (term.includes(" ")) {
+        const [a, b] = term.split(" ");
+        if (tf < 2 || a === b)
+            return false;
+    }
+    else if (folderTokens.has(term)) {
+        return false;
+    }
+    // Require corroboration: real keywords either appear outside headings or in
+    // multiple heading lines. Single heading-only terms are usually section labels.
+    return hasNonHeading.has(term) || (headingLineCount.get(term) || 0) >= 2;
+}
+function extractKeywordsTFIDF(folderTF, documentFrequency, totalFolders, maxKeywords, folderPath, headingLineCount, hasNonHeading) {
+    const folderTokens = folderKeywordTokens(folderPath);
+    const scored = [];
+    for (const [term, tf] of folderTF) {
+        if (!isCandidateKeyword(term, tf, folderTokens, headingLineCount, hasNonHeading)) {
+            continue;
+        }
+        const df = documentFrequency.get(term) || 1;
+        const idf = Math.log(1 + totalFolders / df);
+        scored.push([term, tf * idf]);
+    }
+    const sorted = scored.sort((a, b) => b[1] - a[1]);
+    const selected = [];
+    const suppressed = new Set();
+    for (const [term] of sorted) {
+        if (selected.length >= maxKeywords)
+            break;
+        if (suppressed.has(term))
+            continue;
+        selected.push(term);
+        if (term.includes(" ")) {
+            for (const part of term.split(" ")) {
+                suppressed.add(part);
+            }
+        }
+    }
+    return selected;
+}
+function groupFilesByFolder(files, templatesFolder) {
+    const folderFiles = new Map();
+    for (const file of files) {
+        const dir = path.dirname(file);
+        const folder = dir === "." ? "/" : dir;
+        if (shouldSkipOverviewFile(file, folder, templatesFolder))
+            continue;
+        if (!folderFiles.has(folder))
+            folderFiles.set(folder, []);
+        folderFiles.get(folder)?.push(file);
+    }
+    return folderFiles;
+}
+function buildFolderData(vaultPath, folderFileList, warnings) {
+    const allTags = new Set();
+    const headings = new Set();
+    const weightedSources = [];
+    for (const file of folderFileList) {
+        const content = fs.readFileSync(path.join(vaultPath, file), "utf-8");
+        let properties = {};
+        try {
+            ({ properties } = parseFrontmatter(content));
+        }
+        catch {
+            warnings.push(`Skipping ${file} (malformed YAML frontmatter)`);
+            continue;
+        }
+        for (const tag of extractTags(content))
+            allTags.add(tag);
+        if (Array.isArray(properties.tags)) {
+            for (const tag of properties.tags)
+                allTags.add(String(tag));
+        }
+        for (const heading of extractHeadings(content)) {
+            headings.add(heading.text.trim());
+        }
+        weightedSources.push({ text: path.basename(file, ".md"), weight: 2 });
+        if (properties.title) {
+            weightedSources.push({ text: String(properties.title), weight: 2 });
+        }
+        for (const value of frontmatterText(properties)) {
+            weightedSources.push({ text: value, weight: 2 });
+        }
+        weightedSources.push({ text: markdownBodyText(content), weight: 1 });
+    }
+    const tf = buildTF(weightedSources);
+    const hasNonHeading = new Set(tf.keys());
+    const headingSignals = buildHeadingSignals(headings);
+    for (const [term, weight] of headingSignals.weightedTerms) {
+        tf.set(term, (tf.get(term) || 0) + weight);
+    }
+    return {
+        tf,
+        headingLineCount: headingSignals.lineCount,
+        hasNonHeading,
+        tags: allTags,
+        noteCount: folderFileList.length,
+    };
+}
+function buildOverviewFolders(vaultPath, maxDepth, maxKeywords, templatesFolder) {
+    const files = listFiles(vaultPath, { ext: "md" });
+    const warnings = [];
+    const folderFiles = groupFilesByFolder(files, templatesFolder);
+    const folderData = new Map();
+    for (const [folder, folderFileList] of folderFiles) {
+        const depth = folder === "/" ? 0 : folder.split("/").length;
+        if (depth > maxDepth)
+            continue;
+        folderData.set(folder, buildFolderData(vaultPath, folderFileList, warnings));
+    }
+    const documentFrequency = new Map();
+    for (const { tf } of folderData.values()) {
+        for (const term of tf.keys()) {
+            documentFrequency.set(term, (documentFrequency.get(term) || 0) + 1);
+        }
+    }
+    const totalFolders = folderData.size;
+    const folders = [];
+    for (const [folder, data] of [...folderData.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
+        folders.push({
+            path: folder,
+            notes: data.noteCount,
+            keywords: extractKeywordsTFIDF(data.tf, documentFrequency, totalFolders, maxKeywords, folder, data.headingLineCount, data.hasNonHeading),
+            tags: [...data.tags].sort(),
+        });
+    }
+    return { folders, warnings };
+}
+export function getOverview(contentPath, configPath, opts) {
+    const config = loadConfig(configPath);
+    const maxDepth = opts?.depth ?? config.overview.depth;
+    const maxKeywords = opts?.keywords ?? config.overview.keywords;
+    const { folders, warnings } = buildOverviewFolders(contentPath, maxDepth, maxKeywords, config.templates.folder);
+    const contextPath = path.join(contentPath, "NAPKIN.md");
+    const context = fs.existsSync(contextPath)
+        ? fs.readFileSync(contextPath, "utf-8").trim()
+        : undefined;
+    return {
+        ...(context ? { context } : {}),
+        overview: folders,
+        ...(warnings.length > 0 ? { warnings } : {}),
+    };
+}

package/dist/core/properties.d.ts

@@ -0,0 +1,14 @@
+export declare function collectProperties(vaultPath: string, fileFilter?: string): Map<string, number>;
+export declare function setProperty(vaultPath: string, fileRef: string, name: string, value: string): {
+    path: string;
+    property: string;
+    value: unknown;
+};
+export declare function removeProperty(vaultPath: string, fileRef: string, name: string): {
+    path: string;
+    removed: string;
+};
+export declare function readProperty(vaultPath: string, fileRef: string, name: string): {
+    property: string;
+    value: unknown;
+};

package/dist/core/properties.js

@@ -0,0 +1,60 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { listFiles, resolveFile } from "../utils/files.js";
+import { parseFrontmatter, removeProperty as removeProp, setProperty as setProp, } from "../utils/frontmatter.js";
+export function collectProperties(vaultPath, fileFilter) {
+    const propCounts = new Map();
+    const files = fileFilter
+        ? (() => {
+            const r = resolveFile(vaultPath, fileFilter);
+            return r ? [r] : [];
+        })()
+        : listFiles(vaultPath, { ext: "md" });
+    for (const file of files) {
+        const content = fs.readFileSync(path.join(vaultPath, file), "utf-8");
+        const { properties } = parseFrontmatter(content);
+        for (const key of Object.keys(properties)) {
+            propCounts.set(key, (propCounts.get(key) || 0) + 1);
+        }
+    }
+    return propCounts;
+}
+export function setProperty(vaultPath, fileRef, name, value) {
+    const resolved = resolveFile(vaultPath, fileRef);
+    if (!resolved) {
+        throw new Error(`File not found: ${fileRef}`);
+    }
+    const fullPath = path.join(vaultPath, resolved);
+    const content = fs.readFileSync(fullPath, "utf-8");
+    let parsedValue = value;
+    if (value === "true")
+        parsedValue = true;
+    else if (value === "false")
+        parsedValue = false;
+    else if (!Number.isNaN(Number(value)) && value.trim() !== "")
+        parsedValue = Number(value);
+    const updated = setProp(content, name, parsedValue);
+    fs.writeFileSync(fullPath, updated);
+    return { path: resolved, property: name, value: parsedValue };
+}
+export function removeProperty(vaultPath, fileRef, name) {
+    const resolved = resolveFile(vaultPath, fileRef);
+    if (!resolved) {
+        throw new Error(`File not found: ${fileRef}`);
+    }
+    const fullPath = path.join(vaultPath, resolved);
+    const content = fs.readFileSync(fullPath, "utf-8");
+    const updated = removeProp(content, name);
+    fs.writeFileSync(fullPath, updated);
+    return { path: resolved, removed: name };
+}
+export function readProperty(vaultPath, fileRef, name) {
+    const resolved = resolveFile(vaultPath, fileRef);
+    if (!resolved) {
+        throw new Error(`File not found: ${fileRef}`);
+    }
+    const fullPath = path.join(vaultPath, resolved);
+    const content = fs.readFileSync(fullPath, "utf-8");
+    const { properties: props } = parseFrontmatter(content);
+    return { property: name, value: props[name] ?? null };
+}

package/dist/core/search.d.ts

@@ -0,0 +1,17 @@
+export interface SearchResult {
+    file: string;
+    score: number;
+    links: number;
+    modified: string;
+    snippets: {
+        line: number;
+        text: string;
+    }[];
+}
+export interface SearchOptions {
+    path?: string;
+    limit?: number;
+    snippetLines?: number;
+    snippets?: boolean;
+}
+export declare function searchVault(contentPath: string, configPath: string, query: string, opts?: SearchOptions): SearchResult[];

package/dist/core/search.js

@@ -0,0 +1,153 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import MiniSearch from "minisearch";
+import { loadConfig } from "../utils/config.js";
+import { listFiles, resolveFileLoose } from "../utils/files.js";
+import { extractLinks } from "../utils/markdown.js";
+import { computeFingerprint, loadSearchCache, saveSearchCache, } from "../utils/search-cache.js";
+function buildIndex(vaultPath, folder) {
+    const files = listFiles(vaultPath, { folder, ext: "md" });
+    const docs = files.map((file, id) => {
+        const fullPath = path.join(vaultPath, file);
+        const content = fs.readFileSync(fullPath, "utf-8");
+        const stat = fs.statSync(fullPath);
+        const basename = path.basename(file, ".md");
+        return { id, file, basename, content, mtime: stat.mtimeMs };
+    });
+    const index = new MiniSearch({
+        fields: ["basename", "content"],
+        storeFields: ["file"],
+        searchOptions: {
+            boost: { basename: 2 },
+            fuzzy: 0.2,
+            prefix: true,
+        },
+    });
+    index.addAll(docs);
+    return { index, docs };
+}
+function buildBacklinkCounts(vaultPath) {
+    const files = listFiles(vaultPath, { ext: "md" });
+    const counts = new Map();
+    for (const file of files) {
+        const content = fs.readFileSync(path.join(vaultPath, file), "utf-8");
+        const links = extractLinks(content);
+        for (const target of links.wikilinks) {
+            const resolved = resolveFileLoose(vaultPath, target);
+            if (resolved) {
+                counts.set(resolved, (counts.get(resolved) || 0) + 1);
+            }
+        }
+    }
+    return counts;
+}
+function extractSnippets(content, query, contextLines) {
+    const terms = query
+        .toLowerCase()
+        .split(/\s+/)
+        .filter((t) => t.length > 0);
+    const lines = content.split("\n");
+    const matchedLines = new Set();
+    for (let i = 0; i < lines.length; i++) {
+        const lower = lines[i].toLowerCase();
+        if (terms.some((t) => lower.includes(t))) {
+            matchedLines.add(i);
+        }
+    }
+    const ranges = [];
+    for (const lineIdx of [...matchedLines].sort((a, b) => a - b)) {
+        const start = Math.max(0, lineIdx - contextLines);
+        const end = Math.min(lines.length - 1, lineIdx + contextLines);
+        if (ranges.length > 0 && start <= ranges[ranges.length - 1][1] + 1) {
+            ranges[ranges.length - 1][1] = end;
+        }
+        else {
+            ranges.push([start, end]);
+        }
+    }
+    const snippets = [];
+    for (const [start, end] of ranges) {
+        for (let i = start; i <= end; i++) {
+            const line = lines[i];
+            if (line.trim() === "")
+                continue;
+            snippets.push({ line: i + 1, text: line });
+        }
+    }
+    return snippets;
+}
+function relativeTime(mtimeMs) {
+    const diff = Date.now() - mtimeMs;
+    const minutes = Math.floor(diff / 60000);
+    if (minutes < 60)
+        return `${minutes}m ago`;
+    const hours = Math.floor(minutes / 60);
+    if (hours < 24)
+        return `${hours}h ago`;
+    const days = Math.floor(hours / 24);
+    if (days < 30)
+        return `${days}d ago`;
+    const months = Math.floor(days / 30);
+    return `${months}mo ago`;
+}
+export function searchVault(contentPath, configPath, query, opts) {
+    const config = loadConfig(configPath);
+    const fingerprint = computeFingerprint(contentPath, opts?.path);
+    const cached = loadSearchCache(configPath, fingerprint);
+    let index;
+    let docs;
+    let backlinkCounts;
+    if (cached) {
+        index = MiniSearch.loadJSON(cached.index, {
+            fields: ["basename", "content"],
+            storeFields: ["file"],
+            searchOptions: {
+                boost: { basename: 2 },
+                fuzzy: 0.2,
+                prefix: true,
+            },
+        });
+        docs = cached.docs.map((d) => {
+            const fullPath = path.join(contentPath, d.file);
+            const content = fs.readFileSync(fullPath, "utf-8");
+            return { ...d, content };
+        });
+        backlinkCounts = new Map(Object.entries(cached.backlinkCounts));
+    }
+    else {
+        const built = buildIndex(contentPath, opts?.path);
+        index = built.index;
+        docs = built.docs;
+        backlinkCounts = buildBacklinkCounts(contentPath);
+        saveSearchCache(configPath, {
+            fingerprint,
+            index: JSON.stringify(index),
+            docs: docs.map(({ content: _, ...rest }) => rest),
+            backlinkCounts: Object.fromEntries(backlinkCounts),
+        });
+    }
+    const results = index.search(query);
+    const contextLines = opts?.snippetLines ?? config.search.snippetLines;
+    const limit = opts?.limit ?? config.search.limit;
+    const maxMtime = Math.max(...docs.map((d) => d.mtime));
+    const minMtime = Math.min(...docs.map((d) => d.mtime));
+    const mtimeRange = maxMtime - minMtime || 1;
+    const scored = results.map((r) => {
+        const doc = docs[r.id];
+        const bm25Score = r.score;
+        const links = backlinkCounts.get(doc.file) || 0;
+        const recency = (doc.mtime - minMtime) / mtimeRange;
+        const composite = bm25Score + links * 0.5 + recency * 1.0;
+        return {
+            file: doc.file,
+            score: Math.round(composite * 10) / 10,
+            links,
+            modified: relativeTime(doc.mtime),
+            snippets: opts?.snippets === false
+                ? []
+                : extractSnippets(doc.content, query, contextLines),
+        };
+    });
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, limit);
+}

package/dist/core/tags.d.ts

@@ -0,0 +1,11 @@
+export interface TagData {
+    tagCounts: Map<string, number>;
+    tagFiles: Map<string, string[]>;
+}
+export interface TagInfo {
+    tag: string;
+    count: number;
+    files: string[];
+}
+export declare function collectTags(vaultPath: string, fileFilter?: string): TagData;
+export declare function getTagInfo(vaultPath: string, tagName: string): TagInfo;

package/dist/core/tags.js

@@ -0,0 +1,40 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { listFiles, resolveFile } from "../utils/files.js";
+import { parseFrontmatter } from "../utils/frontmatter.js";
+import { extractTags } from "../utils/markdown.js";
+export function collectTags(vaultPath, fileFilter) {
+    const tagCounts = new Map();
+    const tagFiles = new Map();
+    const files = fileFilter
+        ? (() => {
+            const r = resolveFile(vaultPath, fileFilter);
+            return r ? [r] : [];
+        })()
+        : listFiles(vaultPath, { ext: "md" });
+    for (const file of files) {
+        const content = fs.readFileSync(path.join(vaultPath, file), "utf-8");
+        const { properties } = parseFrontmatter(content);
+        const inlineTags = extractTags(content);
+        const allTags = new Set(inlineTags);
+        if (Array.isArray(properties.tags)) {
+            for (const t of properties.tags)
+                allTags.add(String(t));
+        }
+        for (const tag of allTags) {
+            tagCounts.set(tag, (tagCounts.get(tag) || 0) + 1);
+            if (!tagFiles.has(tag))
+                tagFiles.set(tag, []);
+            tagFiles.get(tag)?.push(file);
+        }
+    }
+    return { tagCounts, tagFiles };
+}
+export function getTagInfo(vaultPath, tagName) {
+    const { tagCounts, tagFiles } = collectTags(vaultPath);
+    return {
+        tag: tagName,
+        count: tagCounts.get(tagName) || 0,
+        files: tagFiles.get(tagName) || [],
+    };
+}