@wgtechlabs/mdd-engine 0.1.0-pr.0cb765a → 0.1.0-pr.8fa2dc9

package/README.md

@@ -85,6 +85,10 @@ import { search } from '@wgtechlabs/mdd-engine/search';
 const index = createSearchIndex(result.site);
 const restored = JSON.parse(JSON.stringify(index));
 console.log(search(restored, 'installation', { limit: 10 }));
+// Multiple heading hits, breadcrumbs, and ranges for UI highlights.
+console.log(search(restored, 'instalation', {
+  mode: 'sections', fuzzy: true, limit: 8,
+}));
 // [{ title, url, section?, excerpt, score }]
 ```
 

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import type { CompileOptions, CompileResult } from "./types.js";
-export { type SearchIndex, type SearchOptions, type SearchPage, type SearchResult, type SearchSection, search, validateSearchIndex, } from "./search.js";
+export { type SearchIndex, type SearchMatch, type SearchOptions, type SearchPage, type SearchResult, type SearchSection, search, validateSearchIndex, } from "./search.js";
 export { createSearchIndex } from "./search-index.js";
 export type { Asset, CompileOptions, CompileResult, Diagnostic, Footer, Heading, Metadata, NavigationItem, Page, Site, SocialLink, Theme, } from "./types.js";
 /** Compile a local checkout; never fetch repositories, execute author code, or emit files. */

package/dist/search-index.js

@@ -7,8 +7,9 @@ import { transformAlerts } from "./alerts.js";
 import { validateSearchIndex, } from "./search.js";
 const parser = unified().use(remarkParse).use(remarkGfm);
 const blockTypes = new Set(["paragraph", "code", "tableCell", "listItem"]);
-function indexPage(page) {
+function indexPage(page, breadcrumbs) {
     const sections = [{ title: "", url: page.url, text: "" }];
+    const ancestors = [];
     let headingIndex = 0;
     const chunks = [[]];
     const tree = parser.parse(page.markdown);
@@ -23,11 +24,20 @@ function indexPage(page) {
                 heading.depth !== node.depth) {
                 throw new TypeError("Search indexing requires matching compiled Markdown and headings.");
             }
+            while (ancestors.length &&
+                (ancestors.at(-1)?.depth ?? 0) >= heading.depth)
+                ancestors.pop();
             sections.push({
                 title: heading.text,
                 url: `${page.url}#${encodeURIComponent(heading.id)}`,
                 text: "",
+                breadcrumbs: ancestors
+                    .filter((ancestor) => !(ancestor.depth === 1 &&
+                    ancestor.text.normalize("NFKC").toLowerCase() ===
+                        page.title.normalize("NFKC").toLowerCase()))
+                    .map((ancestor) => ancestor.text),
             });
+            ancestors.push(heading);
             chunks.push([]);
             return SKIP;
         }
@@ -55,12 +65,33 @@ function indexPage(page) {
         url: page.url,
         title: page.title,
         description: page.description ?? "",
+        breadcrumbs,
         sections,
     };
 }
 /** Build only from successfully compiled pages; no source paths or HTML are indexed. */
 export function createSearchIndex(site) {
-    const index = { version: 1, pages: site.pages.map(indexPage) };
+    const navigation = new Map();
+    const pending = site.navigation.map((item) => ({
+        item,
+        ancestors: [],
+    }));
+    while (pending.length) {
+        const entry = pending.pop();
+        if (!entry)
+            break;
+        const { item, ancestors } = entry;
+        if (ancestors.length > 64)
+            throw new TypeError("Search navigation exceeds 64 ancestor labels.");
+        if (item.url)
+            navigation.set(item.url, ancestors);
+        for (const child of item.children ?? [])
+            pending.push({ item: child, ancestors: [...ancestors, item.title] });
+    }
+    const index = {
+        version: 1,
+        pages: site.pages.map((page) => indexPage(page, navigation.get(page.url) ?? [])),
+    };
     index.pages.sort((a, b) => (a.url < b.url ? -1 : a.url > b.url ? 1 : 0));
     validateSearchIndex(index);
     return index;

package/dist/search.d.ts

@@ -7,6 +7,8 @@ export interface SearchPage {
     url: string;
     title: string;
     description: string;
+    /** Navigation ancestor labels, excluding this page's own label. */
+    breadcrumbs?: string[];
     sections: SearchSection[];
 }
 export interface SearchSection {
@@ -14,20 +16,37 @@ export interface SearchSection {
     title: string;
     url: string;
     text: string;
+    /** Ancestor headings, excluding this heading and a duplicate page-title H1. */
+    breadcrumbs?: string[];
 }
 export interface SearchOptions {
     /** Maximum results, from 0 to 100. Defaults to 10. */
     limit?: number;
+    /** Defaults to one result per page; sections returns independent heading hits. */
+    mode?: "pages" | "sections";
+    /** Allow one single-edit query correction per result. Requires sections mode. */
+    fuzzy?: boolean;
 }
+/** UTF-16 offsets into the original displayed field; end is exclusive. */
+export type SearchMatch = [start: number, end: number];
 export interface SearchResult {
+    kind: "page" | "section";
     title: string;
     url: string;
+    pageUrl: string;
+    breadcrumbs: string[];
     /** Present when the destination is a compiled heading. */
     section?: string;
     excerpt: string;
     score: number;
+    matches: {
+        title: SearchMatch[];
+        section: SearchMatch[];
+        excerpt: SearchMatch[];
+        breadcrumbs: SearchMatch[][];
+    };
 }
 /** Validate deserialized data before any result can become a link. */
 export declare function validateSearchIndex(value: unknown): asserts value is SearchIndex;
-/** Return at most one hit per page, without filesystem, network, or DOM access. */
+/** Query without filesystem, network, or DOM access; page mode preserves legacy ranking. */
 export declare function search(index: SearchIndex, query: string, options?: SearchOptions): SearchResult[];

package/dist/search.js

@@ -1,6 +1,17 @@
 function record(value) {
     return value !== null && typeof value === "object" && !Array.isArray(value);
 }
+function breadcrumbs(value, maximum) {
+    if (value === undefined)
+        return true;
+    if (!Array.isArray(value) || value.length > maximum)
+        return false;
+    // Iterate holes too: a sparse array does not contain only string labels.
+    for (const label of value)
+        if (typeof label !== "string")
+            return false;
+    return true;
+}
 function pageUrl(value) {
     if (typeof value !== "string" ||
         !value.startsWith("/") ||
@@ -48,6 +59,7 @@ export function validateSearchIndex(value) {
             urls.has(page.url) ||
             typeof page.title !== "string" ||
             typeof page.description !== "string" ||
+            !breadcrumbs(page.breadcrumbs, 64) ||
             !Array.isArray(page.sections))
             throw new TypeError("Invalid search index page.");
         urls.add(page.url);
@@ -57,7 +69,8 @@ export function validateSearchIndex(value) {
                 !sectionUrl(section.url, page.url) ||
                 sections.has(section.url) ||
                 typeof section.title !== "string" ||
-                typeof section.text !== "string")
+                typeof section.text !== "string" ||
+                !breadcrumbs(section.breadcrumbs, 6))
                 throw new TypeError("Invalid search index section.");
             sections.add(section.url);
         }
@@ -70,23 +83,51 @@ function normalize(value) {
         .replace(/[^\p{L}\p{N}\p{M}_]+/gu, " ")
         .trim();
 }
-/** Map a normalized match back to the original text, including NFKC expansions. */
-function matchRange(value, terms) {
+function contains(value, pattern) {
+    return pattern.whole
+        ? ` ${value} `.includes(` ${pattern.text} `)
+        : value.includes(pattern.text);
+}
+function mergeRanges(ranges) {
+    ranges.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
+    const merged = [];
+    for (const range of ranges) {
+        const previous = merged.at(-1);
+        if (previous && range[0] <= previous[1])
+            previous[1] = Math.max(previous[1], range[1]);
+        else
+            merged.push([...range]);
+    }
+    return merged;
+}
+/** Map normalized matches to complete original graphemes, including NFKC expansions. */
+function matchRanges(value, patterns, firstOnly = false) {
     const normalized = value.normalize("NFKC").toLowerCase();
-    let start = -1;
-    let end = -1;
-    for (const term of terms) {
-        const found = normalized.indexOf(term);
-        if (found >= 0 &&
-            (start < 0 ||
-                found < start ||
-                (found === start && found + term.length > end))) {
-            start = found;
-            end = found + term.length;
+    const found = [];
+    for (const pattern of patterns) {
+        let offset = 0;
+        while (offset < normalized.length) {
+            const start = normalized.indexOf(pattern.text, offset);
+            if (start < 0)
+                break;
+            const end = start + pattern.text.length;
+            // One neighboring code point needs at most two UTF-16 units. Avoid scanning
+            // a growing prefix for each occurrence, or any boundary work for substrings.
+            if (!pattern.whole ||
+                (!/[\p{L}\p{N}\p{M}_]$/u.test(normalized.slice(Math.max(0, start - 2), start)) &&
+                    !/^[\p{L}\p{N}\p{M}_]/u.test(normalized.slice(end, end + 2)))) {
+                found.push([start, end]);
+                if (firstOnly)
+                    break;
+            }
+            offset = start + 1;
         }
     }
-    if (start < 0)
-        return;
+    let ranges = mergeRanges(found);
+    if (!ranges.length)
+        return [];
+    if (firstOnly)
+        ranges = ranges.slice(0, 1);
     const groups = [];
     for (const { segment, index } of new Intl.Segmenter(undefined, {
         granularity: "grapheme",
@@ -110,25 +151,35 @@ function matchRange(value, terms) {
         }
         groups.push(group);
     }
-    // Match against whole-field lowercase for contextual letters such as sigma.
-    // Per-group lowercase is used only for lengths (including expansions like İ).
+    // Whole-field lowercase retains contextual letters such as final sigma.
+    // Per-group lowercase supplies only lengths, including expansions like İ.
+    const mapped = [];
     let offset = 0;
+    let rangeIndex = 0;
     let originalStart = 0;
     for (const group of groups) {
         const next = offset + group.text.toLowerCase().length;
-        if (offset <= start && start < next)
-            originalStart = group.start;
-        if (offset < end && end <= next)
-            return [originalStart, group.end];
+        let range = ranges[rangeIndex];
+        while (range && range[0] < next) {
+            if (range[0] >= offset)
+                originalStart = group.start;
+            if (range[1] > next)
+                break;
+            mapped.push([originalStart, group.end]);
+            range = ranges[++rangeIndex];
+        }
+        if (!range)
+            break;
         offset = next;
     }
+    return mergeRanges(mapped);
 }
-function excerpt(value, terms) {
+function excerpt(value, patterns) {
     const text = value.replace(/\s+/gu, " ").trim();
     const characters = Array.from(text);
     if (characters.length <= 160)
         return text;
-    const match = matchRange(text, terms);
+    const match = matchRanges(text, patterns, true)[0];
     const matchStart = match ? Array.from(text.slice(0, match[0])).length : 0;
     const matchLength = match
         ? Array.from(text.slice(match[0], match[1])).length
@@ -139,25 +190,45 @@ function excerpt(value, terms) {
     let end = Math.min(characters.length, start + 160 - (start > 0 ? 1 : 0));
     if (end < characters.length)
         end--;
-    return `${start > 0 ? "…" : ""}${characters.slice(start, end).join("")}${end < characters.length ? "…" : ""}`;
-}
-/** Return at most one hit per page, without filesystem, network, or DOM access. */
-export function search(index, query, options = {}) {
-    validateSearchIndex(index);
-    const limit = options.limit ?? 10;
-    if (!Number.isInteger(limit) || limit < 0 || limit > 100) {
-        throw new RangeError("Search limit must be an integer from 0 to 100.");
+    const startOffset = characters.slice(0, start).join("").length;
+    const endOffset = characters.slice(0, end).join("").length;
+    // Move clipped edges inward to complete graphemes without exceeding the cap.
+    let safeStart = text.length;
+    let safeEnd = 0;
+    for (const { segment, index } of new Intl.Segmenter(undefined, {
+        granularity: "grapheme",
+    }).segment(text)) {
+        if (index >= startOffset && safeStart === text.length)
+            safeStart = index;
+        if (index + segment.length <= endOffset)
+            safeEnd = index + segment.length;
+        if (index >= endOffset)
+            break;
     }
-    if (typeof query !== "string")
-        throw new TypeError("Search query must be a string.");
-    if (query.length > 512)
-        throw new RangeError("Search query must not exceed 512 characters.");
-    const terms = [...new Set(normalize(query).split(" ").filter(Boolean))];
+    return `${safeStart > 0 ? "…" : ""}${text.slice(safeStart, safeEnd)}${safeEnd < text.length ? "…" : ""}`;
+}
+function candidateUrl(candidate) {
+    return candidate.destination?.url ?? candidate.page.url;
+}
+function compareCandidates(a, b) {
+    const aUrl = candidateUrl(a);
+    const bUrl = candidateUrl(b);
+    return (b.rank - a.rank ||
+        b.score - a.score ||
+        (aUrl < bUrl ? -1 : aUrl > bUrl ? 1 : 0));
+}
+function sourceText(sources, patterns, fallback) {
+    return (sources.find((text) => {
+        const normalized = normalize(text);
+        return patterns.some((pattern) => contains(normalized, pattern));
+    }) ??
+        sources.find((text) => text) ??
+        fallback);
+}
+/** Preserve page-mode weighting, cross-section coverage, and destination selection. */
+function pageCandidates(index, terms) {
     const phrase = terms.join(" ");
-    if (terms.length > 32)
-        throw new RangeError("Search query must not exceed 32 unique terms.");
-    if (!terms.length || !limit)
-        return [];
+    const patterns = terms.map((text) => ({ text }));
     const results = [];
     for (const page of index.pages) {
         const title = normalize(page.title);
@@ -196,35 +267,219 @@ export function search(index, query, options = {}) {
         // A stable sort preserves document order when multiple sections tie.
         matches.sort((a, b) => b.score - a.score);
         const best = matches[0]?.section;
-        const destination = metadataMatch ? undefined : best;
-        const sources = [
-            best?.text ?? "",
-            page.description,
-            ...page.sections.map((section) => section.text),
-        ];
-        const source = sources.find((text) => {
-            const normalized = normalize(text);
-            return terms.some((term) => normalized.includes(term));
-        }) ??
-            sources.find((text) => text) ??
-            page.title;
         results.push({
-            title: page.title,
-            url: destination?.url ?? page.url,
-            ...(destination?.title ? { section: destination.title } : {}),
-            // Keep source text until ranking so only returned hits need a snippet.
-            excerpt: source,
+            page,
+            destination: metadataMatch ? undefined : best,
+            source: sourceText([
+                best?.text ?? "",
+                page.description,
+                ...page.sections.map((section) => section.text),
+            ], patterns, page.title),
             score: weights.reduce((sum, weight) => sum + weight, 0) +
                 (title === phrase
                     ? 24
                     : sections.some((section) => section.title === phrase)
                         ? 12
                         : 0),
+            rank: 0,
+            patterns,
         });
     }
-    results.sort((a, b) => b.score - a.score || (a.url < b.url ? -1 : a.url > b.url ? 1 : 0));
-    return results.slice(0, limit).map((result) => ({
-        ...result,
-        excerpt: excerpt(result.excerpt, terms),
-    }));
+    return results;
+}
+/** Single insertion, deletion, substitution, or adjacent transposition, on code points. */
+function singleEdit(query, token) {
+    if (token.length > 66)
+        return false;
+    const text = Array.from(token);
+    if (Math.abs(query.length - text.length) > 1)
+        return false;
+    let start = 0;
+    while (start < query.length && query[start] === text[start])
+        start++;
+    if (start === Math.min(query.length, text.length))
+        return true;
+    if (query.length === text.length) {
+        if (query.slice(start + 1).join("") === text.slice(start + 1).join(""))
+            return true;
+        return (query[start] === text[start + 1] &&
+            query[start + 1] === text[start] &&
+            query.slice(start + 2).join("") === text.slice(start + 2).join(""));
+    }
+    return query.length > text.length
+        ? query.slice(start + 1).join("") === text.slice(start).join("")
+        : query.slice(start).join("") === text.slice(start + 1).join("");
+}
+function fieldScore(fields, pattern) {
+    return Math.max(0, ...fields
+        .filter((field) => contains(field.text, pattern))
+        .map((field) => field.weight));
+}
+function matchFields(fields, terms, fuzzy, required = []) {
+    const patterns = terms.map((text) => ({ text }));
+    const weights = patterns.map((pattern) => fieldScore(fields, pattern));
+    const missing = weights.flatMap((weight, index) => (weight ? [] : [index]));
+    if (!missing.length)
+        return {
+            patterns,
+            score: weights.reduce((a, b) => a + b, 0),
+            corrected: false,
+        };
+    const missingIndex = missing[0];
+    if (!fuzzy || missing.length !== 1 || missingIndex === undefined)
+        return;
+    const query = Array.from(terms[missingIndex] ?? "");
+    if (query.length < 4 || query.length > 32)
+        return;
+    const needsOwnCorrection = required.length > 0 &&
+        !patterns.some((pattern) => fieldScore(required, pattern));
+    let correction;
+    let correctionWeight = 0;
+    for (const field of fields) {
+        if (field.weight < correctionWeight)
+            continue;
+        for (const token of field.text.split(" ")) {
+            if (singleEdit(query, token) &&
+                (!needsOwnCorrection ||
+                    fieldScore(required, { text: token, whole: true }) > 0) &&
+                (field.weight > correctionWeight ||
+                    correction === undefined ||
+                    token < correction)) {
+                correction = token;
+                correctionWeight = field.weight;
+            }
+        }
+    }
+    if (!correction)
+        return;
+    patterns[missingIndex] = { text: correction, whole: true };
+    weights[missingIndex] = correctionWeight;
+    return {
+        patterns,
+        score: weights.reduce((a, b) => a + b, 0),
+        corrected: true,
+    };
+}
+/** Score only page metadata plus a single coherent section, never sibling bodies. */
+function sectionCandidates(index, terms, fuzzy) {
+    const phrase = terms.join(" ");
+    const results = [];
+    for (const page of index.pages) {
+        const title = normalize(page.title);
+        const metadata = [
+            { text: title, weight: 8 },
+            { text: normalize(page.description), weight: 2 },
+        ];
+        const pageMatch = matchFields(metadata, terms, fuzzy);
+        let pageCandidate;
+        if (pageMatch) {
+            const exact = title === phrase;
+            pageCandidate = {
+                page,
+                source: sourceText([page.description, ...page.sections.map((section) => section.text)], pageMatch.patterns, page.title),
+                score: pageMatch.score + (exact ? 24 : 0),
+                rank: pageMatch.corrected ? 0 : exact ? 2 : 1,
+                patterns: pageMatch.patterns,
+            };
+        }
+        for (const section of page.sections) {
+            const heading = normalize(section.title);
+            const body = normalize(section.text);
+            const ownFields = [
+                { text: heading, weight: 4 },
+                { text: body, weight: 1 },
+            ];
+            const match = matchFields([...metadata, ...ownFields], terms, fuzzy, ownFields);
+            if (!match?.patterns.some((pattern) => fieldScore(ownFields, pattern)))
+                continue;
+            // Repeated page headings without their own matching body add no useful destination.
+            if (pageMatch &&
+                heading === title &&
+                !match.patterns.some((pattern) => contains(body, pattern)))
+                continue;
+            const exact = heading === phrase;
+            const candidate = {
+                page,
+                destination: section.url === page.url ? undefined : section,
+                source: sourceText([section.text, page.description], match.patterns, section.title || page.title),
+                score: match.score + (exact ? 12 : 0),
+                rank: match.corrected ? 0 : exact ? 2 : 1,
+                patterns: match.patterns,
+            };
+            if (candidate.destination)
+                results.push(candidate);
+            else if (!pageCandidate ||
+                compareCandidates(candidate, pageCandidate) < 0)
+                pageCandidate = candidate;
+        }
+        if (pageCandidate)
+            results.push(pageCandidate);
+    }
+    return results;
+}
+// Greater than the largest raw score: 32 terms × weight 8 + title bonus 24.
+const sectionTierWeight = 281;
+function displayResult(candidate) {
+    const { page, destination, patterns } = candidate;
+    const isSection = destination !== undefined && destination.url !== page.url;
+    const trail = isSection
+        ? [
+            ...(page.breadcrumbs ?? []),
+            page.title,
+            ...(destination.breadcrumbs ?? []),
+        ]
+        : [...(page.breadcrumbs ?? [])];
+    const snippet = excerpt(candidate.source, patterns);
+    return {
+        kind: isSection ? "section" : "page",
+        title: page.title,
+        pageUrl: page.url,
+        url: candidateUrl(candidate),
+        ...(destination?.title ? { section: destination.title } : {}),
+        breadcrumbs: trail,
+        excerpt: snippet,
+        score: candidate.rank * sectionTierWeight + candidate.score,
+        matches: {
+            title: matchRanges(page.title, patterns),
+            section: matchRanges(destination?.title ?? "", patterns),
+            excerpt: matchRanges(snippet, patterns),
+            breadcrumbs: trail.map((label) => matchRanges(label, patterns)),
+        },
+    };
+}
+/** Query without filesystem, network, or DOM access; page mode preserves legacy ranking. */
+export function search(index, query, options = {}) {
+    validateSearchIndex(index);
+    if (options === null || typeof options !== "object" || Array.isArray(options))
+        throw new TypeError("Search options must be an object.");
+    const limit = options.limit ?? 10;
+    const mode = options.mode === undefined ? "pages" : options.mode;
+    const fuzzy = options.fuzzy === undefined ? false : options.fuzzy;
+    if (!Number.isInteger(limit) || limit < 0 || limit > 100)
+        throw new RangeError("Search limit must be an integer from 0 to 100.");
+    if (mode !== "pages" && mode !== "sections")
+        throw new RangeError("Search mode must be pages or sections.");
+    if (typeof fuzzy !== "boolean")
+        throw new TypeError("Search fuzzy option must be a boolean.");
+    if (fuzzy && mode !== "sections")
+        throw new RangeError("Fuzzy search requires sections mode.");
+    if (typeof query !== "string")
+        throw new TypeError("Search query must be a string.");
+    if (query.length > 512)
+        throw new RangeError("Search query must not exceed 512 characters.");
+    const terms = [...new Set(normalize(query).split(" ").filter(Boolean))];
+    if (terms.length > 32)
+        throw new RangeError("Search query must not exceed 32 unique terms.");
+    if (!terms.length || !limit)
+        return [];
+    const results = mode === "sections"
+        ? sectionCandidates(index, terms, false)
+        : pageCandidates(index, terms);
+    if (fuzzy && results.length < limit) {
+        const literalUrls = new Set(results.map(candidateUrl));
+        results.push(...sectionCandidates(index, terms, true).filter((candidate) => candidate.rank === 0 && !literalUrls.has(candidateUrl(candidate))));
+    }
+    results.sort(compareCandidates);
+    // Original-text mapping and excerpt windows are computed only for final hits.
+    return results.slice(0, limit).map(displayResult);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wgtechlabs/mdd-engine",
-  "version": "0.1.0-pr.0cb765a",
+  "version": "0.1.0-pr.8fa2dc9",
   "description": "Headless Markdown documentation compiler for mdd",
   "type": "module",
   "license": "MIT",