@mintlify/common 1.0.1138 → 1.0.1140

package/dist/search/highlight.d.ts

@@ -0,0 +1,7 @@
+export declare function getHighlightedText(text: string, query: string): string;
+export type HighlightSnippetOptions = {
+    leadingContext?: number;
+    maxLength?: number;
+};
+export declare function stripDuplicatedTitleLine(content: string, header: string): string;
+export declare function getHighlightedSnippet(text: string, query: string, options?: HighlightSnippetOptions): string;

package/dist/search/highlight.js

@@ -0,0 +1,400 @@
+const STOP_WORDS = new Set([
+    'i',
+    'me',
+    'my',
+    'myself',
+    'we',
+    'our',
+    'ours',
+    'ourselves',
+    'you',
+    'your',
+    'yours',
+    'yourself',
+    'yourselves',
+    'he',
+    'him',
+    'his',
+    'himself',
+    'she',
+    'her',
+    'hers',
+    'herself',
+    'it',
+    'its',
+    'itself',
+    'they',
+    'them',
+    'their',
+    'theirs',
+    'themselves',
+    'what',
+    'which',
+    'who',
+    'whom',
+    'this',
+    'that',
+    'these',
+    'those',
+    'am',
+    'is',
+    'are',
+    'was',
+    'were',
+    'be',
+    'been',
+    'being',
+    'have',
+    'has',
+    'had',
+    'having',
+    'do',
+    'does',
+    'did',
+    'doing',
+    'a',
+    'an',
+    'the',
+    'and',
+    'but',
+    'if',
+    'or',
+    'because',
+    'as',
+    'until',
+    'while',
+    'of',
+    'at',
+    'by',
+    'for',
+    'with',
+    'about',
+    'against',
+    'between',
+    'into',
+    'through',
+    'during',
+    'before',
+    'after',
+    'above',
+    'below',
+    'to',
+    'from',
+    'up',
+    'down',
+    'in',
+    'out',
+    'on',
+    'off',
+    'over',
+    'under',
+    'again',
+    'further',
+    'then',
+    'once',
+    'here',
+    'there',
+    'when',
+    'where',
+    'why',
+    'how',
+    'all',
+    'any',
+    'both',
+    'each',
+    'few',
+    'more',
+    'most',
+    'other',
+    'some',
+    'such',
+    'no',
+    'nor',
+    'not',
+    'only',
+    'own',
+    'same',
+    'so',
+    'than',
+    'too',
+    'very',
+    's',
+    't',
+    'can',
+    'will',
+    'just',
+    'don',
+    'should',
+    'now',
+    "aren't",
+    "it's",
+    "don't",
+    'get',
+]);
+const MAX_QUERY_WORDS = 20;
+const HIGHLIGHT_TIMEOUT_MS = 100;
+const PRE_TAG = '<mark><b>';
+const POST_TAG = '</b></mark>';
+function escapeHtml(str) {
+    return str
+        .replace(/&/g, '&amp;')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/"/g, '&quot;');
+}
+// Server-side search results arrive pre-wrapped with <mark><b>…</b></mark>
+// highlight tags. We re-highlight on the client now, so strip them here —
+// otherwise escapeHtml below would turn them into visible &lt;mark&gt; text.
+const SERVER_HIGHLIGHT_TAGS = /<\/?(?:mark|b)\b[^>]*>/gi;
+function stripServerHighlightTags(text) {
+    return text.replace(SERVER_HIGHLIGHT_TAGS, '');
+}
+function isStopWord(word) {
+    return STOP_WORDS.has(word.toLowerCase());
+}
+function chunkByNonStopWords(words) {
+    const groups = [];
+    let current = [];
+    for (const w of words) {
+        if (!w)
+            continue;
+        if (isStopWord(w)) {
+            if (current.length > 0) {
+                groups.push(current);
+                current = [];
+            }
+            continue;
+        }
+        current.push(w);
+    }
+    if (current.length > 0)
+        groups.push(current);
+    return groups;
+}
+function findAllCaseInsensitive(text, search) {
+    const lower = text.toLowerCase();
+    const searchLower = search.toLowerCase();
+    const indices = [];
+    let start = 0;
+    while (start <= lower.length - searchLower.length) {
+        const idx = lower.indexOf(searchLower, start);
+        if (idx === -1)
+            break;
+        indices.push(idx);
+        start = idx + 1;
+    }
+    return indices;
+}
+function generateQueryVariations(query) {
+    const cleanedQuery = query.replace(/[?,.!\n"]/g, '');
+    const queryWords = cleanedQuery.split(/\s+/).filter(Boolean).slice(0, MAX_QUERY_WORDS);
+    const groups = chunkByNonStopWords(queryWords);
+    const multiToken = [];
+    for (const group of groups) {
+        if (group.length > 1)
+            multiToken.push(group.join(' '));
+    }
+    const groupStarts = groups
+        .map((g) => cleanedQuery.indexOf(g.join(' ')))
+        .filter((i) => i >= 0)
+        .sort((a, b) => a - b);
+    for (let i = 0; i < groupStarts.length - 1; i++) {
+        const span = cleanedQuery.slice(groupStarts[i], groupStarts[i + 1]).trim();
+        const nonStopCount = span.split(/\s+/).filter((w) => !isStopWord(w)).length;
+        if (nonStopCount > 1) {
+            if (span.split(/\s+/).length > 1)
+                multiToken.push(span);
+        }
+    }
+    if (groupStarts.length > 0) {
+        const tail = cleanedQuery.slice(groupStarts[groupStarts.length - 1]).trim();
+        if (tail)
+            multiToken.push(tail);
+    }
+    for (let len = queryWords.length - 1; len > 0; len--) {
+        for (let skip = 0; skip <= queryWords.length - len; skip++) {
+            const sub = queryWords
+                .slice(skip, skip + len)
+                .map((w) => w.trim())
+                .join(' ');
+            const nonStopCount = sub.split(/\s+/).filter((w) => !isStopWord(w)).length;
+            if (len > 2 || nonStopCount >= 1) {
+                multiToken.push(sub);
+            }
+        }
+    }
+    const variations = [query, cleanedQuery, ...multiToken].filter((x) => x.trim().length > 0);
+    const seen = new Set();
+    const unique = [];
+    for (const v of variations) {
+        if (!seen.has(v)) {
+            seen.add(v);
+            unique.push(v);
+        }
+    }
+    const numWords = queryWords.length;
+    unique.sort((a, b) => {
+        if (numWords >= MAX_QUERY_WORDS)
+            return b.trim().length - a.trim().length;
+        const aWords = a.split(/\s+/).filter(Boolean).length;
+        const bWords = b.split(/\s+/).filter(Boolean).length;
+        if (bWords !== aWords)
+            return bWords - aWords;
+        return b.trim().length - a.trim().length;
+    });
+    return unique;
+}
+function computeMergedSpans(text, matchedQueries) {
+    var _a;
+    const spans = [];
+    const isWordChar = /\w/;
+    for (const q of matchedQueries) {
+        const trimmed = q.trim();
+        if (!trimmed)
+            continue;
+        const isSingleWord = !/\s/.test(trimmed);
+        for (const idx of findAllCaseInsensitive(text, trimmed)) {
+            const end = idx + trimmed.length;
+            if (isSingleWord) {
+                const beforeOk = idx === 0 || !isWordChar.test((_a = text[idx - 1]) !== null && _a !== void 0 ? _a : '');
+                if (!beforeOk)
+                    continue;
+            }
+            spans.push({ start: idx, end });
+        }
+    }
+    spans.sort((a, b) => a.start - b.start || b.end - a.end);
+    const mergedSpans = [];
+    for (const span of spans) {
+        const last = mergedSpans[mergedSpans.length - 1];
+        if (last && span.start <= last.end) {
+            last.end = Math.max(last.end, span.end);
+        }
+        else {
+            mergedSpans.push({ start: span.start, end: span.end });
+        }
+    }
+    return mergedSpans;
+}
+function renderSpans(text, mergedSpans) {
+    let result = '';
+    let cursor = 0;
+    for (const span of mergedSpans) {
+        result += escapeHtml(text.slice(cursor, span.start));
+        const original = text.slice(span.start, span.end);
+        if (original)
+            result += `${PRE_TAG}${escapeHtml(original)}${POST_TAG}`;
+        cursor = span.end;
+    }
+    result += escapeHtml(text.slice(cursor));
+    return result;
+}
+function renderHighlightedSpans(text, matchedQueries) {
+    return renderSpans(text, computeMergedSpans(text, matchedQueries));
+}
+const VARIATIONS_CACHE_LIMIT = 64;
+const variationsCache = new Map();
+function getCachedVariations(query) {
+    const cached = variationsCache.get(query);
+    if (cached)
+        return cached;
+    const variations = generateQueryVariations(query);
+    variationsCache.set(query, variations);
+    if (variationsCache.size > VARIATIONS_CACHE_LIMIT) {
+        const oldestKey = variationsCache.keys().next().value;
+        if (oldestKey !== undefined)
+            variationsCache.delete(oldestKey);
+    }
+    return variations;
+}
+function getMatchedQueries(text, query) {
+    const deadline = Date.now() + HIGHLIGHT_TIMEOUT_MS;
+    const variations = getCachedVariations(query);
+    const matchedQueries = [];
+    for (const variation of variations) {
+        if (Date.now() > deadline)
+            return [];
+        const trimmed = variation.trim();
+        if (matchedQueries.some((q) => q.trim() === trimmed))
+            continue;
+        if (findAllCaseInsensitive(text, variation).length > 0) {
+            matchedQueries.push(variation);
+        }
+    }
+    return matchedQueries;
+}
+export function getHighlightedText(text, query) {
+    const stripped = stripServerHighlightTags(text);
+    if (!stripped || !query)
+        return escapeHtml(stripped);
+    const matchedQueries = getMatchedQueries(stripped, query);
+    if (matchedQueries.length === 0)
+        return escapeHtml(stripped);
+    return renderHighlightedSpans(stripped, matchedQueries);
+}
+const ELLIPSIS = '…';
+const SNIPPET_LEADING_CONTEXT = 24;
+const SNIPPET_MAX_LENGTH = 240;
+const SNIPPET_BOUNDARY_LOOKAROUND = 12;
+function snapStartToWordBoundary(text, index, lookaround) {
+    if (index <= 0)
+        return 0;
+    const earliest = Math.max(0, index - lookaround);
+    const previousSpace = text.lastIndexOf(' ', index - 1);
+    return previousSpace >= earliest ? previousSpace + 1 : index;
+}
+function snapEndToWordBoundary(text, index, lookaround) {
+    if (index >= text.length)
+        return text.length;
+    const latest = Math.min(text.length, index + lookaround);
+    const nextSpace = text.indexOf(' ', index);
+    return nextSpace !== -1 && nextSpace < latest ? nextSpace : index;
+}
+function leadingWindow(text, maxLength) {
+    if (text.length <= maxLength)
+        return escapeHtml(text);
+    const end = snapEndToWordBoundary(text, maxLength, SNIPPET_BOUNDARY_LOOKAROUND);
+    return `${escapeHtml(text.slice(0, end))} ${ELLIPSIS}`;
+}
+export function stripDuplicatedTitleLine(content, header) {
+    const newlineIdx = content.indexOf('\n');
+    if (newlineIdx < 0 || !header)
+        return content;
+    const normalize = (value) => stripServerHighlightTags(value).replace(/\s+/g, ' ').trim();
+    return normalize(content.slice(0, newlineIdx)) === normalize(header)
+        ? content.slice(newlineIdx + 1)
+        : content;
+}
+export function getHighlightedSnippet(text, query, options) {
+    var _a, _b;
+    const stripped = stripServerHighlightTags(text);
+    const leadingContext = (_a = options === null || options === void 0 ? void 0 : options.leadingContext) !== null && _a !== void 0 ? _a : SNIPPET_LEADING_CONTEXT;
+    const maxLength = (_b = options === null || options === void 0 ? void 0 : options.maxLength) !== null && _b !== void 0 ? _b : SNIPPET_MAX_LENGTH;
+    if (!stripped || !query)
+        return leadingWindow(stripped, maxLength);
+    const matchedQueries = getMatchedQueries(stripped, query);
+    if (matchedQueries.length === 0)
+        return leadingWindow(stripped, maxLength);
+    const mergedSpans = computeMergedSpans(stripped, matchedQueries);
+    const firstMatch = mergedSpans[0];
+    if (!firstMatch)
+        return leadingWindow(stripped, maxLength);
+    let windowStart = firstMatch.start <= leadingContext ? 0 : firstMatch.start - leadingContext;
+    let windowEnd = Math.min(stripped.length, Math.max(firstMatch.end, windowStart + maxLength));
+    windowStart = snapStartToWordBoundary(stripped, windowStart, SNIPPET_BOUNDARY_LOOKAROUND);
+    windowEnd = snapEndToWordBoundary(stripped, windowEnd, SNIPPET_BOUNDARY_LOOKAROUND);
+    const windowed = stripped.slice(windowStart, windowEnd);
+    const windowedSpans = [];
+    for (const span of mergedSpans) {
+        if (span.end <= windowStart || span.start >= windowEnd)
+            continue;
+        windowedSpans.push({
+            start: Math.max(0, span.start - windowStart),
+            end: Math.min(windowed.length, span.end - windowStart),
+        });
+    }
+    const body = renderSpans(windowed, windowedSpans);
+    const lead = windowStart > 0 ? `${ELLIPSIS} ` : '';
+    const trail = windowEnd < stripped.length ? ` ${ELLIPSIS}` : '';
+    return `${lead}${body}${trail}`;
+}

package/dist/search/highlight.test.d.ts

@@ -0,0 +1 @@
+export {};