@nbtca/docs 0.3.0 → 0.3.1

package/dist/client.js

@@ -165,9 +165,45 @@ function cacheTtl(value, fallback, name) {
 function isTransientStatus(status) {
     return status === 408 || status === 429 || status >= 500;
 }
+function isSecondaryRateLimitMessage(value) {
+    if (!isRecord(value) || typeof value.message !== 'string')
+        return false;
+    const message = value.message.toLowerCase().replace(/\s+/g, ' ').trim();
+    return (/\b(?:exceeded|hit|triggered) (?:a |the )?secondary rate limit\b/.test(message) ||
+        /\bsecondary rate limit (?:was |has been )?(?:exceeded|hit|triggered)\b/.test(message));
+}
+async function isTransientResponse(response) {
+    if (isTransientStatus(response.status))
+        return true;
+    if (response.status !== 403)
+        return false;
+    if (response.headers.get('x-ratelimit-remaining') === '0' ||
+        response.headers.get('retry-after') !== null) {
+        return true;
+    }
+    try {
+        return isSecondaryRateLimitMessage(await response.clone().json());
+    }
+    catch {
+        return false;
+    }
+}
 function reject(error) {
     return Promise.reject(error instanceof Error ? error : new Error('Operation failed with a non-error value'));
 }
+function cancelUnusedResponseBody(response) {
+    try {
+        if (!response || response.bodyUsed)
+            return;
+        const body = response.body;
+        if (!body)
+            return;
+        void body.cancel().catch(() => undefined);
+    }
+    catch {
+        // Cleanup must not override the request result.
+    }
+}
 function isRecord(value) {
     return typeof value === 'object' && value !== null;
 }
@@ -231,12 +267,15 @@ export function createDocsClient(options = {}) {
         const timer = setTimeout(() => {
             ctrl.abort();
         }, timeoutMs);
+        let response;
         try {
-            const response = await fetch(url, { signal: ctrl.signal, headers: headers() });
+            response = await fetch(url, { signal: ctrl.signal, headers: headers() });
             return await consume(response);
         }
         finally {
             clearTimeout(timer);
+            // Not awaited: a custom transport's cancel may never settle.
+            cancelUnusedResponseBody(response);
         }
     }
     function recoverFailure(cache, key, path, error, copy = (value) => value) {
@@ -265,7 +304,7 @@ export function createDocsClient(options = {}) {
             return await withResponse(url, 10000, async (response) => {
                 if (!response.ok) {
                     const stale = dirCache.getStale(path);
-                    if (isTransientStatus(response.status) && stale !== undefined)
+                    if (stale !== undefined && (await isTransientResponse(response)))
                         return copyItems(stale);
                     throw new DocsFetchError(path, response.status, `HTTP ${String(response.status)}`);
                 }
@@ -301,7 +340,7 @@ export function createDocsClient(options = {}) {
             return await withResponse(url, 20000, async (response) => {
                 if (!response.ok) {
                     const stale = treeCache.getStale(key);
-                    if (isTransientStatus(response.status) && stale !== undefined)
+                    if (stale !== undefined && (await isTransientResponse(response)))
                         return copyItems(stale);
                     throw new DocsFetchError('', response.status, `HTTP ${String(response.status)}`);
                 }
@@ -340,7 +379,7 @@ export function createDocsClient(options = {}) {
             return await withResponse(url, 15000, async (response) => {
                 if (!response.ok) {
                     const stale = fileCache.getStale(path);
-                    if (isTransientStatus(response.status) && stale !== undefined)
+                    if (stale !== undefined && (await isTransientResponse(response)))
                         return stale;
                     throw new DocsFetchError(path, response.status, `HTTP ${String(response.status)}`);
                 }

package/dist/content.js

@@ -1,5 +1,10 @@
 const SUMMARY_LENGTH = 160;
 const EXCERPT_LENGTH = 180;
+const GRAPHEME_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'grapheme' });
+const LIST_ITEM = /^( {0,3}(?:[-+*]|\d{1,9}[.)]))([ \t]+)/;
+const NORMALIZATION_CHUNK = 512;
+// NFKC never joins these characters to what precedes them, so chunks split before them.
+const NORMALIZATION_BOUNDARY = /[ -~\u4e00-\u9fff]/g;
 const COMPONENT_ATTRIBUTES = {
     Band: ['alt', 'source'],
     Figure: ['alt', 'caption', 'date', 'source'],
@@ -9,11 +14,12 @@ const COMPONENT_ATTRIBUTES = {
     TimelineEntry: ['year', 'title'],
 };
 function splitFrontmatter(content) {
-    const match = /^(?:\uFEFF)?---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/.exec(content);
+    const source = content.startsWith('\uFEFF') ? content.slice(1) : content;
+    const match = /^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/.exec(source);
     if (!match)
-        return { body: content, frontmatter: '' };
+        return { body: source, frontmatter: '' };
     return {
-        body: content.slice(match[0].length),
+        body: source.slice(match[0].length),
         frontmatter: match[1] ?? '',
     };
 }
@@ -57,18 +63,113 @@ function cleanInline(value) {
         .replace(/\s+/g, ' ')
         .trim();
 }
-function extractTitle(body) {
+function quoteContainer(line) {
+    let depth = 0;
+    let rest = line;
+    for (;;) {
+        const prefix = /^ {0,3}>[ \t]?/.exec(rest)?.[0];
+        if (!prefix)
+            return { depth, rest };
+        depth += 1;
+        rest = rest.slice(prefix.length);
+    }
+}
+function transitionFence(line, current) {
+    const quote = quoteContainer(line);
+    let candidate = quote.rest;
+    let listIndent = 0;
+    if (current) {
+        if (quote.depth < current.quoteDepth)
+            return transitionFence(line, undefined);
+        if (quote.depth !== current.quoteDepth)
+            return { delimiter: false, fence: current };
+        if (current.listIndent > 0) {
+            const indentation = /^ */.exec(candidate)?.[0].length ?? 0;
+            if (candidate.trim() !== '' && indentation < current.listIndent) {
+                return transitionFence(line, undefined);
+            }
+            candidate = candidate.slice(Math.min(indentation, current.listIndent));
+        }
+    }
+    else {
+        listIndent = LIST_ITEM.exec(candidate)?.[0].length ?? 0;
+        candidate = candidate.slice(listIndent);
+    }
+    const match = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(candidate);
+    const sequence = match?.[1];
+    if (!sequence)
+        return { delimiter: false, fence: current };
+    const marker = sequence.startsWith('`') ? '`' : '~';
+    const suffix = match[2] ?? '';
+    if (!current) {
+        if (marker === '`' && suffix.includes('`')) {
+            return { delimiter: false, fence: undefined };
+        }
+        return {
+            delimiter: true,
+            fence: { length: sequence.length, listIndent, marker, quoteDepth: quote.depth },
+        };
+    }
+    if (marker === current.marker && sequence.length >= current.length && /^[ \t]*$/.test(suffix)) {
+        return { delimiter: true, fence: undefined };
+    }
+    return { delimiter: false, fence: current };
+}
+function indentWidth(line) {
+    let width = 0;
+    for (const character of line) {
+        if (character === ' ')
+            width += 1;
+        else if (character === '\t')
+            width += 4 - (width % 4);
+        else
+            break;
+    }
+    return width;
+}
+function proseLines(body) {
     let fence;
-    for (const line of body.split(/\r?\n/)) {
-        const marker = /^\s*(`{3,}|~{3,})/.exec(line)?.[1]?.slice(0, 1);
-        if (marker) {
-            if (!fence)
-                fence = marker;
-            else if (marker === fence)
-                fence = undefined;
-            continue;
+    let afterBreak = true;
+    let inCode = false;
+    let listIndent = 0;
+    return body.split(/\r?\n/).map((line) => {
+        const transition = transitionFence(line, fence);
+        fence = transition.fence;
+        if (transition.delimiter || fence) {
+            afterBreak = true;
+            inCode = false;
+            return undefined;
         }
-        if (fence)
+        const candidate = quoteContainer(line).rest;
+        if (candidate.trim() === '') {
+            afterBreak = true;
+            return line;
+        }
+        const width = indentWidth(candidate);
+        if (afterBreak && width < listIndent)
+            listIndent = 0;
+        inCode = width - listIndent >= 4 && (afterBreak || inCode);
+        afterBreak = false;
+        if (inCode)
+            return undefined;
+        const item = LIST_ITEM.exec(candidate);
+        if (!item)
+            return line;
+        const marker = item[1]?.length ?? 0;
+        const padding = item[2] ?? '';
+        const rest = candidate.slice(item[0].length);
+        if (padding.includes('\t') || padding.length >= 5 || /^(?: {4}|\t)/.test(rest)) {
+            listIndent = marker + 1;
+            inCode = true;
+            return undefined;
+        }
+        listIndent = marker + padding.length;
+        return line;
+    });
+}
+function extractTitle(body) {
+    for (const line of proseLines(body)) {
+        if (line === undefined)
             continue;
         const match = /^#\s+(.+?)\s*$/.exec(line);
         if (match?.[1])
@@ -77,17 +178,19 @@ function extractTitle(body) {
     return undefined;
 }
 function truncate(value, length) {
-    const characters = [];
-    for (const character of value)
-        characters.push(character);
-    if (characters.length <= length)
-        return value;
-    return `${characters.slice(0, length).join('').trimEnd()}…`;
+    let end = 0;
+    let codePoints = 0;
+    for (const { index, segment } of GRAPHEME_SEGMENTER.segment(value)) {
+        codePoints += Array.from(segment).length;
+        if (codePoints > length)
+            return `${value.slice(0, end).trimEnd()}…`;
+        end = index + segment.length;
+    }
+    return value;
 }
 function extractSummary(body) {
     const paragraphs = [];
     let current = [];
-    let fence;
     let inContainer = false;
     let inTag = false;
     let hiddenTag;
@@ -96,19 +199,12 @@ function extractSummary(body) {
             paragraphs.push(current.join(' '));
         current = [];
     };
-    for (const rawLine of body.split(/\r?\n/)) {
-        const line = rawLine.trim();
-        const marker = /^(`{3,}|~{3,})/.exec(line)?.[1]?.slice(0, 1);
-        if (marker) {
-            if (!fence)
-                fence = marker;
-            else if (marker === fence)
-                fence = undefined;
+    for (const rawLine of proseLines(body)) {
+        if (rawLine === undefined) {
             finishParagraph();
             continue;
         }
-        if (fence)
-            continue;
+        const line = rawLine.trim();
         if (hiddenTag) {
             if (line.toLowerCase().includes(`</${hiddenTag}>`))
                 hiddenTag = undefined;
@@ -175,21 +271,10 @@ function parseAttributes(source) {
     return attributes;
 }
 function componentSource(body) {
-    const lines = [];
-    let fence;
-    for (const line of body.split(/\r?\n/)) {
-        const marker = /^\s*(`{3,}|~{3,})/.exec(line)?.[1]?.slice(0, 1);
-        if (marker) {
-            if (!fence)
-                fence = marker;
-            else if (marker === fence)
-                fence = undefined;
-            continue;
-        }
-        if (!fence)
-            lines.push(line);
-    }
-    return lines.join('\n').replace(/<!--[\s\S]*?-->/g, ' ');
+    return proseLines(body)
+        .filter((line) => line !== undefined)
+        .join('\n')
+        .replace(/<!--[\s\S]*?-->/g, ' ');
 }
 function extractComponents(body) {
     const components = [];
@@ -258,7 +343,75 @@ export function parseDoc(path, content) {
     };
 }
 function normalize(value) {
-    return value.normalize('NFKC').toLowerCase();
+    // Lowercasing keeps final sigma (ς) where a search for Σ yields σ.
+    return value
+        .normalize('NFKC')
+        .toLowerCase()
+        .replace(/\u03c2/g, '\u03c3');
+}
+function normalizeChunks(text) {
+    const offsets = [];
+    const sources = [];
+    const parts = [];
+    let length = 0;
+    for (let start = 0; start < text.length;) {
+        NORMALIZATION_BOUNDARY.lastIndex = start + NORMALIZATION_CHUNK;
+        const end = NORMALIZATION_BOUNDARY.exec(text)?.index ?? text.length;
+        const part = normalize(text.slice(start, end));
+        offsets.push(length);
+        sources.push(start);
+        parts.push(part);
+        length += part.length;
+        start = end;
+    }
+    return { offsets, sources, value: parts.join('') };
+}
+function lastIndexAtMost(count, target, valueAt) {
+    let low = 0;
+    let high = count - 1;
+    while (low < high) {
+        const middle = Math.ceil((low + high) / 2);
+        if (valueAt(middle) <= target)
+            low = middle;
+        else
+            high = middle - 1;
+    }
+    return low;
+}
+function graphemeAt(graphemes, position) {
+    const part = graphemes.containing(position);
+    if (!part)
+        return { codePoints: 0, end: position, start: position };
+    return {
+        codePoints: Array.from(part.segment).length,
+        end: part.index + part.segment.length,
+        start: part.index,
+    };
+}
+function normalizedLength(text, normalized, position) {
+    const { offsets, sources } = normalized;
+    const chunk = lastIndexAtMost(sources.length, position, (index) => sources[index] ?? 0);
+    const start = sources[chunk] ?? 0;
+    return (offsets[chunk] ?? 0) + normalize(text.slice(start, position)).length;
+}
+function sourceBoundary(text, normalized, graphemes, offset) {
+    const { offsets, sources } = normalized;
+    const chunk = lastIndexAtMost(offsets.length, offset, (index) => offsets[index] ?? 0);
+    const chunkEnd = sources[chunk + 1] ?? text.length;
+    const boundaries = [graphemeAt(graphemes, sources[chunk] ?? 0).start];
+    for (let position = boundaries[0] ?? 0; position < chunkEnd;) {
+        position = graphemeAt(graphemes, position).end;
+        boundaries.push(position);
+    }
+    const lengths = new Map();
+    const lengthAt = (index) => {
+        const position = boundaries[index] ?? 0;
+        const length = lengths.get(position) ?? normalizedLength(text, normalized, position);
+        lengths.set(position, length);
+        return length;
+    };
+    const index = lastIndexAtMost(boundaries.length, offset, lengthAt);
+    return { length: lengthAt(index), position: boundaries[index] ?? 0 };
 }
 function countMatches(value, term) {
     let count = 0;
@@ -275,18 +428,50 @@ function countMatches(value, term) {
 function excerpt(text, query, terms) {
     if (!text)
         return '';
-    const normalized = normalize(text);
-    const exactIndex = normalized.indexOf(query);
-    const matchIndex = exactIndex >= 0
-        ? exactIndex
-        : Math.min(...terms.map((term) => normalized.indexOf(term)).filter((index) => index >= 0));
+    const normalized = normalizeChunks(text);
+    const exactIndex = normalized.value.indexOf(query);
+    let matchIndex = exactIndex;
+    let matchLength = query.length;
+    if (matchIndex < 0) {
+        matchIndex = Number.POSITIVE_INFINITY;
+        for (const term of terms) {
+            const index = normalized.value.indexOf(term);
+            if (index >= 0 && index < matchIndex) {
+                matchIndex = index;
+                matchLength = term.length;
+            }
+        }
+    }
     if (!Number.isFinite(matchIndex))
         return truncate(text, EXCERPT_LENGTH);
-    const start = Math.max(0, matchIndex - Math.floor(EXCERPT_LENGTH / 3));
+    const graphemes = GRAPHEME_SEGMENTER.segment(text);
+    const matchStart = sourceBoundary(text, normalized, graphemes, matchIndex).position;
+    const endOffset = Math.min(normalized.value.length, matchIndex + matchLength);
+    const floor = sourceBoundary(text, normalized, graphemes, endOffset);
+    const matchEnd = floor.length < endOffset ? graphemeAt(graphemes, floor.position).end : floor.position;
+    const matchCodePoints = Array.from(text.slice(matchStart, matchEnd)).length;
+    const contextLimit = Math.min(Math.floor(EXCERPT_LENGTH / 3), Math.max(0, EXCERPT_LENGTH - matchCodePoints));
+    let start = matchStart;
+    let contextCodePoints = 0;
+    while (start > 0) {
+        const previous = graphemeAt(graphemes, start - 1);
+        if (contextCodePoints + previous.codePoints > contextLimit)
+            break;
+        contextCodePoints += previous.codePoints;
+        start = previous.start;
+    }
+    let end = start;
+    let selectedCodePoints = 0;
+    while (end < text.length) {
+        const next = graphemeAt(graphemes, end);
+        if (end > start && selectedCodePoints + next.codePoints > EXCERPT_LENGTH)
+            break;
+        selectedCodePoints += next.codePoints;
+        end = next.end;
+    }
     const prefix = start > 0 ? '…' : '';
-    const value = text.slice(start, start + EXCERPT_LENGTH).trim();
-    const suffix = start + EXCERPT_LENGTH < text.length ? '…' : '';
-    return `${prefix}${value}${suffix}`;
+    const suffix = end < text.length ? '…' : '';
+    return `${prefix}${text.slice(start, end).trim()}${suffix}`;
 }
 export function searchDoc(page, query) {
     const normalizedQuery = normalize(query.trim());

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nbtca/docs",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "description": "GitHub-backed document client for NBTCA",
   "type": "module",
   "main": "./dist/index.js",
@@ -60,6 +60,6 @@
     "typescript": "^5.9.3",
     "typescript-eslint": "^8.67.0",
     "vite": "6.4.3",
-    "vitest": "^3.2.7"
+    "vitest": "^4.1.11"
   }
 }