@nbtca/docs 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -1
- package/dist/client.js +130 -10
- package/dist/content.js +240 -55
- package/dist/index.d.ts +1 -1
- package/dist/types.d.ts +7 -0
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -36,6 +36,7 @@ const matches = await docs.search('repair', { pathPrefix: 'repair' });
|
|
|
36
36
|
| `token` | `GITHUB_TOKEN` or `GH_TOKEN` | GitHub token |
|
|
37
37
|
| `cacheTtlMs.dir` | `300000` | Directory and tree cache TTL |
|
|
38
38
|
| `cacheTtlMs.file` | `600000` | File cache TTL |
|
|
39
|
+
| `store` | none | Persistent cache, see below |
|
|
39
40
|
|
|
40
41
|
### `docs.listDir(path?)`
|
|
41
42
|
|
|
@@ -48,7 +49,12 @@ Returns raw file content.
|
|
|
48
49
|
|
|
49
50
|
### `docs.listAll()`
|
|
50
51
|
|
|
51
|
-
Lists every Markdown file through GitHub's recursive tree API.
|
|
52
|
+
Lists every Markdown file through GitHub's recursive tree API. Each item carries its Git blob `sha`.
|
|
53
|
+
|
|
54
|
+
### `docs.peekAll()`
|
|
55
|
+
|
|
56
|
+
Returns the last known tree without a request: the one fetched in this process, else the one in
|
|
57
|
+
`store`, else `undefined`.
|
|
52
58
|
|
|
53
59
|
### `docs.listSections()`
|
|
54
60
|
|
|
@@ -60,6 +66,13 @@ Returns content with its route, section, title, summary, and semantic component
|
|
|
60
66
|
metadata covers `PageHero`, `FactStrip`, `LinkCard`, `Split`, `TimelineEntry`, and `Figure` without
|
|
61
67
|
imposing a renderer.
|
|
62
68
|
|
|
69
|
+
### `store`
|
|
70
|
+
|
|
71
|
+
An object with `read(key): string | undefined` and `write(key, value): void`. The client keeps the
|
|
72
|
+
tree under `tree` and file content under `blob-<sha>`, where `<sha>` is the Git blob id. `getFile`
|
|
73
|
+
serves stored content only when its hash matches the blob id in the last known tree, so a changed
|
|
74
|
+
file is always refetched. Store errors and corrupt entries are ignored; eviction is up to the store.
|
|
75
|
+
|
|
63
76
|
### `docs.search(query, options?)`
|
|
64
77
|
|
|
65
78
|
Searches paths, titles, summaries, Markdown text, and semantic component attributes. Results are
|
package/dist/client.js
CHANGED
|
@@ -30,6 +30,8 @@ const SKIP = new Set([
|
|
|
30
30
|
'README.md',
|
|
31
31
|
'docs',
|
|
32
32
|
]);
|
|
33
|
+
const TREE_KEY = 'tree';
|
|
34
|
+
const SHA = /^[0-9a-f]{40}$/;
|
|
33
35
|
const SEARCH_CONCURRENCY = 6;
|
|
34
36
|
const SEARCH_RESULT_LIMIT = 20;
|
|
35
37
|
function filterAndSort(raw) {
|
|
@@ -41,6 +43,7 @@ function filterAndSort(raw) {
|
|
|
41
43
|
name: item.name,
|
|
42
44
|
path: item.path,
|
|
43
45
|
type: item.type === 'dir' ? 'dir' : 'file',
|
|
46
|
+
...shaOf(item),
|
|
44
47
|
}))
|
|
45
48
|
.sort((a, b) => {
|
|
46
49
|
if (a.type !== b.type)
|
|
@@ -60,9 +63,40 @@ function filterTree(items) {
|
|
|
60
63
|
name: item.path.slice(item.path.lastIndexOf('/') + 1),
|
|
61
64
|
path: item.path,
|
|
62
65
|
type: 'file',
|
|
66
|
+
...shaOf(item),
|
|
63
67
|
}))
|
|
64
68
|
.sort((a, b) => a.path.localeCompare(b.path));
|
|
65
69
|
}
|
|
70
|
+
function shaOf(item) {
|
|
71
|
+
return typeof item.sha === 'string' && SHA.test(item.sha) ? { sha: item.sha } : {};
|
|
72
|
+
}
|
|
73
|
+
async function blobSha(content) {
|
|
74
|
+
const body = new TextEncoder().encode(content);
|
|
75
|
+
const header = new TextEncoder().encode(`blob ${String(body.length)}\0`);
|
|
76
|
+
const object = new Uint8Array(header.length + body.length);
|
|
77
|
+
object.set(header);
|
|
78
|
+
object.set(body, header.length);
|
|
79
|
+
const digest = new Uint8Array(await crypto.subtle.digest('SHA-1', object));
|
|
80
|
+
return Array.from(digest, (byte) => byte.toString(16).padStart(2, '0')).join('');
|
|
81
|
+
}
|
|
82
|
+
function isDocItem(value) {
|
|
83
|
+
return (isRecord(value) &&
|
|
84
|
+
typeof value.name === 'string' &&
|
|
85
|
+
typeof value.path === 'string' &&
|
|
86
|
+
(value.type === 'file' || value.type === 'dir') &&
|
|
87
|
+
(value.sha === undefined || typeof value.sha === 'string'));
|
|
88
|
+
}
|
|
89
|
+
function parseStoredTree(value) {
|
|
90
|
+
if (value === undefined)
|
|
91
|
+
return undefined;
|
|
92
|
+
try {
|
|
93
|
+
const items = JSON.parse(value);
|
|
94
|
+
return Array.isArray(items) && items.every(isDocItem) ? items : undefined;
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
return undefined;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
66
100
|
function copyItems(items) {
|
|
67
101
|
return items.map((item) => ({ ...item }));
|
|
68
102
|
}
|
|
@@ -165,9 +199,45 @@ function cacheTtl(value, fallback, name) {
|
|
|
165
199
|
function isTransientStatus(status) {
|
|
166
200
|
return status === 408 || status === 429 || status >= 500;
|
|
167
201
|
}
|
|
202
|
+
function isSecondaryRateLimitMessage(value) {
|
|
203
|
+
if (!isRecord(value) || typeof value.message !== 'string')
|
|
204
|
+
return false;
|
|
205
|
+
const message = value.message.toLowerCase().replace(/\s+/g, ' ').trim();
|
|
206
|
+
return (/\b(?:exceeded|hit|triggered) (?:a |the )?secondary rate limit\b/.test(message) ||
|
|
207
|
+
/\bsecondary rate limit (?:was |has been )?(?:exceeded|hit|triggered)\b/.test(message));
|
|
208
|
+
}
|
|
209
|
+
async function isTransientResponse(response) {
|
|
210
|
+
if (isTransientStatus(response.status))
|
|
211
|
+
return true;
|
|
212
|
+
if (response.status !== 403)
|
|
213
|
+
return false;
|
|
214
|
+
if (response.headers.get('x-ratelimit-remaining') === '0' ||
|
|
215
|
+
response.headers.get('retry-after') !== null) {
|
|
216
|
+
return true;
|
|
217
|
+
}
|
|
218
|
+
try {
|
|
219
|
+
return isSecondaryRateLimitMessage(await response.clone().json());
|
|
220
|
+
}
|
|
221
|
+
catch {
|
|
222
|
+
return false;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
168
225
|
function reject(error) {
|
|
169
226
|
return Promise.reject(error instanceof Error ? error : new Error('Operation failed with a non-error value'));
|
|
170
227
|
}
|
|
228
|
+
function cancelUnusedResponseBody(response) {
|
|
229
|
+
try {
|
|
230
|
+
if (!response || response.bodyUsed)
|
|
231
|
+
return;
|
|
232
|
+
const body = response.body;
|
|
233
|
+
if (!body)
|
|
234
|
+
return;
|
|
235
|
+
void body.cancel().catch(() => undefined);
|
|
236
|
+
}
|
|
237
|
+
catch {
|
|
238
|
+
// Cleanup must not override the request result.
|
|
239
|
+
}
|
|
240
|
+
}
|
|
171
241
|
function isRecord(value) {
|
|
172
242
|
return typeof value === 'object' && value !== null;
|
|
173
243
|
}
|
|
@@ -217,7 +287,32 @@ export function createDocsClient(options = {}) {
|
|
|
217
287
|
const dirRequests = new Map();
|
|
218
288
|
const fileRequests = new Map();
|
|
219
289
|
const treeRequests = new Map();
|
|
290
|
+
const store = options.store;
|
|
291
|
+
let storedTree;
|
|
220
292
|
let cacheGeneration = 0;
|
|
293
|
+
function storeRead(key) {
|
|
294
|
+
try {
|
|
295
|
+
return store?.read(key);
|
|
296
|
+
}
|
|
297
|
+
catch {
|
|
298
|
+
return undefined;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
function storeWrite(key, value) {
|
|
302
|
+
try {
|
|
303
|
+
store?.write(key, value);
|
|
304
|
+
}
|
|
305
|
+
catch {
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
function knownTree() {
|
|
310
|
+
const fetched = treeCache.getStale(TREE_KEY);
|
|
311
|
+
if (fetched)
|
|
312
|
+
return fetched;
|
|
313
|
+
storedTree ?? (storedTree = parseStoredTree(storeRead(TREE_KEY)));
|
|
314
|
+
return storedTree;
|
|
315
|
+
}
|
|
221
316
|
function headers() {
|
|
222
317
|
const requestHeaders = {
|
|
223
318
|
Accept: 'application/vnd.github.v3+json',
|
|
@@ -231,12 +326,15 @@ export function createDocsClient(options = {}) {
|
|
|
231
326
|
const timer = setTimeout(() => {
|
|
232
327
|
ctrl.abort();
|
|
233
328
|
}, timeoutMs);
|
|
329
|
+
let response;
|
|
234
330
|
try {
|
|
235
|
-
|
|
331
|
+
response = await fetch(url, { signal: ctrl.signal, headers: headers() });
|
|
236
332
|
return await consume(response);
|
|
237
333
|
}
|
|
238
334
|
finally {
|
|
239
335
|
clearTimeout(timer);
|
|
336
|
+
// Not awaited: a custom transport's cancel may never settle.
|
|
337
|
+
cancelUnusedResponseBody(response);
|
|
240
338
|
}
|
|
241
339
|
}
|
|
242
340
|
function recoverFailure(cache, key, path, error, copy = (value) => value) {
|
|
@@ -265,7 +363,7 @@ export function createDocsClient(options = {}) {
|
|
|
265
363
|
return await withResponse(url, 10000, async (response) => {
|
|
266
364
|
if (!response.ok) {
|
|
267
365
|
const stale = dirCache.getStale(path);
|
|
268
|
-
if (
|
|
366
|
+
if (stale !== undefined && (await isTransientResponse(response)))
|
|
269
367
|
return copyItems(stale);
|
|
270
368
|
throw new DocsFetchError(path, response.status, `HTTP ${String(response.status)}`);
|
|
271
369
|
}
|
|
@@ -295,13 +393,13 @@ export function createDocsClient(options = {}) {
|
|
|
295
393
|
return shareRequest(dirRequests, path, () => loadDir(path, cacheGeneration)).then(copyItems);
|
|
296
394
|
}
|
|
297
395
|
async function loadAll(generation) {
|
|
298
|
-
const key =
|
|
396
|
+
const key = TREE_KEY;
|
|
299
397
|
const url = `${apiRepoUrl}/git/trees/${encodedBranch}?recursive=1`;
|
|
300
398
|
try {
|
|
301
399
|
return await withResponse(url, 20000, async (response) => {
|
|
302
400
|
if (!response.ok) {
|
|
303
401
|
const stale = treeCache.getStale(key);
|
|
304
|
-
if (
|
|
402
|
+
if (stale !== undefined && (await isTransientResponse(response)))
|
|
305
403
|
return copyItems(stale);
|
|
306
404
|
throw new DocsFetchError('', response.status, `HTTP ${String(response.status)}`);
|
|
307
405
|
}
|
|
@@ -313,8 +411,11 @@ export function createDocsClient(options = {}) {
|
|
|
313
411
|
throw new DocsFetchError('', null, 'GitHub truncated the repository tree (too many files) -- results would be incomplete');
|
|
314
412
|
}
|
|
315
413
|
const items = filterTree(data.tree);
|
|
316
|
-
if (generation === cacheGeneration)
|
|
414
|
+
if (generation === cacheGeneration) {
|
|
317
415
|
treeCache.set(key, copyItems(items));
|
|
416
|
+
if (store)
|
|
417
|
+
storeWrite(key, JSON.stringify(items));
|
|
418
|
+
}
|
|
318
419
|
return items;
|
|
319
420
|
});
|
|
320
421
|
}
|
|
@@ -325,28 +426,46 @@ export function createDocsClient(options = {}) {
|
|
|
325
426
|
}
|
|
326
427
|
}
|
|
327
428
|
function listAll() {
|
|
328
|
-
const
|
|
329
|
-
const hit = treeCache.get(key);
|
|
429
|
+
const hit = treeCache.get(TREE_KEY);
|
|
330
430
|
if (hit)
|
|
331
431
|
return Promise.resolve(copyItems(hit));
|
|
332
|
-
return shareRequest(treeRequests,
|
|
432
|
+
return shareRequest(treeRequests, TREE_KEY, () => loadAll(cacheGeneration)).then(copyItems);
|
|
433
|
+
}
|
|
434
|
+
function peekAll() {
|
|
435
|
+
const items = knownTree();
|
|
436
|
+
return items && copyItems(items);
|
|
333
437
|
}
|
|
334
438
|
async function listSections() {
|
|
335
439
|
return sectionsFromItems(await listAll());
|
|
336
440
|
}
|
|
441
|
+
async function loadStoredFile(path) {
|
|
442
|
+
const sha = knownTree()?.find((item) => item.path === path)?.sha;
|
|
443
|
+
if (sha === undefined)
|
|
444
|
+
return undefined;
|
|
445
|
+
const content = storeRead(`blob-${sha}`);
|
|
446
|
+
return content !== undefined && (await blobSha(content)) === sha ? content : undefined;
|
|
447
|
+
}
|
|
337
448
|
async function loadFile(path, generation) {
|
|
449
|
+
const stored = store && (await loadStoredFile(path));
|
|
450
|
+
if (stored !== undefined) {
|
|
451
|
+
if (generation === cacheGeneration)
|
|
452
|
+
fileCache.set(path, stored);
|
|
453
|
+
return stored;
|
|
454
|
+
}
|
|
338
455
|
const url = `${rawRepoUrl}/${encodedBranch}/${encodePath(path)}`;
|
|
339
456
|
try {
|
|
340
457
|
return await withResponse(url, 15000, async (response) => {
|
|
341
458
|
if (!response.ok) {
|
|
342
459
|
const stale = fileCache.getStale(path);
|
|
343
|
-
if (
|
|
460
|
+
if (stale !== undefined && (await isTransientResponse(response)))
|
|
344
461
|
return stale;
|
|
345
462
|
throw new DocsFetchError(path, response.status, `HTTP ${String(response.status)}`);
|
|
346
463
|
}
|
|
347
464
|
const content = await response.text();
|
|
348
465
|
if (generation === cacheGeneration)
|
|
349
466
|
fileCache.set(path, content);
|
|
467
|
+
if (store)
|
|
468
|
+
storeWrite(`blob-${await blobSha(content)}`, content);
|
|
350
469
|
return content;
|
|
351
470
|
});
|
|
352
471
|
}
|
|
@@ -412,6 +531,7 @@ export function createDocsClient(options = {}) {
|
|
|
412
531
|
}
|
|
413
532
|
function clear() {
|
|
414
533
|
cacheGeneration += 1;
|
|
534
|
+
storedTree = undefined;
|
|
415
535
|
dirCache.clear();
|
|
416
536
|
fileCache.clear();
|
|
417
537
|
treeCache.clear();
|
|
@@ -419,5 +539,5 @@ export function createDocsClient(options = {}) {
|
|
|
419
539
|
fileRequests.clear();
|
|
420
540
|
treeRequests.clear();
|
|
421
541
|
}
|
|
422
|
-
return { listDir, listAll, listSections, getFile, getDocument, search, clear };
|
|
542
|
+
return { listDir, listAll, peekAll, listSections, getFile, getDocument, search, clear };
|
|
423
543
|
}
|
package/dist/content.js
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
const SUMMARY_LENGTH = 160;
|
|
2
2
|
const EXCERPT_LENGTH = 180;
|
|
3
|
+
const GRAPHEME_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'grapheme' });
|
|
4
|
+
const LIST_ITEM = /^( {0,3}(?:[-+*]|\d{1,9}[.)]))([ \t]+)/;
|
|
5
|
+
const NORMALIZATION_CHUNK = 512;
|
|
6
|
+
// NFKC never joins these characters to what precedes them, so chunks split before them.
|
|
7
|
+
const NORMALIZATION_BOUNDARY = /[ -~\u4e00-\u9fff]/g;
|
|
3
8
|
const COMPONENT_ATTRIBUTES = {
|
|
4
9
|
Band: ['alt', 'source'],
|
|
5
10
|
Figure: ['alt', 'caption', 'date', 'source'],
|
|
@@ -9,11 +14,12 @@ const COMPONENT_ATTRIBUTES = {
|
|
|
9
14
|
TimelineEntry: ['year', 'title'],
|
|
10
15
|
};
|
|
11
16
|
function splitFrontmatter(content) {
|
|
12
|
-
const
|
|
17
|
+
const source = content.startsWith('\uFEFF') ? content.slice(1) : content;
|
|
18
|
+
const match = /^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/.exec(source);
|
|
13
19
|
if (!match)
|
|
14
|
-
return { body:
|
|
20
|
+
return { body: source, frontmatter: '' };
|
|
15
21
|
return {
|
|
16
|
-
body:
|
|
22
|
+
body: source.slice(match[0].length),
|
|
17
23
|
frontmatter: match[1] ?? '',
|
|
18
24
|
};
|
|
19
25
|
}
|
|
@@ -57,18 +63,113 @@ function cleanInline(value) {
|
|
|
57
63
|
.replace(/\s+/g, ' ')
|
|
58
64
|
.trim();
|
|
59
65
|
}
|
|
60
|
-
function
|
|
66
|
+
function quoteContainer(line) {
|
|
67
|
+
let depth = 0;
|
|
68
|
+
let rest = line;
|
|
69
|
+
for (;;) {
|
|
70
|
+
const prefix = /^ {0,3}>[ \t]?/.exec(rest)?.[0];
|
|
71
|
+
if (!prefix)
|
|
72
|
+
return { depth, rest };
|
|
73
|
+
depth += 1;
|
|
74
|
+
rest = rest.slice(prefix.length);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
function transitionFence(line, current) {
|
|
78
|
+
const quote = quoteContainer(line);
|
|
79
|
+
let candidate = quote.rest;
|
|
80
|
+
let listIndent = 0;
|
|
81
|
+
if (current) {
|
|
82
|
+
if (quote.depth < current.quoteDepth)
|
|
83
|
+
return transitionFence(line, undefined);
|
|
84
|
+
if (quote.depth !== current.quoteDepth)
|
|
85
|
+
return { delimiter: false, fence: current };
|
|
86
|
+
if (current.listIndent > 0) {
|
|
87
|
+
const indentation = /^ */.exec(candidate)?.[0].length ?? 0;
|
|
88
|
+
if (candidate.trim() !== '' && indentation < current.listIndent) {
|
|
89
|
+
return transitionFence(line, undefined);
|
|
90
|
+
}
|
|
91
|
+
candidate = candidate.slice(Math.min(indentation, current.listIndent));
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
else {
|
|
95
|
+
listIndent = LIST_ITEM.exec(candidate)?.[0].length ?? 0;
|
|
96
|
+
candidate = candidate.slice(listIndent);
|
|
97
|
+
}
|
|
98
|
+
const match = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(candidate);
|
|
99
|
+
const sequence = match?.[1];
|
|
100
|
+
if (!sequence)
|
|
101
|
+
return { delimiter: false, fence: current };
|
|
102
|
+
const marker = sequence.startsWith('`') ? '`' : '~';
|
|
103
|
+
const suffix = match[2] ?? '';
|
|
104
|
+
if (!current) {
|
|
105
|
+
if (marker === '`' && suffix.includes('`')) {
|
|
106
|
+
return { delimiter: false, fence: undefined };
|
|
107
|
+
}
|
|
108
|
+
return {
|
|
109
|
+
delimiter: true,
|
|
110
|
+
fence: { length: sequence.length, listIndent, marker, quoteDepth: quote.depth },
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
if (marker === current.marker && sequence.length >= current.length && /^[ \t]*$/.test(suffix)) {
|
|
114
|
+
return { delimiter: true, fence: undefined };
|
|
115
|
+
}
|
|
116
|
+
return { delimiter: false, fence: current };
|
|
117
|
+
}
|
|
118
|
+
function indentWidth(line) {
|
|
119
|
+
let width = 0;
|
|
120
|
+
for (const character of line) {
|
|
121
|
+
if (character === ' ')
|
|
122
|
+
width += 1;
|
|
123
|
+
else if (character === '\t')
|
|
124
|
+
width += 4 - (width % 4);
|
|
125
|
+
else
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
return width;
|
|
129
|
+
}
|
|
130
|
+
function proseLines(body) {
|
|
61
131
|
let fence;
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
132
|
+
let afterBreak = true;
|
|
133
|
+
let inCode = false;
|
|
134
|
+
let listIndent = 0;
|
|
135
|
+
return body.split(/\r?\n/).map((line) => {
|
|
136
|
+
const transition = transitionFence(line, fence);
|
|
137
|
+
fence = transition.fence;
|
|
138
|
+
if (transition.delimiter || fence) {
|
|
139
|
+
afterBreak = true;
|
|
140
|
+
inCode = false;
|
|
141
|
+
return undefined;
|
|
70
142
|
}
|
|
71
|
-
|
|
143
|
+
const candidate = quoteContainer(line).rest;
|
|
144
|
+
if (candidate.trim() === '') {
|
|
145
|
+
afterBreak = true;
|
|
146
|
+
return line;
|
|
147
|
+
}
|
|
148
|
+
const width = indentWidth(candidate);
|
|
149
|
+
if (afterBreak && width < listIndent)
|
|
150
|
+
listIndent = 0;
|
|
151
|
+
inCode = width - listIndent >= 4 && (afterBreak || inCode);
|
|
152
|
+
afterBreak = false;
|
|
153
|
+
if (inCode)
|
|
154
|
+
return undefined;
|
|
155
|
+
const item = LIST_ITEM.exec(candidate);
|
|
156
|
+
if (!item)
|
|
157
|
+
return line;
|
|
158
|
+
const marker = item[1]?.length ?? 0;
|
|
159
|
+
const padding = item[2] ?? '';
|
|
160
|
+
const rest = candidate.slice(item[0].length);
|
|
161
|
+
if (padding.includes('\t') || padding.length >= 5 || /^(?: {4}|\t)/.test(rest)) {
|
|
162
|
+
listIndent = marker + 1;
|
|
163
|
+
inCode = true;
|
|
164
|
+
return undefined;
|
|
165
|
+
}
|
|
166
|
+
listIndent = marker + padding.length;
|
|
167
|
+
return line;
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
function extractTitle(body) {
|
|
171
|
+
for (const line of proseLines(body)) {
|
|
172
|
+
if (line === undefined)
|
|
72
173
|
continue;
|
|
73
174
|
const match = /^#\s+(.+?)\s*$/.exec(line);
|
|
74
175
|
if (match?.[1])
|
|
@@ -77,17 +178,19 @@ function extractTitle(body) {
|
|
|
77
178
|
return undefined;
|
|
78
179
|
}
|
|
79
180
|
function truncate(value, length) {
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
181
|
+
let end = 0;
|
|
182
|
+
let codePoints = 0;
|
|
183
|
+
for (const { index, segment } of GRAPHEME_SEGMENTER.segment(value)) {
|
|
184
|
+
codePoints += Array.from(segment).length;
|
|
185
|
+
if (codePoints > length)
|
|
186
|
+
return `${value.slice(0, end).trimEnd()}…`;
|
|
187
|
+
end = index + segment.length;
|
|
188
|
+
}
|
|
189
|
+
return value;
|
|
86
190
|
}
|
|
87
191
|
function extractSummary(body) {
|
|
88
192
|
const paragraphs = [];
|
|
89
193
|
let current = [];
|
|
90
|
-
let fence;
|
|
91
194
|
let inContainer = false;
|
|
92
195
|
let inTag = false;
|
|
93
196
|
let hiddenTag;
|
|
@@ -96,19 +199,12 @@ function extractSummary(body) {
|
|
|
96
199
|
paragraphs.push(current.join(' '));
|
|
97
200
|
current = [];
|
|
98
201
|
};
|
|
99
|
-
for (const rawLine of body
|
|
100
|
-
|
|
101
|
-
const marker = /^(`{3,}|~{3,})/.exec(line)?.[1]?.slice(0, 1);
|
|
102
|
-
if (marker) {
|
|
103
|
-
if (!fence)
|
|
104
|
-
fence = marker;
|
|
105
|
-
else if (marker === fence)
|
|
106
|
-
fence = undefined;
|
|
202
|
+
for (const rawLine of proseLines(body)) {
|
|
203
|
+
if (rawLine === undefined) {
|
|
107
204
|
finishParagraph();
|
|
108
205
|
continue;
|
|
109
206
|
}
|
|
110
|
-
|
|
111
|
-
continue;
|
|
207
|
+
const line = rawLine.trim();
|
|
112
208
|
if (hiddenTag) {
|
|
113
209
|
if (line.toLowerCase().includes(`</${hiddenTag}>`))
|
|
114
210
|
hiddenTag = undefined;
|
|
@@ -175,21 +271,10 @@ function parseAttributes(source) {
|
|
|
175
271
|
return attributes;
|
|
176
272
|
}
|
|
177
273
|
function componentSource(body) {
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
if (marker) {
|
|
183
|
-
if (!fence)
|
|
184
|
-
fence = marker;
|
|
185
|
-
else if (marker === fence)
|
|
186
|
-
fence = undefined;
|
|
187
|
-
continue;
|
|
188
|
-
}
|
|
189
|
-
if (!fence)
|
|
190
|
-
lines.push(line);
|
|
191
|
-
}
|
|
192
|
-
return lines.join('\n').replace(/<!--[\s\S]*?-->/g, ' ');
|
|
274
|
+
return proseLines(body)
|
|
275
|
+
.filter((line) => line !== undefined)
|
|
276
|
+
.join('\n')
|
|
277
|
+
.replace(/<!--[\s\S]*?-->/g, ' ');
|
|
193
278
|
}
|
|
194
279
|
function extractComponents(body) {
|
|
195
280
|
const components = [];
|
|
@@ -258,7 +343,75 @@ export function parseDoc(path, content) {
|
|
|
258
343
|
};
|
|
259
344
|
}
|
|
260
345
|
function normalize(value) {
|
|
261
|
-
|
|
346
|
+
// Lowercasing keeps final sigma (ς) where a search for Σ yields σ.
|
|
347
|
+
return value
|
|
348
|
+
.normalize('NFKC')
|
|
349
|
+
.toLowerCase()
|
|
350
|
+
.replace(/\u03c2/g, '\u03c3');
|
|
351
|
+
}
|
|
352
|
+
function normalizeChunks(text) {
|
|
353
|
+
const offsets = [];
|
|
354
|
+
const sources = [];
|
|
355
|
+
const parts = [];
|
|
356
|
+
let length = 0;
|
|
357
|
+
for (let start = 0; start < text.length;) {
|
|
358
|
+
NORMALIZATION_BOUNDARY.lastIndex = start + NORMALIZATION_CHUNK;
|
|
359
|
+
const end = NORMALIZATION_BOUNDARY.exec(text)?.index ?? text.length;
|
|
360
|
+
const part = normalize(text.slice(start, end));
|
|
361
|
+
offsets.push(length);
|
|
362
|
+
sources.push(start);
|
|
363
|
+
parts.push(part);
|
|
364
|
+
length += part.length;
|
|
365
|
+
start = end;
|
|
366
|
+
}
|
|
367
|
+
return { offsets, sources, value: parts.join('') };
|
|
368
|
+
}
|
|
369
|
+
function lastIndexAtMost(count, target, valueAt) {
|
|
370
|
+
let low = 0;
|
|
371
|
+
let high = count - 1;
|
|
372
|
+
while (low < high) {
|
|
373
|
+
const middle = Math.ceil((low + high) / 2);
|
|
374
|
+
if (valueAt(middle) <= target)
|
|
375
|
+
low = middle;
|
|
376
|
+
else
|
|
377
|
+
high = middle - 1;
|
|
378
|
+
}
|
|
379
|
+
return low;
|
|
380
|
+
}
|
|
381
|
+
function graphemeAt(graphemes, position) {
|
|
382
|
+
const part = graphemes.containing(position);
|
|
383
|
+
if (!part)
|
|
384
|
+
return { codePoints: 0, end: position, start: position };
|
|
385
|
+
return {
|
|
386
|
+
codePoints: Array.from(part.segment).length,
|
|
387
|
+
end: part.index + part.segment.length,
|
|
388
|
+
start: part.index,
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
function normalizedLength(text, normalized, position) {
|
|
392
|
+
const { offsets, sources } = normalized;
|
|
393
|
+
const chunk = lastIndexAtMost(sources.length, position, (index) => sources[index] ?? 0);
|
|
394
|
+
const start = sources[chunk] ?? 0;
|
|
395
|
+
return (offsets[chunk] ?? 0) + normalize(text.slice(start, position)).length;
|
|
396
|
+
}
|
|
397
|
+
function sourceBoundary(text, normalized, graphemes, offset) {
|
|
398
|
+
const { offsets, sources } = normalized;
|
|
399
|
+
const chunk = lastIndexAtMost(offsets.length, offset, (index) => offsets[index] ?? 0);
|
|
400
|
+
const chunkEnd = sources[chunk + 1] ?? text.length;
|
|
401
|
+
const boundaries = [graphemeAt(graphemes, sources[chunk] ?? 0).start];
|
|
402
|
+
for (let position = boundaries[0] ?? 0; position < chunkEnd;) {
|
|
403
|
+
position = graphemeAt(graphemes, position).end;
|
|
404
|
+
boundaries.push(position);
|
|
405
|
+
}
|
|
406
|
+
const lengths = new Map();
|
|
407
|
+
const lengthAt = (index) => {
|
|
408
|
+
const position = boundaries[index] ?? 0;
|
|
409
|
+
const length = lengths.get(position) ?? normalizedLength(text, normalized, position);
|
|
410
|
+
lengths.set(position, length);
|
|
411
|
+
return length;
|
|
412
|
+
};
|
|
413
|
+
const index = lastIndexAtMost(boundaries.length, offset, lengthAt);
|
|
414
|
+
return { length: lengthAt(index), position: boundaries[index] ?? 0 };
|
|
262
415
|
}
|
|
263
416
|
function countMatches(value, term) {
|
|
264
417
|
let count = 0;
|
|
@@ -275,18 +428,50 @@ function countMatches(value, term) {
|
|
|
275
428
|
function excerpt(text, query, terms) {
|
|
276
429
|
if (!text)
|
|
277
430
|
return '';
|
|
278
|
-
const normalized =
|
|
279
|
-
const exactIndex = normalized.indexOf(query);
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
431
|
+
const normalized = normalizeChunks(text);
|
|
432
|
+
const exactIndex = normalized.value.indexOf(query);
|
|
433
|
+
let matchIndex = exactIndex;
|
|
434
|
+
let matchLength = query.length;
|
|
435
|
+
if (matchIndex < 0) {
|
|
436
|
+
matchIndex = Number.POSITIVE_INFINITY;
|
|
437
|
+
for (const term of terms) {
|
|
438
|
+
const index = normalized.value.indexOf(term);
|
|
439
|
+
if (index >= 0 && index < matchIndex) {
|
|
440
|
+
matchIndex = index;
|
|
441
|
+
matchLength = term.length;
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
}
|
|
283
445
|
if (!Number.isFinite(matchIndex))
|
|
284
446
|
return truncate(text, EXCERPT_LENGTH);
|
|
285
|
-
const
|
|
447
|
+
const graphemes = GRAPHEME_SEGMENTER.segment(text);
|
|
448
|
+
const matchStart = sourceBoundary(text, normalized, graphemes, matchIndex).position;
|
|
449
|
+
const endOffset = Math.min(normalized.value.length, matchIndex + matchLength);
|
|
450
|
+
const floor = sourceBoundary(text, normalized, graphemes, endOffset);
|
|
451
|
+
const matchEnd = floor.length < endOffset ? graphemeAt(graphemes, floor.position).end : floor.position;
|
|
452
|
+
const matchCodePoints = Array.from(text.slice(matchStart, matchEnd)).length;
|
|
453
|
+
const contextLimit = Math.min(Math.floor(EXCERPT_LENGTH / 3), Math.max(0, EXCERPT_LENGTH - matchCodePoints));
|
|
454
|
+
let start = matchStart;
|
|
455
|
+
let contextCodePoints = 0;
|
|
456
|
+
while (start > 0) {
|
|
457
|
+
const previous = graphemeAt(graphemes, start - 1);
|
|
458
|
+
if (contextCodePoints + previous.codePoints > contextLimit)
|
|
459
|
+
break;
|
|
460
|
+
contextCodePoints += previous.codePoints;
|
|
461
|
+
start = previous.start;
|
|
462
|
+
}
|
|
463
|
+
let end = start;
|
|
464
|
+
let selectedCodePoints = 0;
|
|
465
|
+
while (end < text.length) {
|
|
466
|
+
const next = graphemeAt(graphemes, end);
|
|
467
|
+
if (end > start && selectedCodePoints + next.codePoints > EXCERPT_LENGTH)
|
|
468
|
+
break;
|
|
469
|
+
selectedCodePoints += next.codePoints;
|
|
470
|
+
end = next.end;
|
|
471
|
+
}
|
|
286
472
|
const prefix = start > 0 ? '…' : '';
|
|
287
|
-
const
|
|
288
|
-
|
|
289
|
-
return `${prefix}${value}${suffix}`;
|
|
473
|
+
const suffix = end < text.length ? '…' : '';
|
|
474
|
+
return `${prefix}${text.slice(start, end).trim()}${suffix}`;
|
|
290
475
|
}
|
|
291
476
|
export function searchDoc(page, query) {
|
|
292
477
|
const normalizedQuery = normalize(query.trim());
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { createDocsClient } from './client.js';
|
|
2
2
|
export { parseDoc } from './content.js';
|
|
3
|
-
export type { DocComponent, DocItem, DocPage, DocSection, DocsClient, DocsClientOptions, DocsSearchOptions, DocsSearchResult, } from './types.js';
|
|
3
|
+
export type { DocComponent, DocItem, DocPage, DocSection, DocsClient, DocsClientOptions, DocsSearchOptions, DocsSearchResult, DocsStore, } from './types.js';
|
|
4
4
|
export { DocsFetchError } from './types.js';
|
package/dist/types.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ export interface DocItem {
|
|
|
2
2
|
name: string;
|
|
3
3
|
path: string;
|
|
4
4
|
type: 'file' | 'dir';
|
|
5
|
+
sha?: string;
|
|
5
6
|
}
|
|
6
7
|
export interface DocComponent {
|
|
7
8
|
attributes: Readonly<Record<string, string | true>>;
|
|
@@ -36,6 +37,10 @@ export interface DocsSearchResult {
|
|
|
36
37
|
summary: string;
|
|
37
38
|
title: string;
|
|
38
39
|
}
|
|
40
|
+
export interface DocsStore {
|
|
41
|
+
read(key: string): string | undefined;
|
|
42
|
+
write(key: string, value: string): void;
|
|
43
|
+
}
|
|
39
44
|
export interface DocsClientOptions {
|
|
40
45
|
owner?: string;
|
|
41
46
|
repo?: string;
|
|
@@ -45,10 +50,12 @@ export interface DocsClientOptions {
|
|
|
45
50
|
dir?: number;
|
|
46
51
|
file?: number;
|
|
47
52
|
};
|
|
53
|
+
store?: DocsStore;
|
|
48
54
|
}
|
|
49
55
|
export interface DocsClient {
|
|
50
56
|
listDir(path?: string): Promise<DocItem[]>;
|
|
51
57
|
listAll(): Promise<DocItem[]>;
|
|
58
|
+
peekAll(): DocItem[] | undefined;
|
|
52
59
|
listSections(): Promise<DocSection[]>;
|
|
53
60
|
getFile(path: string): Promise<string>;
|
|
54
61
|
getDocument(path: string): Promise<DocPage>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nbtca/docs",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.2",
|
|
4
4
|
"description": "GitHub-backed document client for NBTCA",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -60,6 +60,6 @@
|
|
|
60
60
|
"typescript": "^5.9.3",
|
|
61
61
|
"typescript-eslint": "^8.67.0",
|
|
62
62
|
"vite": "6.4.3",
|
|
63
|
-
"vitest": "^
|
|
63
|
+
"vitest": "^4.1.11"
|
|
64
64
|
}
|
|
65
65
|
}
|