@nbtca/docs 0.3.1 → 0.3.3

package/README.md

@@ -36,6 +36,8 @@ const matches = await docs.search('repair', { pathPrefix: 'repair' });
 | `token`           | `GITHUB_TOKEN` or `GH_TOKEN` | GitHub token                 |
 | `cacheTtlMs.dir`  | `300000`                     | Directory and tree cache TTL |
 | `cacheTtlMs.file` | `600000`                     | File cache TTL               |
+| `store`           | none                         | Persistent cache, see below  |
+| `mirror`          | none                         | Mirror base URL, see below   |
 
 ### `docs.listDir(path?)`
 
@@ -48,7 +50,12 @@ Returns raw file content.
 
 ### `docs.listAll()`
 
-Lists every Markdown file through GitHub's recursive tree API.
+Lists every Markdown file through GitHub's recursive tree API. Each item carries its Git blob `sha`.
+
+### `docs.peekAll()`
+
+Returns the last known tree without a request: the one fetched in this process, else the one in
+`store`, else `undefined`.
 
 ### `docs.listSections()`
 
@@ -60,6 +67,20 @@ Returns content with its route, section, title, summary, and semantic component
 metadata covers `PageHero`, `FactStrip`, `LinkCard`, `Split`, `TimelineEntry`, and `Figure` without
 imposing a renderer.
 
+### `store`
+
+An object with `read(key): string | undefined` and `write(key, value): void`. The client keeps the
+tree under `tree` and file content under `blob-<sha>`, where `<sha>` is the Git blob id. `getFile`
+serves stored content only when its hash matches the blob id in the last known tree, so a changed
+file is always refetched. Store errors and corrupt entries are ignored; eviction is up to the store.
+
+### `mirror`
+
+A base URL tried before GitHub by `listAll` and `getFile`, such as
+`https://docs.nbtca.space/docs-api`. It serves `index.json` in the shape of GitHub's recursive tree
+response and each file at `raw/<path>`. Any mirror failure, invalid or truncated index, or wait past
+5 seconds falls back to GitHub. The GitHub token is never sent to the mirror.
+
 ### `docs.search(query, options?)`
 
 Searches paths, titles, summaries, Markdown text, and semantic component attributes. Results are

package/dist/client.js

@@ -30,6 +30,9 @@ const SKIP = new Set([
     'README.md',
     'docs',
 ]);
+const TREE_KEY = 'tree';
+const MIRROR_TIMEOUT_MS = 5000;
+const SHA = /^[0-9a-f]{40}$/;
 const SEARCH_CONCURRENCY = 6;
 const SEARCH_RESULT_LIMIT = 20;
 function filterAndSort(raw) {
@@ -41,6 +44,7 @@ function filterAndSort(raw) {
         name: item.name,
         path: item.path,
         type: item.type === 'dir' ? 'dir' : 'file',
+        ...shaOf(item),
     }))
         .sort((a, b) => {
         if (a.type !== b.type)
@@ -60,9 +64,40 @@ function filterTree(items) {
         name: item.path.slice(item.path.lastIndexOf('/') + 1),
         path: item.path,
         type: 'file',
+        ...shaOf(item),
     }))
         .sort((a, b) => a.path.localeCompare(b.path));
 }
+function shaOf(item) {
+    return typeof item.sha === 'string' && SHA.test(item.sha) ? { sha: item.sha } : {};
+}
+async function blobSha(content) {
+    const body = new TextEncoder().encode(content);
+    const header = new TextEncoder().encode(`blob ${String(body.length)}\0`);
+    const object = new Uint8Array(header.length + body.length);
+    object.set(header);
+    object.set(body, header.length);
+    const digest = new Uint8Array(await crypto.subtle.digest('SHA-1', object));
+    return Array.from(digest, (byte) => byte.toString(16).padStart(2, '0')).join('');
+}
+function isDocItem(value) {
+    return (isRecord(value) &&
+        typeof value.name === 'string' &&
+        typeof value.path === 'string' &&
+        (value.type === 'file' || value.type === 'dir') &&
+        (value.sha === undefined || typeof value.sha === 'string'));
+}
+function parseStoredTree(value) {
+    if (value === undefined)
+        return undefined;
+    try {
+        const items = JSON.parse(value);
+        return Array.isArray(items) && items.every(isDocItem) ? items : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
 function copyItems(items) {
     return items.map((item) => ({ ...item }));
 }
@@ -155,6 +190,15 @@ function assertBranchRef(value) {
         throw new TypeError('branch must be a valid Git ref');
     }
 }
+function mirrorBase(value) {
+    if (value === undefined)
+        return undefined;
+    const url = URL.canParse(value) ? new URL(value) : undefined;
+    if (!url || !['https:', 'http:'].includes(url.protocol) || url.search || url.hash) {
+        throw new TypeError('mirror must be an http(s) URL without a query or fragment');
+    }
+    return url.href.replace(/\/+$/, '');
+}
 function cacheTtl(value, fallback, name) {
     const ttl = value ?? fallback;
     if (!Number.isFinite(ttl) || ttl < 0) {
@@ -245,6 +289,8 @@ export function createDocsClient(options = {}) {
     const apiRepoUrl = `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`;
     const rawRepoUrl = `https://raw.githubusercontent.com/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`;
     const encodedBranch = encodeURIComponent(branch);
+    const mirror = mirrorBase(options.mirror);
+    let mirrorFailed = false;
     const dirTtlMs = cacheTtl(options.cacheTtlMs?.dir, DEFAULTS.dirTtlMs, 'cacheTtlMs.dir');
     const fileTtlMs = cacheTtl(options.cacheTtlMs?.file, DEFAULTS.fileTtlMs, 'cacheTtlMs.file');
     const dirCache = new TtlCache(dirTtlMs, 30);
@@ -253,7 +299,32 @@ export function createDocsClient(options = {}) {
     const dirRequests = new Map();
     const fileRequests = new Map();
     const treeRequests = new Map();
+    const store = options.store;
+    let storedTree;
     let cacheGeneration = 0;
+    function storeRead(key) {
+        try {
+            return store?.read(key);
+        }
+        catch {
+            return undefined;
+        }
+    }
+    function storeWrite(key, value) {
+        try {
+            store?.write(key, value);
+        }
+        catch {
+            return;
+        }
+    }
+    function knownTree() {
+        const fetched = treeCache.getStale(TREE_KEY);
+        if (fetched)
+            return fetched;
+        storedTree ?? (storedTree = parseStoredTree(storeRead(TREE_KEY)));
+        return storedTree;
+    }
     function headers() {
         const requestHeaders = {
             Accept: 'application/vnd.github.v3+json',
@@ -262,14 +333,14 @@ export function createDocsClient(options = {}) {
             requestHeaders.Authorization = `Bearer ${token}`;
         return requestHeaders;
     }
-    async function withResponse(url, timeoutMs, consume) {
+    async function withResponse(url, timeoutMs, consume, requestHeaders = headers()) {
         const ctrl = new AbortController();
         const timer = setTimeout(() => {
             ctrl.abort();
         }, timeoutMs);
         let response;
         try {
-            response = await fetch(url, { signal: ctrl.signal, headers: headers() });
+            response = await fetch(url, { signal: ctrl.signal, headers: requestHeaders });
             return await consume(response);
         }
         finally {
@@ -278,6 +349,36 @@ export function createDocsClient(options = {}) {
             cancelUnusedResponseBody(response);
         }
     }
+    async function fromMirror(base, path, read) {
+        if (mirrorFailed)
+            return undefined;
+        try {
+            return await withResponse(`${base}/${path}`, MIRROR_TIMEOUT_MS, async (response) => {
+                if (response.status >= 500)
+                    mirrorFailed = true;
+                return response.ok ? await read(response) : undefined;
+            }, {});
+        }
+        catch {
+            mirrorFailed = true;
+            return undefined;
+        }
+    }
+    function keepTree(items, generation) {
+        if (generation === cacheGeneration) {
+            treeCache.set(TREE_KEY, copyItems(items));
+            if (store)
+                storeWrite(TREE_KEY, JSON.stringify(items));
+        }
+        return items;
+    }
+    async function keepFile(path, content, generation) {
+        if (generation === cacheGeneration)
+            fileCache.set(path, content);
+        if (store)
+            storeWrite(`blob-${await blobSha(content)}`, content);
+        return content;
+    }
     function recoverFailure(cache, key, path, error, copy = (value) => value) {
         const stale = cache.getStale(key);
         if (stale !== undefined)
@@ -334,7 +435,15 @@ export function createDocsClient(options = {}) {
         return shareRequest(dirRequests, path, () => loadDir(path, cacheGeneration)).then(copyItems);
     }
     async function loadAll(generation) {
-        const key = '__tree__';
+        const key = TREE_KEY;
+        if (mirror !== undefined) {
+            const mirrored = await fromMirror(mirror, 'index.json', async (response) => {
+                const data = parseTreeResponse(await response.json());
+                return data.truncated ? undefined : filterTree(data.tree);
+            });
+            if (mirrored)
+                return keepTree(mirrored, generation);
+        }
         const url = `${apiRepoUrl}/git/trees/${encodedBranch}?recursive=1`;
         try {
             return await withResponse(url, 20000, async (response) => {
@@ -351,10 +460,7 @@ export function createDocsClient(options = {}) {
                         return copyItems(stale);
                     throw new DocsFetchError('', null, 'GitHub truncated the repository tree (too many files) -- results would be incomplete');
                 }
-                const items = filterTree(data.tree);
-                if (generation === cacheGeneration)
-                    treeCache.set(key, copyItems(items));
-                return items;
+                return keepTree(filterTree(data.tree), generation);
             });
         }
         catch (error) {
@@ -364,16 +470,37 @@ export function createDocsClient(options = {}) {
         }
     }
     function listAll() {
-        const key = '__tree__';
-        const hit = treeCache.get(key);
+        const hit = treeCache.get(TREE_KEY);
         if (hit)
             return Promise.resolve(copyItems(hit));
-        return shareRequest(treeRequests, key, () => loadAll(cacheGeneration)).then(copyItems);
+        return shareRequest(treeRequests, TREE_KEY, () => loadAll(cacheGeneration)).then(copyItems);
+    }
+    function peekAll() {
+        const items = knownTree();
+        return items && copyItems(items);
     }
     async function listSections() {
         return sectionsFromItems(await listAll());
     }
+    async function loadStoredFile(path) {
+        const sha = knownTree()?.find((item) => item.path === path)?.sha;
+        if (sha === undefined)
+            return undefined;
+        const content = storeRead(`blob-${sha}`);
+        return content !== undefined && (await blobSha(content)) === sha ? content : undefined;
+    }
     async function loadFile(path, generation) {
+        const stored = store && (await loadStoredFile(path));
+        if (stored !== undefined) {
+            if (generation === cacheGeneration)
+                fileCache.set(path, stored);
+            return stored;
+        }
+        if (mirror !== undefined) {
+            const mirrored = await fromMirror(mirror, `raw/${encodePath(path)}`, (response) => response.text());
+            if (mirrored !== undefined)
+                return keepFile(path, mirrored, generation);
+        }
         const url = `${rawRepoUrl}/${encodedBranch}/${encodePath(path)}`;
         try {
             return await withResponse(url, 15000, async (response) => {
@@ -383,10 +510,7 @@ export function createDocsClient(options = {}) {
                         return stale;
                     throw new DocsFetchError(path, response.status, `HTTP ${String(response.status)}`);
                 }
-                const content = await response.text();
-                if (generation === cacheGeneration)
-                    fileCache.set(path, content);
-                return content;
+                return keepFile(path, await response.text(), generation);
             });
         }
         catch (error) {
@@ -451,6 +575,7 @@ export function createDocsClient(options = {}) {
     }
     function clear() {
         cacheGeneration += 1;
+        storedTree = undefined;
         dirCache.clear();
         fileCache.clear();
         treeCache.clear();
@@ -458,5 +583,5 @@ export function createDocsClient(options = {}) {
         fileRequests.clear();
         treeRequests.clear();
     }
-    return { listDir, listAll, listSections, getFile, getDocument, search, clear };
+    return { listDir, listAll, peekAll, listSections, getFile, getDocument, search, clear };
 }

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 export { createDocsClient } from './client.js';
 export { parseDoc } from './content.js';
-export type { DocComponent, DocItem, DocPage, DocSection, DocsClient, DocsClientOptions, DocsSearchOptions, DocsSearchResult, } from './types.js';
+export type { DocComponent, DocItem, DocPage, DocSection, DocsClient, DocsClientOptions, DocsSearchOptions, DocsSearchResult, DocsStore, } from './types.js';
 export { DocsFetchError } from './types.js';

package/dist/types.d.ts

@@ -2,6 +2,7 @@ export interface DocItem {
     name: string;
     path: string;
     type: 'file' | 'dir';
+    sha?: string;
 }
 export interface DocComponent {
     attributes: Readonly<Record<string, string | true>>;
@@ -36,6 +37,10 @@ export interface DocsSearchResult {
     summary: string;
     title: string;
 }
+export interface DocsStore {
+    read(key: string): string | undefined;
+    write(key: string, value: string): void;
+}
 export interface DocsClientOptions {
     owner?: string;
     repo?: string;
@@ -45,10 +50,13 @@ export interface DocsClientOptions {
         dir?: number;
         file?: number;
     };
+    store?: DocsStore;
+    mirror?: string;
 }
 export interface DocsClient {
     listDir(path?: string): Promise<DocItem[]>;
     listAll(): Promise<DocItem[]>;
+    peekAll(): DocItem[] | undefined;
     listSections(): Promise<DocSection[]>;
     getFile(path: string): Promise<string>;
     getDocument(path: string): Promise<DocPage>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nbtca/docs",
-  "version": "0.3.1",
+  "version": "0.3.3",
   "description": "GitHub-backed document client for NBTCA",
   "type": "module",
   "main": "./dist/index.js",