@nbtca/docs 0.3.2 → 0.3.3

package/README.md

@@ -37,6 +37,7 @@ const matches = await docs.search('repair', { pathPrefix: 'repair' });
 | `cacheTtlMs.dir`  | `300000`                     | Directory and tree cache TTL |
 | `cacheTtlMs.file` | `600000`                     | File cache TTL               |
 | `store`           | none                         | Persistent cache, see below  |
+| `mirror`          | none                         | Mirror base URL, see below   |
 
 ### `docs.listDir(path?)`
 
@@ -73,6 +74,13 @@ tree under `tree` and file content under `blob-<sha>`, where `<sha>` is the Git
 serves stored content only when its hash matches the blob id in the last known tree, so a changed
 file is always refetched. Store errors and corrupt entries are ignored; eviction is up to the store.
 
+### `mirror`
+
+A base URL tried before GitHub by `listAll` and `getFile`, such as
+`https://docs.nbtca.space/docs-api`. It serves `index.json` in the shape of GitHub's recursive tree
+response and each file at `raw/<path>`. Any mirror failure, invalid or truncated index, or wait past
+5 seconds falls back to GitHub. The GitHub token is never sent to the mirror.
+
 ### `docs.search(query, options?)`
 
 Searches paths, titles, summaries, Markdown text, and semantic component attributes. Results are

package/dist/client.js

@@ -31,6 +31,7 @@ const SKIP = new Set([
     'docs',
 ]);
 const TREE_KEY = 'tree';
+const MIRROR_TIMEOUT_MS = 5000;
 const SHA = /^[0-9a-f]{40}$/;
 const SEARCH_CONCURRENCY = 6;
 const SEARCH_RESULT_LIMIT = 20;
@@ -189,6 +190,15 @@ function assertBranchRef(value) {
         throw new TypeError('branch must be a valid Git ref');
     }
 }
+function mirrorBase(value) {
+    if (value === undefined)
+        return undefined;
+    const url = URL.canParse(value) ? new URL(value) : undefined;
+    if (!url || !['https:', 'http:'].includes(url.protocol) || url.search || url.hash) {
+        throw new TypeError('mirror must be an http(s) URL without a query or fragment');
+    }
+    return url.href.replace(/\/+$/, '');
+}
 function cacheTtl(value, fallback, name) {
     const ttl = value ?? fallback;
     if (!Number.isFinite(ttl) || ttl < 0) {
@@ -279,6 +289,8 @@ export function createDocsClient(options = {}) {
     const apiRepoUrl = `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`;
     const rawRepoUrl = `https://raw.githubusercontent.com/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`;
     const encodedBranch = encodeURIComponent(branch);
+    const mirror = mirrorBase(options.mirror);
+    let mirrorFailed = false;
     const dirTtlMs = cacheTtl(options.cacheTtlMs?.dir, DEFAULTS.dirTtlMs, 'cacheTtlMs.dir');
     const fileTtlMs = cacheTtl(options.cacheTtlMs?.file, DEFAULTS.fileTtlMs, 'cacheTtlMs.file');
     const dirCache = new TtlCache(dirTtlMs, 30);
@@ -321,14 +333,14 @@ export function createDocsClient(options = {}) {
             requestHeaders.Authorization = `Bearer ${token}`;
         return requestHeaders;
     }
-    async function withResponse(url, timeoutMs, consume) {
+    async function withResponse(url, timeoutMs, consume, requestHeaders = headers()) {
         const ctrl = new AbortController();
         const timer = setTimeout(() => {
             ctrl.abort();
         }, timeoutMs);
         let response;
         try {
-            response = await fetch(url, { signal: ctrl.signal, headers: headers() });
+            response = await fetch(url, { signal: ctrl.signal, headers: requestHeaders });
             return await consume(response);
         }
         finally {
@@ -337,6 +349,36 @@ export function createDocsClient(options = {}) {
             cancelUnusedResponseBody(response);
         }
     }
+    async function fromMirror(base, path, read) {
+        if (mirrorFailed)
+            return undefined;
+        try {
+            return await withResponse(`${base}/${path}`, MIRROR_TIMEOUT_MS, async (response) => {
+                if (response.status >= 500)
+                    mirrorFailed = true;
+                return response.ok ? await read(response) : undefined;
+            }, {});
+        }
+        catch {
+            mirrorFailed = true;
+            return undefined;
+        }
+    }
+    function keepTree(items, generation) {
+        if (generation === cacheGeneration) {
+            treeCache.set(TREE_KEY, copyItems(items));
+            if (store)
+                storeWrite(TREE_KEY, JSON.stringify(items));
+        }
+        return items;
+    }
+    async function keepFile(path, content, generation) {
+        if (generation === cacheGeneration)
+            fileCache.set(path, content);
+        if (store)
+            storeWrite(`blob-${await blobSha(content)}`, content);
+        return content;
+    }
     function recoverFailure(cache, key, path, error, copy = (value) => value) {
         const stale = cache.getStale(key);
         if (stale !== undefined)
@@ -394,6 +436,14 @@ export function createDocsClient(options = {}) {
     }
     async function loadAll(generation) {
         const key = TREE_KEY;
+        if (mirror !== undefined) {
+            const mirrored = await fromMirror(mirror, 'index.json', async (response) => {
+                const data = parseTreeResponse(await response.json());
+                return data.truncated ? undefined : filterTree(data.tree);
+            });
+            if (mirrored)
+                return keepTree(mirrored, generation);
+        }
         const url = `${apiRepoUrl}/git/trees/${encodedBranch}?recursive=1`;
         try {
             return await withResponse(url, 20000, async (response) => {
@@ -410,13 +460,7 @@ export function createDocsClient(options = {}) {
                         return copyItems(stale);
                     throw new DocsFetchError('', null, 'GitHub truncated the repository tree (too many files) -- results would be incomplete');
                 }
-                const items = filterTree(data.tree);
-                if (generation === cacheGeneration) {
-                    treeCache.set(key, copyItems(items));
-                    if (store)
-                        storeWrite(key, JSON.stringify(items));
-                }
-                return items;
+                return keepTree(filterTree(data.tree), generation);
             });
         }
         catch (error) {
@@ -452,6 +496,11 @@ export function createDocsClient(options = {}) {
                 fileCache.set(path, stored);
             return stored;
         }
+        if (mirror !== undefined) {
+            const mirrored = await fromMirror(mirror, `raw/${encodePath(path)}`, (response) => response.text());
+            if (mirrored !== undefined)
+                return keepFile(path, mirrored, generation);
+        }
         const url = `${rawRepoUrl}/${encodedBranch}/${encodePath(path)}`;
         try {
             return await withResponse(url, 15000, async (response) => {
@@ -461,12 +510,7 @@ export function createDocsClient(options = {}) {
                         return stale;
                     throw new DocsFetchError(path, response.status, `HTTP ${String(response.status)}`);
                 }
-                const content = await response.text();
-                if (generation === cacheGeneration)
-                    fileCache.set(path, content);
-                if (store)
-                    storeWrite(`blob-${await blobSha(content)}`, content);
-                return content;
+                return keepFile(path, await response.text(), generation);
             });
         }
         catch (error) {

package/dist/types.d.ts

@@ -51,6 +51,7 @@ export interface DocsClientOptions {
         file?: number;
     };
     store?: DocsStore;
+    mirror?: string;
 }
 export interface DocsClient {
     listDir(path?: string): Promise<DocItem[]>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nbtca/docs",
-  "version": "0.3.2",
+  "version": "0.3.3",
   "description": "GitHub-backed document client for NBTCA",
   "type": "module",
   "main": "./dist/index.js",