@nbtca/docs 0.3.1 → 0.3.2

package/README.md

@@ -36,6 +36,7 @@ const matches = await docs.search('repair', { pathPrefix: 'repair' });
 | `token`           | `GITHUB_TOKEN` or `GH_TOKEN` | GitHub token                 |
 | `cacheTtlMs.dir`  | `300000`                     | Directory and tree cache TTL |
 | `cacheTtlMs.file` | `600000`                     | File cache TTL               |
+| `store`           | none                         | Persistent cache, see below  |
 
 ### `docs.listDir(path?)`
 
@@ -48,7 +49,12 @@ Returns raw file content.
 
 ### `docs.listAll()`
 
-Lists every Markdown file through GitHub's recursive tree API.
+Lists every Markdown file through GitHub's recursive tree API. Each item carries its Git blob `sha`.
+
+### `docs.peekAll()`
+
+Returns the last known tree without a request: the one fetched in this process, else the one in
+`store`, else `undefined`.
 
 ### `docs.listSections()`
 
@@ -60,6 +66,13 @@ Returns content with its route, section, title, summary, and semantic component
 metadata covers `PageHero`, `FactStrip`, `LinkCard`, `Split`, `TimelineEntry`, and `Figure` without
 imposing a renderer.
 
+### `store`
+
+An object with `read(key): string | undefined` and `write(key, value): void`. The client keeps the
+tree under `tree` and file content under `blob-<sha>`, where `<sha>` is the Git blob id. `getFile`
+serves stored content only when its hash matches the blob id in the last known tree, so a changed
+file is always refetched. Store errors and corrupt entries are ignored; eviction is up to the store.
+
 ### `docs.search(query, options?)`
 
 Searches paths, titles, summaries, Markdown text, and semantic component attributes. Results are

package/dist/client.js

@@ -30,6 +30,8 @@ const SKIP = new Set([
     'README.md',
     'docs',
 ]);
+const TREE_KEY = 'tree';
+const SHA = /^[0-9a-f]{40}$/;
 const SEARCH_CONCURRENCY = 6;
 const SEARCH_RESULT_LIMIT = 20;
 function filterAndSort(raw) {
@@ -41,6 +43,7 @@ function filterAndSort(raw) {
         name: item.name,
         path: item.path,
         type: item.type === 'dir' ? 'dir' : 'file',
+        ...shaOf(item),
     }))
         .sort((a, b) => {
         if (a.type !== b.type)
@@ -60,9 +63,40 @@ function filterTree(items) {
         name: item.path.slice(item.path.lastIndexOf('/') + 1),
         path: item.path,
         type: 'file',
+        ...shaOf(item),
     }))
         .sort((a, b) => a.path.localeCompare(b.path));
 }
+function shaOf(item) {
+    return typeof item.sha === 'string' && SHA.test(item.sha) ? { sha: item.sha } : {};
+}
+async function blobSha(content) {
+    const body = new TextEncoder().encode(content);
+    const header = new TextEncoder().encode(`blob ${String(body.length)}\0`);
+    const object = new Uint8Array(header.length + body.length);
+    object.set(header);
+    object.set(body, header.length);
+    const digest = new Uint8Array(await crypto.subtle.digest('SHA-1', object));
+    return Array.from(digest, (byte) => byte.toString(16).padStart(2, '0')).join('');
+}
+function isDocItem(value) {
+    return (isRecord(value) &&
+        typeof value.name === 'string' &&
+        typeof value.path === 'string' &&
+        (value.type === 'file' || value.type === 'dir') &&
+        (value.sha === undefined || typeof value.sha === 'string'));
+}
+function parseStoredTree(value) {
+    if (value === undefined)
+        return undefined;
+    try {
+        const items = JSON.parse(value);
+        return Array.isArray(items) && items.every(isDocItem) ? items : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
 function copyItems(items) {
     return items.map((item) => ({ ...item }));
 }
@@ -253,7 +287,32 @@ export function createDocsClient(options = {}) {
     const dirRequests = new Map();
     const fileRequests = new Map();
     const treeRequests = new Map();
+    const store = options.store;
+    let storedTree;
     let cacheGeneration = 0;
+    function storeRead(key) {
+        try {
+            return store?.read(key);
+        }
+        catch {
+            return undefined;
+        }
+    }
+    function storeWrite(key, value) {
+        try {
+            store?.write(key, value);
+        }
+        catch {
+            return;
+        }
+    }
+    function knownTree() {
+        const fetched = treeCache.getStale(TREE_KEY);
+        if (fetched)
+            return fetched;
+        storedTree ?? (storedTree = parseStoredTree(storeRead(TREE_KEY)));
+        return storedTree;
+    }
     function headers() {
         const requestHeaders = {
             Accept: 'application/vnd.github.v3+json',
@@ -334,7 +393,7 @@ export function createDocsClient(options = {}) {
         return shareRequest(dirRequests, path, () => loadDir(path, cacheGeneration)).then(copyItems);
     }
     async function loadAll(generation) {
-        const key = '__tree__';
+        const key = TREE_KEY;
         const url = `${apiRepoUrl}/git/trees/${encodedBranch}?recursive=1`;
         try {
             return await withResponse(url, 20000, async (response) => {
@@ -352,8 +411,11 @@ export function createDocsClient(options = {}) {
                     throw new DocsFetchError('', null, 'GitHub truncated the repository tree (too many files) -- results would be incomplete');
                 }
                 const items = filterTree(data.tree);
-                if (generation === cacheGeneration)
+                if (generation === cacheGeneration) {
                     treeCache.set(key, copyItems(items));
+                    if (store)
+                        storeWrite(key, JSON.stringify(items));
+                }
                 return items;
             });
         }
@@ -364,16 +426,32 @@ export function createDocsClient(options = {}) {
         }
     }
     function listAll() {
-        const key = '__tree__';
-        const hit = treeCache.get(key);
+        const hit = treeCache.get(TREE_KEY);
         if (hit)
             return Promise.resolve(copyItems(hit));
-        return shareRequest(treeRequests, key, () => loadAll(cacheGeneration)).then(copyItems);
+        return shareRequest(treeRequests, TREE_KEY, () => loadAll(cacheGeneration)).then(copyItems);
+    }
+    function peekAll() {
+        const items = knownTree();
+        return items && copyItems(items);
     }
     async function listSections() {
         return sectionsFromItems(await listAll());
     }
+    async function loadStoredFile(path) {
+        const sha = knownTree()?.find((item) => item.path === path)?.sha;
+        if (sha === undefined)
+            return undefined;
+        const content = storeRead(`blob-${sha}`);
+        return content !== undefined && (await blobSha(content)) === sha ? content : undefined;
+    }
     async function loadFile(path, generation) {
+        const stored = store && (await loadStoredFile(path));
+        if (stored !== undefined) {
+            if (generation === cacheGeneration)
+                fileCache.set(path, stored);
+            return stored;
+        }
         const url = `${rawRepoUrl}/${encodedBranch}/${encodePath(path)}`;
         try {
             return await withResponse(url, 15000, async (response) => {
@@ -386,6 +464,8 @@ export function createDocsClient(options = {}) {
                 const content = await response.text();
                 if (generation === cacheGeneration)
                     fileCache.set(path, content);
+                if (store)
+                    storeWrite(`blob-${await blobSha(content)}`, content);
                 return content;
             });
         }
@@ -451,6 +531,7 @@ export function createDocsClient(options = {}) {
     }
     function clear() {
         cacheGeneration += 1;
+        storedTree = undefined;
         dirCache.clear();
         fileCache.clear();
         treeCache.clear();
@@ -458,5 +539,5 @@ export function createDocsClient(options = {}) {
         fileRequests.clear();
         treeRequests.clear();
     }
-    return { listDir, listAll, listSections, getFile, getDocument, search, clear };
+    return { listDir, listAll, peekAll, listSections, getFile, getDocument, search, clear };
 }

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 export { createDocsClient } from './client.js';
 export { parseDoc } from './content.js';
-export type { DocComponent, DocItem, DocPage, DocSection, DocsClient, DocsClientOptions, DocsSearchOptions, DocsSearchResult, } from './types.js';
+export type { DocComponent, DocItem, DocPage, DocSection, DocsClient, DocsClientOptions, DocsSearchOptions, DocsSearchResult, DocsStore, } from './types.js';
 export { DocsFetchError } from './types.js';

package/dist/types.d.ts

@@ -2,6 +2,7 @@ export interface DocItem {
     name: string;
     path: string;
     type: 'file' | 'dir';
+    sha?: string;
 }
 export interface DocComponent {
     attributes: Readonly<Record<string, string | true>>;
@@ -36,6 +37,10 @@ export interface DocsSearchResult {
     summary: string;
     title: string;
 }
+export interface DocsStore {
+    read(key: string): string | undefined;
+    write(key: string, value: string): void;
+}
 export interface DocsClientOptions {
     owner?: string;
     repo?: string;
@@ -45,10 +50,12 @@ export interface DocsClientOptions {
         dir?: number;
         file?: number;
     };
+    store?: DocsStore;
 }
 export interface DocsClient {
     listDir(path?: string): Promise<DocItem[]>;
     listAll(): Promise<DocItem[]>;
+    peekAll(): DocItem[] | undefined;
     listSections(): Promise<DocSection[]>;
     getFile(path: string): Promise<string>;
     getDocument(path: string): Promise<DocPage>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nbtca/docs",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "GitHub-backed document client for NBTCA",
   "type": "module",
   "main": "./dist/index.js",