@ai-ecoverse/slicc-man 1.1.0 → 1.1.2

package/README.md

@@ -134,7 +134,7 @@ Pages, the sitemap and the topic index are cached:
 
 Each site has its own directory in the cache, named by the base URL's host and port (`www.sliccy.com`, `127.0.0.1-8080`), with one JSON file per page, `sitemap.json` and `index.json`. A cached copy is fresh for the response's `Cache-Control` `max-age` (the site sends 4 hours), and 4 hours without one. A stale copy is revalidated with `If-Modified-Since` (the response's `Last-Modified`): `304` keeps it and makes it fresh again, `200` replaces it, and `404` drops it, so the topic has no page. When the site can't be reached, or answers 5xx, a cached copy is shown however old, after one line on stderr, `<cmd>: <url>: <reason>; showing the copy cached <date>`. With nothing cached, the command prints `<cmd>: <url>: <reason>` and exits 2. The cache is written atomically (a temporary file renamed into place), so two commands at once never see half a page; a cache that can't be written is skipped without a word.
 
-`apropos` and `whatis` read the index: every `/man/<topic>` entry of the sitemap, with its description, the first paragraph of its page, rendered as plain text on one line. The index is built from the cached sitemap and pages, fetching what is missing or stale (pages through the same cache, several at a time), and cached itself under the same rules. A page that fails to load is listed without a description.
+`apropos` and `whatis` read the index: every `/man/<topic>` entry of the sitemap, with its description, the first paragraph of its page, rendered as plain text on one line. The index is built from the cached sitemap and pages, fetching what is missing or stale (pages through the same cache, several at a time), and cached itself under the same rules. A page that fails to load is listed without a description, and an index with a failed page isn't cached, so the next command tries that page again. After an HTTP error (a 404 or a 5xx), the other pages are still fetched; after a network error or a timeout, no more pages are fetched, and the rest come from the cache.
 
 `SLICC_MAN_URL` overrides the base URL `https://www.sliccy.com`: pages are `$SLICC_MAN_URL/man/<topic>.md` and the sitemap `$SLICC_MAN_URL/sitemap.xml`, whose entries count by their path, whatever their host. The tests point it at a local server.
 

package/host/main.mjs

@@ -32,6 +32,7 @@ export async function main({
   columns,
   fetch = globalThis.fetch,
   now,
+  timeout,
 }) {
   const child = { ...env };
   delete child.SCMAN_INSTALL_DIR;
@@ -41,6 +42,7 @@ export async function main({
     columns,
     fetch,
     now,
+    timeout,
     version: version(),
     out: (text) => say(fds[1], text),
     err: (text) => say(fds[2], text),

package/lib/man.mjs

@@ -201,7 +201,14 @@ export async function run({ kind = 'man', name, argv, env, io }) {
     values: parsed.values,
     width: width(env, io),
     styled: Boolean(io.tty) && !env.NO_COLOR,
-    site: site({ url, store: io.store(folder(url)), fetch: io.fetch, now: io.now, name }),
+    site: site({
+      url,
+      store: io.store(folder(url)),
+      fetch: io.fetch,
+      now: io.now,
+      name,
+      timeout: io.timeout,
+    }),
   };
   if (kind !== 'man') return keywords(kind, ctx, parsed);
   if (parsed.flags.apropos) return keywords('apropos', ctx, parsed);

package/lib/site.mjs

@@ -69,7 +69,7 @@ export function topics(xml) {
   return out;
 }
 
-export function site({ url, store, fetch, now = Date.now, name }) {
+export function site({ url, store, fetch, now = Date.now, name, timeout = TIMEOUT }) {
   const root = url.replace(/\/+$/, '');
 
   async function load(id) {
@@ -101,7 +101,7 @@ export function site({ url, store, fetch, now = Date.now, name }) {
   async function request(address, id, entry) {
     const headers = {};
     if (entry?.lastModified) headers['if-modified-since'] = entry.lastModified;
-    const response = await fetch(address, { headers, signal: AbortSignal.timeout(TIMEOUT) });
+    const response = await fetch(address, { headers, signal: AbortSignal.timeout(timeout) });
     if (response.status === 304 && typeof entry?.body === 'string') {
       await response.body?.cancel();
       await save(id, { ...entry, fetched: now(), maxAge: age(response.headers) ?? entry.maxAge });
@@ -131,12 +131,20 @@ export function site({ url, store, fetch, now = Date.now, name }) {
     try {
       found = await request(address, id, entry);
     } catch (error) {
-      found = { why: reason(error) };
+      found = { why: reason(error), transport: true };
     }
     if (found.status) return found;
-    if (!kept)
-      return { status: 'error', message: `${name}: ${address}: ${found.why}`, offline: true };
+    const transport = found.transport ? { transport: true } : {};
+    if (!kept) {
+      return {
+        status: 'error',
+        message: `${name}: ${address}: ${found.why}`,
+        offline: true,
+        ...transport,
+      };
+    }
     return {
+      ...transport,
       status: 'ok',
       body: entry.body,
       stale: true,
@@ -154,12 +162,14 @@ export function site({ url, store, fetch, now = Date.now, name }) {
   async function build(list, offline) {
     const entries = new Array(list.length);
     let next = 0;
-    let down = offline;
+    let down = false;
+    let broken = offline;
     async function worker() {
       while (next < list.length) {
         const at = next++;
         const topic = list[at];
-        const found = await page(topic, { offline: down });
+        const found = await page(topic, { offline: broken });
+        if (found.transport) broken = true;
         if (found.stale || found.offline) down = true;
         entries[at] = { topic, description: found.status === 'ok' ? describe(found.body) : '' };
       }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-ecoverse/slicc-man",
-  "version": "1.1.0",
+  "version": "1.1.2",
   "description": "man, apropos and whatis for SLICC, and scman for SLICC, macOS, Linux and Windows, reading pages from www.sliccy.com/man",
   "type": "module",
   "license": "Apache-2.0",