@riboseinc/anafero-cli 0.0.66 → 0.0.68

package/build-site.mjs

@@ -294425,7 +294425,88 @@ function makeDummyInMemoryCache() {
         throw new Error("Requested key does not exist");
       }
     },
-    dump: () => cache3
+    dump: async (stream4, signal) => {
+      async function* buildKeypaths(obj, currentPath, seen, paths2, signal2) {
+        if (signal2.aborted) {
+          throw new Error("Aborted");
+        }
+        for (const key of Object.keys(Object(obj))) {
+          const val = obj[key];
+          if (typeof val === "string" || Object.keys(Object(val)).length === 0) {
+            yield [...currentPath, key];
+          } else {
+            const complexVal = val;
+            if (!seen.has(complexVal)) {
+              seen.add(complexVal);
+              yield* buildKeypaths(
+                complexVal,
+                [...currentPath, key],
+                seen,
+                paths2,
+                signal2
+              );
+            }
+          }
+        }
+      }
+      async function write4(data, end6) {
+        return new Promise((resolve4) => {
+          if (end6) {
+            stream4.end(data, function() {
+              resolve4();
+            });
+          } else {
+            stream4.write(data, function() {
+              resolve4();
+            });
+          }
+        });
+      }
+      function serializeKeypath(idx2, keyPaths) {
+        const keyPath = keyPaths[idx2];
+        if (keyPath) {
+          const keyString = keyPath.join(".");
+          keyPath.reverse();
+          let valueCursor = cache3;
+          while (true) {
+            let part = keyPath.pop();
+            if (part) {
+              valueCursor = valueCursor[part];
+            } else {
+              break;
+            }
+          }
+          return `
+${JSON.stringify(keyString)}: ${valueCursor}`;
+        } else {
+          return null;
+        }
+      }
+      const pathGen = buildKeypaths(cache3, [], /* @__PURE__ */ new Set(), [], signal);
+      const paths = [];
+      for await (const p3 of pathGen) {
+        paths.push(p3);
+      }
+      await write4(`
+-- starting dumping ${paths.length} paths --`);
+      let idx = 0;
+      const chunkSize = 1e3;
+      let chunk4 = "";
+      while (true) {
+        if (idx > 0 && idx % chunkSize === 0) {
+          await write4(chunk4);
+          chunk4 = "";
+        }
+        const nextItem = serializeKeypath(idx, paths);
+        if (nextItem) {
+          chunk4 = chunk4 + nextItem;
+        } else {
+          break;
+        }
+        idx += 1;
+      }
+      return;
+    }
   };
 }
 
@@ -313425,6 +313506,11 @@ var BuildConfigSchema = Struct({
   version: Literal4("0.1"),
   /** The entry point, path to XML of a Metanorma document or collection. */
   entryPoint: String$2.pipe(nonEmptyString4()),
+  /**
+   * Workspace title.
+   * If not provided, “document ID • document title” is used.
+   */
+  workspaceTitle: String$2.pipe(nonEmptyString4()).pipe(optional),
   /**
    * Which reader modules to use.
    *
@@ -313648,12 +313734,48 @@ function getTextContent(graph, subject, partPredicate) {
 // ../anafero/search.mts
 init_cjs_shim();
 function preprocessStringForIndexing(text9) {
-  return text9.normalize("NFKD").replace(/\p{Diacritic}/gu, "").trim();
+  return text9.normalize("NFKD").replace(/\p{Diacritic}/gu, "").replace(/[\p{P}$+<=>^`|~]/gu, " ").replace(/[\u200B-\u200D\uFEFF]/g, "").trim();
 }
-function extractRelationsForIndexing(uri, graph, isDefinedSubject) {
-  return graph.filter(
-    ([s2, p3, o2]) => p3 === "hasPart" && (s2 === uri || s2 === ROOT_SUBJECT) && !o2.startsWith("data:") && (!isURIString(o2) || !isDefinedSubject(o2))
+function extractRelationsForIndexing(uri, graph, isIndexable, isAlreadyIndexed, _seen, _log) {
+  const seen = _seen ?? /* @__PURE__ */ new Set();
+  seen.add(uri);
+  const nonData = graph.filter(
+    ([, , o2]) => !o2.startsWith("data:")
+  );
+  const immediateGraph = nonData.filter(
+    ([s2, ,]) => s2 === uri || s2 === ROOT_SUBJECT
   );
+  const references = immediateGraph.filter(
+    ([, , o2]) => isURIString(o2)
+  );
+  const indexable = immediateGraph.filter(
+    ([s2, p3, o2]) => !isURIString(o2) && isIndexable([s2, p3, o2])
+  ).map(([, , o2]) => o2).filter((o2) => o2.trim() !== "");
+  for (const [, , o2] of references) {
+    if (_log) {
+      console.debug(
+        "search: processing triple object",
+        { o: o2, isAlreadyIndexed: isAlreadyIndexed(o2) }
+      );
+    }
+    if (!isAlreadyIndexed(o2) && !seen.has(o2)) {
+      indexable.push(...extractRelationsForIndexing(
+        o2,
+        graph,
+        isIndexable,
+        isAlreadyIndexed,
+        seen,
+        _log
+      ));
+    }
+  }
+  if (_log) {
+    console.debug(
+      "search: obtained indexable from graph",
+      { uri, graph, indexable }
+    );
+  }
+  return indexable;
 }
 
 // ../anafero-gui/loader.mts
@@ -317589,7 +317711,7 @@ ${inject.head ?? ""}`;
   const assetsToWrite = {};
   const rootMeta = reader.describeRoot();
   const maybePrimaryLanguageID = rootMeta.primaryLanguageID ?? "en";
-  const maybeMainTitle = rootMeta.labelInPlainText ?? "Document";
+  const maybeMainTitle = cfg.workspaceTitle ?? rootMeta.labelInPlainText ?? "Document";
   const allLanguages = /* @__PURE__ */ new Set(["en"]);
   function getReverseResourceMap() {
     return Object.fromEntries(Object.entries(resourceMap).map(([k, v]) => [v, k]));
@@ -317656,41 +317778,66 @@ ${inject.head ?? ""}`;
     );
     pathProgress({ state: "updating artifacts" });
     if (content?.content) {
+      let isIndexable2 = function(rel) {
+        return rel[1] === "hasPart";
+      }, isIndexed2 = function(uri) {
+        return reader.exists(uri) && describedResourceIDs.has(uri);
+      };
+      var isIndexable = isIndexable2, isIndexed = isIndexed2;
       if (meta.primaryLanguageID) {
         allLanguages.add(meta.primaryLanguageID);
       }
       pathProgress({ state: "indexing page resource" });
+      const describedResourceIDs = gatherDescribedResourcesFromJsonifiedProseMirrorNode(
+        content.content.contentDoc
+      );
       resourceMap[path3] = resourceURI;
-      resourceGraph.push([resourceURI, "isDefinedBy", `${path3}/resource.json`]);
+      resourceGraph.push([
+        resourceURI,
+        "isDefinedBy",
+        `${path3}/resource.json`
+      ]);
       resourceDescriptions[resourceURI] = resourceMeta;
       searchableResources.pages[resourceURI] = {
         name: resourceURI,
         title: preprocessStringForIndexing(meta.labelInPlainText),
         lang: meta.primaryLanguageID || "",
         body: preprocessStringForIndexing(
-          extractRelationsForIndexing(resourceURI, relations, reader.exists).join("").trim()
+          extractRelationsForIndexing(
+            resourceURI,
+            relations,
+            isIndexable2,
+            isIndexed2
+          ).join("\n").trim()
         )
       };
       pathProgress({ state: "indexing on-page subresources" });
-      const describedResourceIDs = gatherDescribedResourcesFromJsonifiedProseMirrorNode(
-        content.content.contentDoc
-      );
       for (const inPageResourceID of describedResourceIDs) {
         if (reader.exists(inPageResourceID)) {
           const pathWithFragment = `${path3}#${encodeURIComponent(inPageResourceID)}`;
           const meta2 = reader.describe(inPageResourceID);
           const graph2 = reader.resolve(inPageResourceID);
           resourceMap[pathWithFragment] = inPageResourceID;
-          resourceGraph.push([inPageResourceID, "isDefinedBy", `${path3}/resource.json`]);
+          resourceGraph.push([
+            inPageResourceID,
+            "isDefinedBy",
+            `${path3}/resource.json`
+          ]);
           resourceDescriptions[inPageResourceID] = meta2;
           if (inPageResourceID !== resourceURI) {
+            const body = preprocessStringForIndexing(
+              extractRelationsForIndexing(
+                inPageResourceID,
+                graph2,
+                isIndexable2,
+                isIndexed2
+              ).join("\n").trim()
+            );
             searchableResources.resources[inPageResourceID] = {
               name: inPageResourceID,
               title: preprocessStringForIndexing(meta2.labelInPlainText),
               lang: meta2.primaryLanguageID || "",
-              body: preprocessStringForIndexing(
-                extractRelationsForIndexing(inPageResourceID, graph2, reader.exists).join("").trim()
-              )
+              body
             };
           }
         } else {
@@ -317785,7 +317932,7 @@ ${inject.head ?? ""}`;
         supportedLanguages.join(", ")
       );
       const lunrTokenizer = import_lunr.default.tokenizer;
-      this.tokenizer = function(x2) {
+      this.tokenizer = function anaferoTokenizer(x2) {
         const baseLunrTokens = lunrTokenizer(x2);
         const tokens = [...baseLunrTokens];
         for (const lang of nonDefaultLanguages) {
@@ -317801,7 +317948,7 @@ ${inject.head ?? ""}`;
         return tokens;
       };
       const lunrStopWordFilter = import_lunr.default.stopWordFilter;
-      this.stopWordFilter = function(token) {
+      this.stopWordFilter = function anaferoStopWordFilter(token) {
         return lunrStopWordFilter(token) && !nonDefaultLanguages.map(
           (lang) => !!this[lang].stopWordFilter(token)
         ).includes(false) ? token : void 0;
@@ -317825,7 +317972,9 @@ ${inject.head ?? ""}`;
     }
   });
   indexProgress(null);
-  yield { "/search-index.json": encoder.encode(JSON.stringify(lunrIndex, null, 4)) };
+  yield {
+    "/search-index.json": encoder.encode(JSON.stringify(lunrIndex, null, 4))
+  };
 }
 async function* generateStaticSiteAssets(versions, currentVersionID, opts) {
   const versionIDsSorted = Array.from(Object.entries(versions)).toSorted(
@@ -322415,22 +322564,8 @@ var buildSite = Command_exports2.make(
           if (prefix && (!prefix.startsWith("/") || prefix.endsWith("/"))) {
             throw new Error("Path prefix must have a leading slash and no trailing slash");
           }
-          function maybeDumpCache() {
-            if (debug3) {
-              try {
-                console.warn("Dumping cache to cacheDump.json");
-                fs4.writeFileSync(
-                  "cacheDump.json",
-                  JSON.stringify(cache3.dump(), null, 2),
-                  { encoding: "utf-8" }
-                );
-              } catch (e3) {
-                console.error("Cache dump could not be written!", e3);
-              }
-            }
-          }
-          process.on("SIGINT", maybeDumpCache);
-          render_default(/* @__PURE__ */ import_react129.default.createElement(Processor, { rootTaskName: "build site", onStart: async function({ onProgress }) {
+          let dumping = false;
+          async function generate({ onProgress }) {
             try {
               const generator = generateSite(
                 {
@@ -322457,14 +322592,65 @@ var buildSite = Command_exports2.make(
               }
               writeProgress(null);
               onProgress("build site", null);
+              await maybeDumpCache();
               resolve4(void 0);
             } catch (e3) {
+              await maybeDumpCache();
               reject(e3);
             } finally {
-              maybeDumpCache();
-              process.removeListener("SIGINT", maybeDumpCache);
             }
-          } }));
+          }
+          async function maybeDumpCache() {
+            if (dumping || !debug3) {
+              return;
+            }
+            const ac = new AbortController();
+            const filename = "cacheDump.txt";
+            function abortDumpCache() {
+              console.warn("Aborting cache dump");
+              process.removeListener("SIGTERM", abortDumpCache);
+              process.removeListener("SIGINT", abortDumpCache);
+              ac.abort();
+              reject("Aborted");
+            }
+            process.on("SIGTERM", abortDumpCache);
+            process.on("SIGINT", abortDumpCache);
+            return new Promise((resolve5, reject2) => {
+              process.removeListener("SIGINT", maybeDumpCache);
+              dumping = true;
+              try {
+                const stream4 = fs4.createWriteStream(filename, {
+                  flags: "w",
+                  autoClose: true,
+                  emitClose: true,
+                  flush: true,
+                  encoding: "utf-8",
+                  signal: ac.signal
+                });
+                stream4.on("error", function handleCacheDumpStreamError(e3) {
+                  console.error("Error writing cache dump", e3);
+                  reject2(e3);
+                });
+                stream4.on("close", function handleCloseCacheDumpStream() {
+                  console.warn("Cache dump stream close event");
+                  resolve5(void 0);
+                  stream4.close(function() {
+                    console.warn("Exiting");
+                  });
+                });
+                stream4.on("ready", function handleOpenCacheDumpStream() {
+                  console.warn("Dumping cache due to debug flag");
+                  cache3.dump(stream4, ac.signal).then(resolve5, reject2);
+                });
+              } catch (e3) {
+                console.error("Could not start cache dump stream", e3);
+                reject2(e3);
+              } finally {
+              }
+            });
+          }
+          process.on("SIGINT", maybeDumpCache);
+          render_default(/* @__PURE__ */ import_react129.default.createElement(Processor, { rootTaskName: "build site", onStart: generate }));
         }),
         catch: (e3) => {
           console.error(e3);

package/generate-to-filesystem.tsx

@@ -156,22 +156,10 @@ const buildSite = Command.
             if (prefix && (!prefix.startsWith('/') || prefix.endsWith('/'))) {
               throw new Error("Path prefix must have a leading slash and no trailing slash");
             }
-            function maybeDumpCache() {
-              if (debug) {
-                try {
-                  console.warn("Dumping cache to cacheDump.json");
-                  fs.writeFileSync(
-                    'cacheDump.json',
-                    JSON.stringify(cache.dump(), null, 2),
-                    { encoding: 'utf-8' },
-                  );
-                } catch (e) {
-                  console.error("Cache dump could not be written!", e);
-                }
-              }
-            }
-            process.on('SIGINT', maybeDumpCache);
-            render(<Processor rootTaskName="build site" onStart={async function ({ onProgress }) {
+
+            let dumping = false;
+
+            async function generate({ onProgress }: { onProgress: TaskProgressCallback }) {
               try {
                 const generator = generateSite(
                   {
@@ -198,14 +186,80 @@ const buildSite = Command.
                 }
                 writeProgress(null);
                 onProgress('build site', null);
+                await maybeDumpCache();
                 resolve(void 0);
               } catch (e) {
+                await maybeDumpCache();
                 reject(e);
               } finally {
-                maybeDumpCache();
-                process.removeListener('SIGINT', maybeDumpCache);
+                // Can’t have maybeDumpCache() here, because we want
+                // to call it before resolving/rejecting the promise.
               }
-            }} />);
+            }
+
+            async function maybeDumpCache() {
+              if (dumping || !debug) {
+                return;
+              }
+
+              const ac = new AbortController();
+              const filename = 'cacheDump.txt';
+
+              function abortDumpCache() {
+                console.warn("Aborting cache dump");
+                process.removeListener('SIGTERM', abortDumpCache);
+                process.removeListener('SIGINT', abortDumpCache);
+                ac.abort();
+                reject("Aborted");
+              }
+
+              process.on('SIGTERM', abortDumpCache);
+              process.on('SIGINT', abortDumpCache);
+
+              return new Promise((resolve, reject) => {
+                process.removeListener('SIGINT', maybeDumpCache);
+                dumping = true;
+
+                try {
+                  const stream = fs.createWriteStream(filename, {
+                    flags: 'w',
+                    autoClose: true,
+                    emitClose: true,
+                    flush: true,
+                    encoding: 'utf-8',
+                    signal: ac.signal,
+                  });
+
+                  stream.on('error', function handleCacheDumpStreamError(e) {
+                    console.error("Error writing cache dump", e);
+                    reject(e);
+                  });
+
+                  stream.on('close', function handleCloseCacheDumpStream() {
+                    console.warn("Cache dump stream close event");
+                    resolve(void 0);
+
+                    stream.close(function () {
+                      console.warn("Exiting");
+                    });
+                  });
+
+                  stream.on('ready', function handleOpenCacheDumpStream() {
+                    console.warn("Dumping cache due to debug flag");
+                    cache.dump(stream, ac.signal).then(resolve, reject);
+                  });
+
+                } catch (e) {
+                  console.error("Could not start cache dump stream", e);
+                  reject(e);
+                } finally {
+                }
+              });
+            }
+
+            process.on('SIGINT', maybeDumpCache);
+
+            render(<Processor rootTaskName="build site" onStart={generate} />);
           }),
           catch: (e) => {
             console.error(e);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@riboseinc/anafero-cli",
   "type": "module",
-  "version": "0.0.66",
+  "version": "0.0.68",
   "packageManager": "yarn@4.5.0",
   "bin": {
     "build-site": "build-site.mjs"