@riboseinc/anafero-cli 0.0.53 → 0.0.55

package/build-site.mjs

@@ -316974,6 +316974,15 @@ var makeContentReader = async function(entryPointURI, storeAdapters, findContent
   const canonicalURIs = {};
   const originalURIs = {};
   const contentAdapters = {};
+  function getAdapter(canonicalURI) {
+    const adapter3 = contentAdapters[canonicalURI];
+    if (adapter3) {
+      return adapter3;
+    } else {
+      console.error("Resource does not have associated content adapter on record", canonicalURI);
+      throw new Error("Resource does not have associated content adapter on record");
+    }
+  }
   const resourceReaders = {};
   let totalPathCount = 0;
   let totalRelationCount = 0;
@@ -316982,15 +316991,15 @@ var makeContentReader = async function(entryPointURI, storeAdapters, findContent
     const [relationsProgress] = reportProgress("read resources");
     await processRelations(entryPointURI, relationsProgress);
     relationsProgress(null);
-    const [hierarchyProgress] = reportProgress("prepare site structure");
-    processHierarchy(
+    const [pageProgress] = reportProgress("prepare page structure");
+    processPage(
       canonicalURIs[entryPointURI] ?? entryPointURI,
       "",
-      function reportHierarchyProgress(state) {
-        hierarchyProgress({ state });
+      function reportPageProgress(state) {
+        pageProgress({ state });
       }
     );
-    hierarchyProgress(null);
+    pageProgress(null);
   }
   await process14();
   async function processRelations(entryPointURI2, onProgress, _seenEntryPoints) {
@@ -317080,7 +317089,6 @@ var makeContentReader = async function(entryPointURI, storeAdapters, findContent
             } else {
             }
           } else {
-            console.debug("URI relation is seen again", relation.target);
           }
         }
       }
@@ -317159,14 +317167,38 @@ var makeContentReader = async function(entryPointURI, storeAdapters, findContent
       return false;
     }
   }
+  function describeResource(resourceURI) {
+    if (!cache3.has(`metadata/${resourceURI}`)) {
+      const graph = getResourceGraph(resourceURI);
+      const contentAdapter = getAdapter(resourceURI);
+      const meta = contentAdapter.describe(graph);
+      const path3 = cache3.get(`path-for/${resourceURI}`);
+      if (!meta.primaryLanguageID && !path3.includes("#")) {
+        if (cache3.has(`${path3}/parents`)) {
+          const parentResourcePaths = cache3.get(`${path3}/parents`);
+          for (const p3 of parentResourcePaths) {
+            const parentURI = cache3.get(p3);
+            const parentMeta = describeResource(parentURI);
+            if (parentMeta.primaryLanguageID) {
+              meta.primaryLanguageID = parentMeta.primaryLanguageID;
+              break;
+            }
+            console.warn("describeResource: no language, and no parents with language, for", path3);
+          }
+        } else {
+          console.warn("describeResource: no language, and no parents, for", path3);
+        }
+      }
+      cache3.set({
+        [`metadata/${resourceURI}`]: meta
+      });
+    }
+    return cache3.get(`metadata/${resourceURI}`);
+  }
   function getResourceGraph(resourceURI) {
     if (!cache3.has(`graphs/${resourceURI}`)) {
       cache3.add(`graphs/${resourceURI}`, []);
-      const contentAdapter = contentAdapters[resourceURI];
-      if (!contentAdapter) {
-        console.error("Resource does not have associated content adapter on record", resourceURI);
-        throw new Error("Resource does not have associated content adapter on record");
-      }
+      const contentAdapter = getAdapter(resourceURI);
       const queue = [resourceURI];
       while (queue.length > 0) {
         const currentResource = queue.pop();
@@ -317193,7 +317225,7 @@ var makeContentReader = async function(entryPointURI, storeAdapters, findContent
     }
     return cache3.list(`graphs/${resourceURI}`);
   }
-  function processResourceContents(contentAdapter, resourceURI, containingResourcePath, _seen) {
+  function processFragment(contentAdapter, resourceURI, containingResourcePath, _seen) {
     contentAdapters[resourceURI] = contentAdapter;
     const seen = _seen ?? /* @__PURE__ */ new Set();
     cache3.set({
@@ -317207,12 +317239,12 @@ var makeContentReader = async function(entryPointURI, storeAdapters, findContent
           continue;
         } else {
           seen.add(key);
-          processResourceContents(contentAdapter, rel.target, containingResourcePath, seen);
+          processFragment(contentAdapter, rel.target, containingResourcePath, seen);
         }
       }
     }
   }
-  function processHierarchy(resourceURI, pathPrefix, onProgress) {
+  function processPage(resourceURI, pathPrefix, onProgress, meta) {
     const canonicalURI = canonicalURIs[resourceURI] ?? resourceURI;
     const contentAdapter = findContentAdapter(canonicalURI);
     contentAdapters[canonicalURI] = contentAdapter;
@@ -317244,9 +317276,9 @@ var makeContentReader = async function(entryPointURI, storeAdapters, findContent
           throw new Error("Generated path component is not valid");
         }
         const newPath = pathPrefix ? `${pathPrefix}/${maybePathComponent}` : maybePathComponent;
-        processHierarchy(rel.target, newPath, onProgress);
+        processPage(rel.target, newPath, onProgress, meta);
       } else if (isURIString(rel.target)) {
-        processResourceContents(contentAdapter, rel.target, pathPrefix);
+        processFragment(contentAdapter, rel.target, pathPrefix);
       }
     }
   }
@@ -317264,13 +317296,14 @@ var makeContentReader = async function(entryPointURI, storeAdapters, findContent
       yield {
         path: path3,
         resourceURI,
+        meta: describeResource(resourceURI),
         directDescendants: cache3.has(`${path3}/direct-descendants`) ? cache3.list(`${path3}/direct-descendants`).map((path4) => {
           const res = getCachedResourceURIForPath(path4);
-          return [`/${path4}`, res, getResourceGraph(res)];
+          return [`/${path4}`, res, describeResource(res)];
         }) : [],
         parentChain: path3 !== "" ? cache3.list(`${path3}/parents`).map((path4) => {
           const res = getCachedResourceURIForPath(path4);
-          return [`/${path4}`, res, getResourceGraph(res)];
+          return [`/${path4}`, res, describeResource(res)];
         }) : []
       };
     }
@@ -317304,6 +317337,12 @@ var makeContentReader = async function(entryPointURI, storeAdapters, findContent
         throw new Error("Unable to find URL for resource");
       }
     },
+    describe: function describe(resourceURI) {
+      return describeResource(resourceURI);
+    },
+    describeRoot: function describeRoot() {
+      return describeResource(cache3.get(""));
+    },
     resolve: function resolveGraph(resourceURI) {
       return getResourceGraph(resourceURI);
     }
@@ -317352,21 +317391,21 @@ var lunrLanguageSupport = {
 };
 var encoder = new TextEncoder();
 var decoder = new TextDecoder();
-function* generateResourceAssets(resourceURI, relations, parentChain, directDescendants, resourceProps, expandVersionedPath, getDOMStub, inject, workspaceTitle, primaryLanguageID, describe, generateContent) {
+function* generateResourceAssets(resourceURI, relations, parentChain, directDescendants, resourceProps, expandVersionedPath, getDOMStub, inject, workspaceTitle, primaryLanguageID, generateContent) {
   const generatedContent = generateContent(relations, resourceURI);
   if (!generatedContent) {
     console.warn("Resource has no content", resourceURI);
     return;
   }
-  const generateNavLink = function generateNavLink2(path3, uri, graph) {
+  const generateNavLink = function generateNavLink2(path3, uri, meta) {
     return {
       path: expandVersionedPath(path3),
-      plainTitle: describe(graph, uri)?.labelInPlainText ?? uri
+      plainTitle: meta?.labelInPlainText ?? uri
     };
   };
   const resourceNav = {
-    breadcrumbs: parentChain.map(([path3, uri, graph]) => generateNavLink(path3, uri, graph)),
-    children: directDescendants.map(([path3, uri, graph]) => generateNavLink(path3, uri, graph))
+    breadcrumbs: parentChain.map(([path3, uri, meta]) => generateNavLink(path3, uri, meta)),
+    children: directDescendants.map(([path3, uri, meta]) => generateNavLink(path3, uri, meta))
   };
   yield {
     "/resource.json": encoder.encode(JSON.stringify(relations, null, 4)),
@@ -317507,8 +317546,10 @@ ${inject.head ?? ""}`;
   const resourceDescriptions = {};
   const contentCache = {};
   const assetsToWrite = {};
-  let maybePrimaryLanguageID = void 0;
-  let maybeMainTitle = void 0;
+  const rootMeta = reader.describeRoot();
+  const maybePrimaryLanguageID = rootMeta.primaryLanguageID ?? "en";
+  const maybeMainTitle = rootMeta.labelInPlainText ?? "Document";
+  const allLanguages = /* @__PURE__ */ new Set();
   function getReverseResourceMap() {
     return Object.fromEntries(Object.entries(resourceMap).map(([k, v]) => [v, k]));
   }
@@ -317522,14 +317563,7 @@ ${inject.head ?? ""}`;
   let done11 = 0;
   const [allPathProgress, pathSubtask] = reportProgress("build page content", { total: totalPaths, done: done11 });
   const hierarchicalResources = reader.generatePaths();
-  for (const { path: path3, resourceURI, parentChain, directDescendants } of hierarchicalResources) {
-    let contentAdapter;
-    try {
-      contentAdapter = findContentAdapter(resourceURI)?.[1];
-    } catch (e3) {
-      console.error("Unable to find content adapter for resource", resourceURI, e3);
-      continue;
-    }
+  for (const { path: path3, resourceURI, meta, parentChain, directDescendants } of hierarchicalResources) {
     done11 += 1;
     allPathProgress({ total: totalPaths, done: done11 });
     if (path3.startsWith("/")) {
@@ -317544,23 +317578,7 @@ ${inject.head ?? ""}`;
     const relations = reader.resolve(resourceURI);
     resourceMap[path3] = resourceURI;
     resourceGraph.push([resourceURI, "isDefinedBy", `${path3}/resource.json`]);
-    const resourceMeta = contentAdapter.describe(relations);
-    if (path3 === "" && !maybeMainTitle) {
-      maybeMainTitle = resourceMeta.labelInPlainText ?? "Workspace";
-    }
-    if (!maybePrimaryLanguageID) {
-      if (resourceMeta.primaryLanguageID) {
-        console.debug(
-          "Setting primary language ID:",
-          resourceMeta.primaryLanguageID,
-          "based on resource",
-          resourceURI
-        );
-        maybePrimaryLanguageID = resourceMeta.primaryLanguageID;
-      }
-    } else if (!resourceMeta.primaryLanguageID) {
-      resourceMeta.primaryLanguageID = maybePrimaryLanguageID;
-    }
+    const resourceMeta = meta;
     resourceDescriptions[resourceURI] = resourceMeta;
     pathProgress({ state: "processing referenced files" });
     for (const [, , o2] of relations) {
@@ -317605,12 +317623,8 @@ ${inject.head ?? ""}`;
       expandVersionedPath,
       getDOMStub,
       { head: extraHead, tail: inject.tail, htmlAttrs: inject.htmlAttrs },
-      maybeMainTitle ?? "Workspace",
-      resourceMeta.primaryLanguageID ?? maybePrimaryLanguageID ?? "en",
-      function describe(relations2, uri) {
-        const maybeAdapter = findContentAdapter(uri);
-        return maybeAdapter ? maybeAdapter[1].describe(relations2) : null;
-      },
+      maybeMainTitle,
+      resourceMeta.primaryLanguageID ?? maybePrimaryLanguageID,
       function generateContent(relations2, uri) {
         if (!contentCache[uri]) {
           let content;
@@ -317630,9 +317644,10 @@ ${inject.head ?? ""}`;
               );
               throw e3;
             }
+            const primaryLanguageID = resourceMeta.primaryLanguageID ?? maybePrimaryLanguageID;
             contentCache[uri] = {
               adapterID: maybeAdapter[0],
-              content: content ? { primaryLanguageID: maybePrimaryLanguageID, ...content } : null
+              content: content ? { primaryLanguageID, ...content } : null
             };
             if (content) {
               const describedResourceIDs = gatherDescribedResourcesFromJsonifiedProseMirrorNode(content.contentDoc);
@@ -317641,10 +317656,7 @@ ${inject.head ?? ""}`;
                   const pathWithFragment = `${path3}#${encodeURIComponent(inPageResourceID)}`;
                   resourceMap[pathWithFragment] = inPageResourceID;
                   resourceGraph.push([inPageResourceID, "isDefinedBy", `${path3}/resource.json`]);
-                  resourceDescriptions[inPageResourceID] = {
-                    primaryLanguageID: maybePrimaryLanguageID,
-                    ...maybeAdapter[1].describe(relativeGraph(relations2, inPageResourceID))
-                  };
+                  resourceDescriptions[inPageResourceID] = reader.describe(inPageResourceID);
                 } else {
                   console.warn(
                     "Subresource on page does not exist in the graph",
@@ -317683,46 +317695,59 @@ ${inject.head ?? ""}`;
   import_lunr.default.utils.warn = console.warn;
   const lunrIndex = (0, import_lunr.default)(function() {
     console.debug(`Search index: primary language is \u201C${maybePrimaryLanguageID}\u201D`);
-    if (maybePrimaryLanguageID && maybePrimaryLanguageID !== "en" && lunrLanguageSupport[maybePrimaryLanguageID]) {
-      console.debug("Search index: enabling multi-language Lunr mode & mixed tokenizer");
-      lunrLanguageSupport[maybePrimaryLanguageID](import_lunr.default);
+    const supportedLanguages = [...allLanguages].filter((lang) => !!lunrLanguageSupport[lang]);
+    const supportedNonDefaultLanguages = supportedLanguages.filter((lang) => lang !== "en");
+    if (supportedLanguages.length > 1) {
       (0, import_lunr4.default)(import_lunr.default);
-      this.use(import_lunr.default.multiLanguage("en", maybePrimaryLanguageID));
-      this.tokenizer = function(x2) {
-        return import_lunr.default.tokenizer(x2).concat(import_lunr.default[maybePrimaryLanguageID].tokenizer(x2));
-      };
+    }
+    if (maybePrimaryLanguageID) {
+      if (supportedLanguages.length > 1) {
+        for (const lang of supportedNonDefaultLanguages) {
+          lunrLanguageSupport[lang](import_lunr.default);
+        }
+        for (const lang of supportedNonDefaultLanguages) {
+          this.use(import_lunr.default[lang]);
+        }
+        console.debug(
+          "Search index: enabling multi-language Lunr mode & mixed tokenizer",
+          supportedLanguages.join(", ")
+        );
+        this.use(import_lunr.default.multiLanguage(...supportedLanguages));
+        this.tokenizer = function(x2) {
+          return import_lunr.default.tokenizer(x2).concat(...supportedNonDefaultLanguages.map((lang) => import_lunr.default[lang].tokenizer(x2)));
+        };
+      } else if (maybePrimaryLanguageID !== "en") {
+        this.use(import_lunr.default[maybePrimaryLanguageID]);
+      }
     }
     this.ref("name");
     this.field("body");
     let done12 = 0;
     const total = Object.keys(contentCache).length + Object.keys(resourceDescriptions).length;
+    console.debug("Index pipeline functions during", this.pipeline._stack.map((f3) => f3.label));
+    console.debug("Index search pipeline functions during", this.searchPipeline._stack.map((f3) => f3.label));
     for (const [uri, content] of Object.entries(contentCache)) {
       done12 += 1;
       indexProgress({ state: `adding entry for ${uri}`, total, done: done12 });
-      const label = content?.content?.labelInPlainText;
+      const label = content?.content?.labelInPlainText?.normalize("NFKD").replace(/\p{Diacritic}/gu, "").trim();
       if (label) {
         const entry = {
           name: uri,
           body: label
         };
-        this.add(entry);
+        this.add(entry, { boost: 5 });
       } else {
         console.warn("No label for", uri);
       }
     }
-    for (const [uri, desc] of Object.entries(resourceDescriptions)) {
+    for (const [uri] of Object.entries(resourceDescriptions)) {
       done12 += 1;
       indexProgress({ state: "adding entries for subresources", total, done: done12 });
-      const lang = desc.primaryLanguageID;
-      if (lang && lang !== maybePrimaryLanguageID && lang !== "en" && import_lunr.default.hasOwnProperty(lang)) {
-        console.warn("Resource language is different from primary language, this may not work");
-        this.use(import_lunr.default.multiLanguage("en", lang));
-      }
       const rels = reader.resolve(uri);
       const relationsExcludingReferences = rels.filter(
         ([s2, p3, o2]) => p3 === "hasPart" && (s2 === ROOT_SUBJECT || s2 === uri) && !o2.startsWith("data:") && (!isURIString(o2) || !reader.exists(o2))
       );
-      const body = relationsExcludingReferences.map(([s2, p3, o2]) => o2).join("").trim();
+      const body = relationsExcludingReferences.map(([, , o2]) => o2).join("").trim().normalize("NFKD").replace(/\p{Diacritic}/gu, "").trim();
       if (body) {
         const entry = {
           name: uri,
@@ -317733,6 +317758,8 @@ ${inject.head ?? ""}`;
       }
     }
   });
+  console.debug("Index pipeline functions after", lunrIndex.pipeline._stack.map((f3) => f3.label));
+  console.debug("Index search pipeline functions after", lunrIndex.searchPipeline?._stack?.map((f3) => f3.label));
   indexProgress(null);
   yield { "/search-index.json": encoder.encode(JSON.stringify(lunrIndex, null, 4)) };
 }
@@ -317837,9 +317864,6 @@ async function* generateStaticSiteAssets(versions, currentVersionID, opts) {
     versionProgress(null);
   }
 }
-function relativeGraph(relations, subj) {
-  return relations.filter(([s2]) => s2 !== ROOT_SUBJECT).map(([s2, p3, o2]) => [s2 === subj ? ROOT_SUBJECT : s2, p3, o2]);
-}
 
 // dependencies.mts
 init_cjs_shim();

package/package.json

@@ -1,17 +1,17 @@
 {
   "name": "@riboseinc/anafero-cli",
   "type": "module",
-  "version": "0.0.53",
+  "version": "0.0.55",
   "packageManager": "yarn@4.5.0",
   "bin": {
     "build-site": "build-site.mjs"
   },
   "scripts": {
-    "cbp": "yarn clean-all; yarn compile; yarn build-generator-builder && yarn build-generator -- --debug && sed 's#^import devtools#//import devtools#' build-site.mjs > ._tmp && mv ._tmp build-site.mjs && npm pack",
-    "bp": "rm -f riboseinc-anafero-*.tgz; yarn build-generator -- --debug && npm pack",
+    "cbp": "yarn clean-all; yarn compile; yarn build-generator-builder && yarn build-generator -- --debug && sed 's#^import devtools#//import devtools#' build-site.mjs > ._tmp && mv ._tmp build-site.mjs && npm pack && mv *.tgz dist/",
+    "bp": "rm -f dist/riboseinc-anafero-*.tgz; yarn build-generator -- --debug && npm pack",
     "build-generator-builder": "yarn esbuild build-generator.mts --log-level=debug --packages=external --platform=node --target=node18 --minify=false --bundle --format=esm --outfile=build-generator.mjs",
     "build-generator": "yarn node build-generator.mjs --",
-    "clean-all": "rm -f riboseinc-anafero-*.tgz build-generator.mjs build-site.mjs bootstrap.js *.css",
+    "clean-all": "find compiled -name 'README.rst' -prune -o -delete; rm -f dist/riboseinc-anafero-*.tgz build-generator.mjs build-site.mjs bootstrap.js *.css",
     "compile": "tsc --outdir compiled"
   },
   "dependencies": {

package/tsconfig.json

@@ -18,9 +18,6 @@
     "exactOptionalPropertyTypes": true,
     "skipLibCheck": true,
 
-    "allowImportingTsExtensions": true, // XXX
-    "noEmit": true, // XXX
-
     "isolatedModules": true,
     "esModuleInterop": true,