@riboseinc/anafero-cli 0.0.55 → 0.0.57

package/build-site.mjs

@@ -313300,7 +313300,9 @@ var ResourceMetadataSchema = mutable3(Struct({
   // TODO: rich ProseMirror labels?
   labelInPlainText: String$2.pipe(nonEmptyString4()),
   // TODO: proper schema for ISO language IDs
-  primaryLanguageID: String$2.pipe(optional)
+  primaryLanguageID: String$2.pipe(optional),
+  /** License/copyright for the resource in question. */
+  license: Any.pipe(optional)
 }));
 function gatherDescribedResourcesFromJsonifiedProseMirrorNode(jsonifiedNode, _accum) {
   const accumulator = _accum ?? /* @__PURE__ */ new Set();
@@ -317391,12 +317393,7 @@ var lunrLanguageSupport = {
 };
 var encoder = new TextEncoder();
 var decoder = new TextDecoder();
-function* generateResourceAssets(resourceURI, relations, parentChain, directDescendants, resourceProps, expandVersionedPath, getDOMStub, inject, workspaceTitle, primaryLanguageID, generateContent) {
-  const generatedContent = generateContent(relations, resourceURI);
-  if (!generatedContent) {
-    console.warn("Resource has no content", resourceURI);
-    return;
-  }
+function* generateResourceAssets(resourceURI, relations, parentChain, directDescendants, resourceProps, expandVersionedPath, getDOMStub, inject, workspaceTitle, primaryLanguageID, generatedContent) {
   const generateNavLink = function generateNavLink2(path3, uri, meta) {
     return {
       path: expandVersionedPath(path3),
@@ -317549,7 +317546,7 @@ ${inject.head ?? ""}`;
   const rootMeta = reader.describeRoot();
   const maybePrimaryLanguageID = rootMeta.primaryLanguageID ?? "en";
   const maybeMainTitle = rootMeta.labelInPlainText ?? "Document";
-  const allLanguages = /* @__PURE__ */ new Set();
+  const allLanguages = /* @__PURE__ */ new Set(["en"]);
   function getReverseResourceMap() {
     return Object.fromEntries(Object.entries(resourceMap).map(([k, v]) => [v, k]));
   }
@@ -317575,110 +317572,115 @@ ${inject.head ?? ""}`;
       throw new Error("Malformed resource URL while processing resources: hash fragment");
     }
     const [pathProgress] = pathSubtask(`${resourceURI.replaceAll("|", ":")}`, { state: "resolving relations" });
-    const relations = reader.resolve(resourceURI);
-    resourceMap[path3] = resourceURI;
-    resourceGraph.push([resourceURI, "isDefinedBy", `${path3}/resource.json`]);
     const resourceMeta = meta;
-    resourceDescriptions[resourceURI] = resourceMeta;
-    pathProgress({ state: "processing referenced files" });
-    for (const [, , o2] of relations) {
-      if (o2.startsWith("file:")) {
-        const filePath = o2.split("file:")[1];
-        const filename = o2.replaceAll("/", "_").replaceAll(":", "_");
-        if (!assetsToWrite[filename]) {
+    const relations = reader.resolve(resourceURI);
+    pathProgress({ state: "generating resource page content" });
+    const content = function generateContent(graph, metadata, uri, cache4) {
+      if (!cache4[uri]) {
+        let content2;
+        const maybeAdapter = findContentAdapter(uri);
+        if (maybeAdapter) {
           try {
-            assetsToWrite[filename] = await fetchBlobAtThisVersion(filePath);
-            resourceGraph.push([
-              o2,
-              "isDownloadableAt",
-              filename
-            ]);
-            resourceMap[filename] = o2;
-            resourceDescriptions[o2] = {
-              labelInPlainText: filePath
-            };
+            content2 = maybeAdapter[1].generateContent(graph) ?? null;
           } catch (e3) {
-            console.error("Failed to fetch asset", filePath);
+            console.error(
+              "Failed to generate resource content",
+              path3,
+              uri,
+              graph.slice(0, 40).join("\n")
+            );
+            throw e3;
           }
+          if (content2) {
+            contentCache[uri] = {
+              adapterID: maybeAdapter[0],
+              content: { ...metadata, ...content2 }
+            };
+            if (metadata.primaryLanguageID) {
+              allLanguages.add(metadata.primaryLanguageID);
+            }
+            const describedResourceIDs = gatherDescribedResourcesFromJsonifiedProseMirrorNode(content2.contentDoc);
+            for (const inPageResourceID of describedResourceIDs) {
+              if (reader.exists(inPageResourceID)) {
+                const pathWithFragment = `${path3}#${encodeURIComponent(inPageResourceID)}`;
+                resourceMap[pathWithFragment] = inPageResourceID;
+                resourceGraph.push([inPageResourceID, "isDefinedBy", `${path3}/resource.json`]);
+                resourceDescriptions[inPageResourceID] = reader.describe(inPageResourceID);
+              } else {
+                console.warn(
+                  "Subresource on page does not exist in the graph",
+                  path3,
+                  inPageResourceID
+                );
+              }
+            }
+          }
+        } else {
+          console.warn("No adapter found to render", uri);
+          return null;
         }
+      } else {
+        console.debug("contentCache hit", uri);
       }
-    }
+      return cache4[uri];
+    }(relations, meta, resourceURI, contentCache);
     pathProgress({ state: "generating page content & assets" });
-    const resourceAssetGenerator = generateResourceAssets(
-      resourceURI,
-      relations,
-      parentChain,
-      directDescendants,
-      {
-        useDependency: getDependency,
-        selectedLayout: layouts[0],
-        getResourcePlainTitle: (uri) => resourceDescriptions[uri]?.labelInPlainText ?? uri,
-        onIntegrityViolation: console.warn,
-        reverseResource: (path4) => getReverseResourceMap()[path4],
-        uri: resourceURI,
-        // TODO: Consider slash-prepending the outcome of findURL,
-        // if it’s reliably not slash-prepended
-        locateResource: (uri) => expandVersionedPath(reader.findURL(uri))
-      },
-      expandVersionedPath,
-      getDOMStub,
-      { head: extraHead, tail: inject.tail, htmlAttrs: inject.htmlAttrs },
-      maybeMainTitle,
-      resourceMeta.primaryLanguageID ?? maybePrimaryLanguageID,
-      function generateContent(relations2, uri) {
-        if (!contentCache[uri]) {
-          let content;
-          const maybeAdapter = findContentAdapter(uri);
-          if (maybeAdapter) {
+    if (content !== null) {
+      resourceMap[path3] = resourceURI;
+      resourceGraph.push([resourceURI, "isDefinedBy", `${path3}/resource.json`]);
+      resourceDescriptions[resourceURI] = resourceMeta;
+      pathProgress({ state: "processing resource assets" });
+      for (const [, , o2] of relations) {
+        if (o2.startsWith("file:")) {
+          const filePath = o2.split("file:")[1];
+          const filename = o2.replaceAll("/", "_").replaceAll(":", "_");
+          if (!assetsToWrite[filename]) {
             try {
-              content = maybeAdapter[1].generateContent(
-                relations2
-              ) ?? null;
+              assetsToWrite[filename] = await fetchBlobAtThisVersion(filePath);
+              resourceGraph.push([
+                o2,
+                "isDownloadableAt",
+                filename
+              ]);
+              resourceMap[filename] = o2;
+              resourceDescriptions[o2] = {
+                labelInPlainText: filePath
+              };
             } catch (e3) {
-              console.error(
-                "Failed to generate resource content",
-                path3,
-                resourceMeta,
-                uri,
-                relations2.slice(0, 40).join("\n")
-              );
-              throw e3;
+              console.error("Failed to fetch asset", filePath);
             }
-            const primaryLanguageID = resourceMeta.primaryLanguageID ?? maybePrimaryLanguageID;
-            contentCache[uri] = {
-              adapterID: maybeAdapter[0],
-              content: content ? { primaryLanguageID, ...content } : null
-            };
-            if (content) {
-              const describedResourceIDs = gatherDescribedResourcesFromJsonifiedProseMirrorNode(content.contentDoc);
-              for (const inPageResourceID of describedResourceIDs) {
-                if (reader.exists(inPageResourceID)) {
-                  const pathWithFragment = `${path3}#${encodeURIComponent(inPageResourceID)}`;
-                  resourceMap[pathWithFragment] = inPageResourceID;
-                  resourceGraph.push([inPageResourceID, "isDefinedBy", `${path3}/resource.json`]);
-                  resourceDescriptions[inPageResourceID] = reader.describe(inPageResourceID);
-                } else {
-                  console.warn(
-                    "Subresource on page does not exist in the graph",
-                    path3,
-                    inPageResourceID
-                  );
-                }
-              }
-            }
-          } else {
-            console.warn("No adapter found to render", uri);
-            return null;
           }
         }
-        return contentCache[uri];
       }
-    );
-    for (const blobChunk of resourceAssetGenerator) {
-      yield Object.entries(blobChunk).map(([subpath, blob]) => {
-        const assetBlobPath = `/${path3}${subpath}`;
-        return { [assetBlobPath]: blob };
-      }).reduce((prev, curr) => ({ ...prev, ...curr }));
+      const resourceAssetGenerator = generateResourceAssets(
+        resourceURI,
+        relations,
+        parentChain,
+        directDescendants,
+        {
+          useDependency: getDependency,
+          selectedLayout: layouts[0],
+          getResourcePlainTitle: (uri) => resourceDescriptions[uri]?.labelInPlainText ?? uri,
+          onIntegrityViolation: console.warn,
+          reverseResource: (path4) => getReverseResourceMap()[path4],
+          uri: resourceURI,
+          // TODO: Consider slash-prepending the outcome of findURL,
+          // if it’s reliably not slash-prepended
+          locateResource: (uri) => expandVersionedPath(reader.findURL(uri))
+        },
+        expandVersionedPath,
+        getDOMStub,
+        { head: extraHead, tail: inject.tail, htmlAttrs: inject.htmlAttrs },
+        maybeMainTitle,
+        resourceMeta.primaryLanguageID ?? maybePrimaryLanguageID,
+        content
+      );
+      for (const blobChunk of resourceAssetGenerator) {
+        yield Object.entries(blobChunk).map(([subpath, blob]) => {
+          const assetBlobPath = `/${path3}${subpath}`;
+          return { [assetBlobPath]: blob };
+        }).reduce((prev, curr) => ({ ...prev, ...curr }));
+      }
     }
     pathProgress(null);
   }
@@ -317693,54 +317695,54 @@ ${inject.head ?? ""}`;
   };
   const [indexProgress] = reportProgress("build search index");
   import_lunr.default.utils.warn = console.warn;
+  const supportedLanguages = [...allLanguages].filter((lang) => lang === "en" || !!lunrLanguageSupport[lang]);
+  console.debug(`Search index: primary language is \u201C${maybePrimaryLanguageID}\u201D, enabling ${supportedLanguages.join(", ")}`);
+  const nonDefaultLanguages = supportedLanguages.filter((lang) => lang !== "en");
+  if (supportedLanguages.length > 1) {
+    for (const lang of nonDefaultLanguages) {
+      lunrLanguageSupport[lang](import_lunr.default);
+    }
+    (0, import_lunr4.default)(import_lunr.default);
+  }
   const lunrIndex = (0, import_lunr.default)(function() {
-    console.debug(`Search index: primary language is \u201C${maybePrimaryLanguageID}\u201D`);
-    const supportedLanguages = [...allLanguages].filter((lang) => !!lunrLanguageSupport[lang]);
-    const supportedNonDefaultLanguages = supportedLanguages.filter((lang) => lang !== "en");
     if (supportedLanguages.length > 1) {
-      (0, import_lunr4.default)(import_lunr.default);
-    }
-    if (maybePrimaryLanguageID) {
-      if (supportedLanguages.length > 1) {
-        for (const lang of supportedNonDefaultLanguages) {
-          lunrLanguageSupport[lang](import_lunr.default);
-        }
-        for (const lang of supportedNonDefaultLanguages) {
-          this.use(import_lunr.default[lang]);
+      this.use(import_lunr.default.multiLanguage(...["en", ...nonDefaultLanguages]));
+      console.debug(
+        "Search index: enabling multi-language Lunr mode & mixed tokenizer",
+        supportedLanguages.join(", ")
+      );
+      const lunrTokenizer = import_lunr.default.tokenizer;
+      this.tokenizer = function(x2) {
+        const baseLunrTokens = lunrTokenizer(x2);
+        const tokens = [...baseLunrTokens];
+        for (const lang of nonDefaultLanguages) {
+          const tokenizer = import_lunr.default[lang].tokenizer;
+          if (tokenizer) {
+            const langTokens = tokenizer(x2);
+            tokens.push(...langTokens.filter(
+              (t3) => !baseLunrTokens.find((bt) => bt.toString() === t3.toString())
+            ));
+          } else {
+          }
         }
-        console.debug(
-          "Search index: enabling multi-language Lunr mode & mixed tokenizer",
-          supportedLanguages.join(", ")
-        );
-        this.use(import_lunr.default.multiLanguage(...supportedLanguages));
-        this.tokenizer = function(x2) {
-          return import_lunr.default.tokenizer(x2).concat(...supportedNonDefaultLanguages.map((lang) => import_lunr.default[lang].tokenizer(x2)));
-        };
-      } else if (maybePrimaryLanguageID !== "en") {
-        this.use(import_lunr.default[maybePrimaryLanguageID]);
-      }
+        return tokens;
+      };
+      const lunrStopWordFilter = import_lunr.default.stopWordFilter;
+      this.stopWordFilter = function(token) {
+        return lunrStopWordFilter(token) && !nonDefaultLanguages.map(
+          (lang) => !!this[lang].stopWordFilter(token)
+        ).includes(false) ? token : void 0;
+      };
     }
     this.ref("name");
+    this.field("title", { boost: 2 });
     this.field("body");
     let done12 = 0;
     const total = Object.keys(contentCache).length + Object.keys(resourceDescriptions).length;
     console.debug("Index pipeline functions during", this.pipeline._stack.map((f3) => f3.label));
     console.debug("Index search pipeline functions during", this.searchPipeline._stack.map((f3) => f3.label));
-    for (const [uri, content] of Object.entries(contentCache)) {
-      done12 += 1;
-      indexProgress({ state: `adding entry for ${uri}`, total, done: done12 });
-      const label = content?.content?.labelInPlainText?.normalize("NFKD").replace(/\p{Diacritic}/gu, "").trim();
-      if (label) {
-        const entry = {
-          name: uri,
-          body: label
-        };
-        this.add(entry, { boost: 5 });
-      } else {
-        console.warn("No label for", uri);
-      }
-    }
-    for (const [uri] of Object.entries(resourceDescriptions)) {
+    const searchableResources = { pages: [], resources: [] };
+    for (const [uri, meta] of Object.entries(resourceDescriptions)) {
       done12 += 1;
       indexProgress({ state: "adding entries for subresources", total, done: done12 });
       const rels = reader.resolve(uri);
@@ -317748,15 +317750,28 @@ ${inject.head ?? ""}`;
         ([s2, p3, o2]) => p3 === "hasPart" && (s2 === ROOT_SUBJECT || s2 === uri) && !o2.startsWith("data:") && (!isURIString(o2) || !reader.exists(o2))
       );
       const body = relationsExcludingReferences.map(([, , o2]) => o2).join("").trim().normalize("NFKD").replace(/\p{Diacritic}/gu, "").trim();
-      if (body) {
+      const title = meta.labelInPlainText.normalize("NFKD").replace(/\p{Diacritic}/gu, "").trim();
+      if (body || title) {
         const entry = {
           name: uri,
-          body
+          body,
+          title,
+          lang: meta.primaryLanguageID ?? ""
         };
-        this.add(entry);
+        if (contentCache[uri]) {
+          searchableResources.pages.push(entry);
+        } else {
+          searchableResources.resources.push(entry);
+        }
       } else {
       }
     }
+    for (const entry of searchableResources.pages) {
+      this.add(entry, { boost: 2 });
+    }
+    for (const entry of searchableResources.resources) {
+      this.add(entry);
+    }
   });
   console.debug("Index pipeline functions after", lunrIndex.pipeline._stack.map((f3) => f3.label));
   console.debug("Index search pipeline functions after", lunrIndex.searchPipeline?._stack?.map((f3) => f3.label));

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@riboseinc/anafero-cli",
   "type": "module",
-  "version": "0.0.55",
+  "version": "0.0.57",
   "packageManager": "yarn@4.5.0",
   "bin": {
     "build-site": "build-site.mjs"