@helloao/cli 0.0.19 → 0.1.0

package/dist/cjs/cli.cjs

@@ -12333,7 +12333,10 @@ var verseSchema = external_exports.object({
   id: external_exports.string(),
   bookCode: external_exports.string().optional(),
   chapter: external_exports.number().optional(),
-  verse: external_exports.number().optional()
+  verse: external_exports.number().optional(),
+  data: external_exports.object({
+    globalReferences: external_exports.array(external_exports.string()).optional()
+  }).optional()
 });
 var paratextSchema = external_exports.object({
   type: external_exports.literal("paratext"),
@@ -12363,6 +12366,10 @@ var CodexParser = class {
   //  */
   // preserveMarkdown: boolean = true;
   _noteCounter = 0;
+  _parser;
+  constructor(parser) {
+    this._parser = parser;
+  }
   /**
    * Parses the specified codex content.
    *
@@ -12378,7 +12385,7 @@ var CodexParser = class {
     };
     let chapters = /* @__PURE__ */ new Map();
     let lastChapter = null;
-    function addChapterContent(chapterNumber, content) {
+    function ensureChapter(chapterNumber) {
       let chapter = chapters.get(chapterNumber);
       if (!chapter) {
         chapter = {
@@ -12390,8 +12397,23 @@ var CodexParser = class {
         lastChapter = chapter;
         chapters.set(chapterNumber, chapter);
       }
+      return chapter;
+    }
+    function getDocumentContent(doc) {
+      let content = "";
+      for (let element of doc.childNodes) {
+        content += element.textContent;
+      }
+      return content;
+    }
+    function addChapterContent(chapterNumber, content) {
+      const chapter = ensureChapter(chapterNumber);
       chapter.content.push(...content);
     }
+    function addChapterFootnote(chapterNumber, footnote) {
+      const chapter = ensureChapter(chapterNumber);
+      chapter.footnotes.push(footnote);
+    }
     function addReference(ref2, content) {
       addChapterContent(ref2.chapter, [
         {
@@ -12413,8 +12435,12 @@ var CodexParser = class {
     function addLineBreaks(lines2) {
       return lines2.reduce(
         (prev, current) => {
-          if (prev.length === 0) return [current];
-          else
+          if (typeof current === "string" && current.trim() === "") {
+            return prev;
+          }
+          if (prev.length === 0) {
+            return [current];
+          } else {
             return [
               ...prev,
               {
@@ -12422,6 +12448,7 @@ var CodexParser = class {
               },
               current
             ];
+          }
         },
         []
       );
@@ -12466,6 +12493,15 @@ var CodexParser = class {
                   "Could not find verse reference in metadata."
                 );
               }
+            } else if (metadata.data?.globalReferences?.length) {
+              reference = parseVerseReference(
+                metadata.data.globalReferences[0]
+              );
+              if (!reference) {
+                throw new Error(
+                  "Could not find verse reference in globalReferences metadata."
+                );
+              }
             } else {
               throw new Error(
                 "Could not find verse reference in metadata."
@@ -12478,13 +12514,49 @@ var CodexParser = class {
           }
           previousReference = reference;
           if (!root.id) root.id = reference.book;
-          const content = stripHTML(cell.value);
-          const newLines = content.split("\n");
-          lines.push(...newLines);
+          const doc = this._parser.parseFromString(
+            cell.value,
+            "text/html"
+          );
+          for (let element of doc.children) {
+            const allFootnotes = element.querySelectorAll(
+              "sup.footnote-marker[data-footnote]"
+            );
+            for (const footnote of allFootnotes) {
+              const footnoteData = footnote.getAttribute("data-footnote");
+              if (footnoteData) {
+                const footnoteDoc = this._parser.parseFromString(
+                  footnoteData,
+                  "text/html"
+                );
+                const footnoteText = getDocumentContent(footnoteDoc);
+                if (footnoteText) {
+                  addChapterFootnote(reference.chapter, {
+                    noteId: this._noteCounter++,
+                    text: footnoteText,
+                    caller: null
+                  });
+                }
+              }
+              footnote.remove();
+            }
+          }
+          const content = getDocumentContent(doc);
+          if (content) {
+            const newLines = content.split("\n");
+            lines.push(...newLines);
+          }
         } else if (metadata.type === "paratext") {
           const reference = parseVerseReference(`${metadata.id}:1`);
           if (reference) {
-            const content = stripHTML(cell.value);
+            const doc = this._parser.parseFromString(
+              cell.value,
+              "text/html"
+            );
+            const content = getDocumentContent(doc);
+            if (!content) {
+              continue;
+            }
             const lines2 = content.split("\n").reduce((prev, current) => {
               if (prev.length === 0)
                 return [toHeading(current)];
@@ -12501,8 +12573,12 @@ var CodexParser = class {
               (line) => addChapterContent(reference.chapter, [line])
             );
           } else {
-            const content = stripHTML(cell.value);
-            if (!cell.metadata) {
+            const doc = this._parser.parseFromString(
+              cell.value,
+              "text/html"
+            );
+            const content = getDocumentContent(doc);
+            if (content && !cell.metadata) {
               if (lastChapter) {
                 lastChapter.footnotes.push({
                   noteId: this._noteCounter++,
@@ -12816,7 +12892,7 @@ function generateDataset(files, parser = new globalThis.DOMParser(), bookMap) {
   };
   let usfmParser = new UsfmParser();
   let usxParser = new USXParser(parser);
-  let codexParser = new CodexParser();
+  let codexParser = new CodexParser(parser);
   let csvCommentaryParser = new CommentaryCsvParser();
   let tyndaleXmlParser = new TyndaleXmlParser(parser);
   let parsedTranslations = /* @__PURE__ */ new Map();

package/dist/cjs/index.cjs

@@ -11781,7 +11781,10 @@ var verseSchema = external_exports.object({
   id: external_exports.string(),
   bookCode: external_exports.string().optional(),
   chapter: external_exports.number().optional(),
-  verse: external_exports.number().optional()
+  verse: external_exports.number().optional(),
+  data: external_exports.object({
+    globalReferences: external_exports.array(external_exports.string()).optional()
+  }).optional()
 });
 var paratextSchema = external_exports.object({
   type: external_exports.literal("paratext"),
@@ -11811,6 +11814,10 @@ var CodexParser = class {
   //  */
   // preserveMarkdown: boolean = true;
   _noteCounter = 0;
+  _parser;
+  constructor(parser) {
+    this._parser = parser;
+  }
   /**
    * Parses the specified codex content.
    *
@@ -11826,7 +11833,7 @@ var CodexParser = class {
     };
     let chapters = /* @__PURE__ */ new Map();
     let lastChapter = null;
-    function addChapterContent(chapterNumber, content) {
+    function ensureChapter(chapterNumber) {
       let chapter = chapters.get(chapterNumber);
       if (!chapter) {
         chapter = {
@@ -11838,8 +11845,23 @@ var CodexParser = class {
         lastChapter = chapter;
         chapters.set(chapterNumber, chapter);
       }
+      return chapter;
+    }
+    function getDocumentContent(doc) {
+      let content = "";
+      for (let element of doc.childNodes) {
+        content += element.textContent;
+      }
+      return content;
+    }
+    function addChapterContent(chapterNumber, content) {
+      const chapter = ensureChapter(chapterNumber);
       chapter.content.push(...content);
     }
+    function addChapterFootnote(chapterNumber, footnote) {
+      const chapter = ensureChapter(chapterNumber);
+      chapter.footnotes.push(footnote);
+    }
     function addReference(ref2, content) {
       addChapterContent(ref2.chapter, [
         {
@@ -11861,8 +11883,12 @@ var CodexParser = class {
     function addLineBreaks(lines2) {
       return lines2.reduce(
         (prev, current) => {
-          if (prev.length === 0) return [current];
-          else
+          if (typeof current === "string" && current.trim() === "") {
+            return prev;
+          }
+          if (prev.length === 0) {
+            return [current];
+          } else {
             return [
               ...prev,
               {
@@ -11870,6 +11896,7 @@ var CodexParser = class {
               },
               current
             ];
+          }
         },
         []
       );
@@ -11914,6 +11941,15 @@ var CodexParser = class {
                   "Could not find verse reference in metadata."
                 );
               }
+            } else if (metadata.data?.globalReferences?.length) {
+              reference = parseVerseReference(
+                metadata.data.globalReferences[0]
+              );
+              if (!reference) {
+                throw new Error(
+                  "Could not find verse reference in globalReferences metadata."
+                );
+              }
             } else {
               throw new Error(
                 "Could not find verse reference in metadata."
@@ -11926,13 +11962,49 @@ var CodexParser = class {
           }
           previousReference = reference;
           if (!root.id) root.id = reference.book;
-          const content = stripHTML(cell.value);
-          const newLines = content.split("\n");
-          lines.push(...newLines);
+          const doc = this._parser.parseFromString(
+            cell.value,
+            "text/html"
+          );
+          for (let element of doc.children) {
+            const allFootnotes = element.querySelectorAll(
+              "sup.footnote-marker[data-footnote]"
+            );
+            for (const footnote of allFootnotes) {
+              const footnoteData = footnote.getAttribute("data-footnote");
+              if (footnoteData) {
+                const footnoteDoc = this._parser.parseFromString(
+                  footnoteData,
+                  "text/html"
+                );
+                const footnoteText = getDocumentContent(footnoteDoc);
+                if (footnoteText) {
+                  addChapterFootnote(reference.chapter, {
+                    noteId: this._noteCounter++,
+                    text: footnoteText,
+                    caller: null
+                  });
+                }
+              }
+              footnote.remove();
+            }
+          }
+          const content = getDocumentContent(doc);
+          if (content) {
+            const newLines = content.split("\n");
+            lines.push(...newLines);
+          }
         } else if (metadata.type === "paratext") {
           const reference = parseVerseReference(`${metadata.id}:1`);
           if (reference) {
-            const content = stripHTML(cell.value);
+            const doc = this._parser.parseFromString(
+              cell.value,
+              "text/html"
+            );
+            const content = getDocumentContent(doc);
+            if (!content) {
+              continue;
+            }
             const lines2 = content.split("\n").reduce((prev, current) => {
               if (prev.length === 0)
                 return [toHeading(current)];
@@ -11949,8 +12021,12 @@ var CodexParser = class {
               (line) => addChapterContent(reference.chapter, [line])
             );
           } else {
-            const content = stripHTML(cell.value);
-            if (!cell.metadata) {
+            const doc = this._parser.parseFromString(
+              cell.value,
+              "text/html"
+            );
+            const content = getDocumentContent(doc);
+            if (content && !cell.metadata) {
               if (lastChapter) {
                 lastChapter.footnotes.push({
                   noteId: this._noteCounter++,
@@ -12264,7 +12340,7 @@ function generateDataset(files, parser = new globalThis.DOMParser(), bookMap) {
   };
   let usfmParser = new UsfmParser();
   let usxParser = new USXParser(parser);
-  let codexParser = new CodexParser();
+  let codexParser = new CodexParser(parser);
   let csvCommentaryParser = new CommentaryCsvParser();
   let tyndaleXmlParser = new TyndaleXmlParser(parser);
   let parsedTranslations = /* @__PURE__ */ new Map();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helloao/cli",
-  "version": "0.0.19",
+  "version": "0.1.0",
   "description": "A CLI and related tools for managing HelloAO's Free Bible API",
   "main": "./dist/cjs/index.cjs",
   "module": "./dist/esm/index.js",
@@ -43,7 +43,7 @@
     "all-iso-language-codes": "1.0.17",
     "papaparse": "5.4.1",
     "luxon": "3.5.0",
-    "@helloao/tools": "0.0.19"
+    "@helloao/tools": "0.1.0"
   },
   "files": [
     "/README.md",