@helloao/tools 0.0.9 → 0.0.11

package/dist/cjs/generation/dataset.cjs

@@ -27,14 +27,22 @@ var import_book_order = require("./book-order.js");
 var import_lodash = require("lodash");
 var import_audio = require("./audio.js");
 var import_codex_parser = require("../parser/codex-parser.js");
-function generateDataset(files, parser = new globalThis.DOMParser()) {
+var import_commentary_csv_parser = require("../parser/commentary-csv-parser.js");
+var import_tyndale_xml_parser = require("../parser/tyndale-xml-parser.js");
+var import_log = require("../log.js");
+function generateDataset(files, parser = new globalThis.DOMParser(), bookMap) {
+  const logger = (0, import_log.getLogger)();
   let output = {
-    translations: []
+    translations: [],
+    commentaries: []
   };
   let usfmParser = new import_usfm_parser.UsfmParser();
   let usxParser = new import_usx_parser.USXParser(parser);
   let codexParser = new import_codex_parser.CodexParser();
+  let csvCommentaryParser = new import_commentary_csv_parser.CommentaryCsvParser();
+  let tyndaleXmlParser = new import_tyndale_xml_parser.TyndaleXmlParser(parser);
   let parsedTranslations = /* @__PURE__ */ new Map();
+  let parsedCommentaries = /* @__PURE__ */ new Map();
   const unknownLanguages = /* @__PURE__ */ new Set();
   for (let file of files) {
     try {
@@ -45,109 +53,198 @@ function generateDataset(files, parser = new globalThis.DOMParser()) {
         parser2 = usxParser;
       } else if (file.fileType === "json") {
         parser2 = codexParser;
+      } else if (file.fileType === "commentary/csv") {
+        parser2 = csvCommentaryParser;
+      } else if (file.fileType === "commentary/tyndale-xml") {
+        parser2 = tyndaleXmlParser;
       } else {
-        console.warn(
+        logger.warn(
           "[generate] File does not have a valid type!",
           file.name
         );
         continue;
       }
       const parsed = parser2.parse(file.content);
-      const id = parsed.id;
-      if (!id) {
-        console.warn(
-          "[generate] File does not have a valid book ID!",
+      if (parsed.type === "root") {
+        addTranslationTree(file, parsed);
+      } else {
+        addCommentaryTree(file, parsed);
+      }
+    } catch (err) {
+      logger.error(
+        `[generate] Error occurred while parsing ${file.name}`,
+        err
+      );
+    }
+  }
+  function addTranslationTree(file, parsed) {
+    const id = parsed.id;
+    if (!id) {
+      logger.warn(
+        "[generate] File does not have a valid book ID!",
+        file.name,
+        id
+      );
+      return;
+    }
+    const order = import_book_order.bookOrderMap.get(id);
+    if (typeof order !== "number") {
+      logger.warn("[generate] Book does not have an order!", id);
+      return;
+    }
+    const bookName = getBookNames(file, file.metadata, id);
+    let translation = parsedTranslations.get(file.metadata.id);
+    if (!translation) {
+      translation = {
+        ...(0, import_lodash.omit)(file.metadata, "direction"),
+        textDirection: file.metadata.direction,
+        books: []
+      };
+      output.translations.push(translation);
+      parsedTranslations.set(file.metadata.id, translation);
+    }
+    const name = parsed.header ?? bookName?.bookName?.commonName ?? parsed.title;
+    if (!name) {
+      logger.warn("[generate] Book does not have a name!", file.name, id);
+      return;
+    }
+    const commonName = bookName?.bookName?.commonName ?? parsed.header ?? parsed.title ?? id;
+    const book = {
+      id,
+      name,
+      commonName,
+      title: parsed.title ?? null,
+      order,
+      chapters: []
+    };
+    for (let content of parsed.content) {
+      if (content.type === "chapter") {
+        book.chapters.push({
+          chapter: {
+            number: content.number,
+            content: content.content,
+            footnotes: content.footnotes
+          },
+          thisChapterAudioLinks: (0, import_audio.getAudioUrlsForChapter)(
+            translation.id,
+            id,
+            content.number
+          )
+        });
+      }
+    }
+    book.chapters = (0, import_lodash.sortBy)(book.chapters, (c) => c.chapter.number);
+    const index = (0, import_lodash.sortedIndexBy)(translation.books, book, (b) => b.order);
+    translation.books.splice(index, 0, book);
+  }
+  function addCommentaryTree(file, parsed) {
+    let commentary = parsedCommentaries.get(file.metadata.id);
+    if (!commentary) {
+      commentary = {
+        ...(0, import_lodash.omit)(file.metadata, "direction"),
+        textDirection: file.metadata.direction,
+        books: [],
+        profiles: []
+      };
+      output.commentaries.push(commentary);
+      parsedCommentaries.set(file.metadata.id, commentary);
+    }
+    for (let parsedBook of parsed.books) {
+      let book = commentary.books.find((b) => b.id === parsedBook.book);
+      if (book && book.introduction && parsedBook.introduction) {
+        logger.warn(
+          "[generate] Book already exists in commentary!",
           file.name,
-          id
+          parsedBook.book
         );
         continue;
       }
-      const order = import_book_order.bookOrderMap.get(id);
-      if (typeof order !== "number") {
-        console.warn("[generate] Book does not have an order!", id);
-        continue;
+      if (!book) {
+        const name = getBookNames(file, file.metadata, parsedBook.book);
+        book = {
+          id: parsedBook.book,
+          order: import_book_order.bookOrderMap.get(parsedBook.book) ?? -1,
+          commonName: name?.bookName?.commonName ?? parsedBook.book,
+          name: name?.bookName?.commonName ?? parsedBook.book,
+          chapters: []
+        };
       }
-      const bookMap = import_book_order.bookIdMap.get(file.metadata.translation.language);
-      if (!bookMap) {
-        if (!unknownLanguages.has(file.metadata.translation.language)) {
-          console.warn(
-            "[generate] File does not have a known language!",
-            file.name,
-            file.metadata.translation.language
-          );
-          unknownLanguages.add(file.metadata.translation.language);
-        }
+      if (parsedBook.introduction) {
+        book.introduction = parsedBook.introduction;
       }
-      const bookName = bookMap?.get(id);
-      if (!!bookMap && !bookName) {
-        console.warn(
-          "[generate] Book name not found for ID!",
-          file.name,
-          id
-        );
+      if (parsedBook.introductionSummary && !book.introductionSummary) {
+        book.introductionSummary = parsedBook.introductionSummary;
       }
-      let translation = parsedTranslations.get(
-        file.metadata.translation.id
-      );
-      if (!translation) {
-        translation = {
-          ...(0, import_lodash.omit)(file.metadata.translation, "direction"),
-          textDirection: file.metadata.translation.direction,
-          books: []
+      for (let chapter of parsedBook.chapters) {
+        let data = {
+          number: chapter.number,
+          content: chapter.verses
         };
-        output.translations.push(translation);
-        parsedTranslations.set(
-          file.metadata.translation.id,
-          translation
-        );
+        if (chapter.introduction) {
+          data.introduction = chapter.introduction;
+        }
+        book.chapters.push({
+          chapter: data
+        });
       }
-      const name = parsed.header ?? bookName?.commonName ?? parsed.title;
-      if (!name) {
-        console.warn(
-          "[generate] Book does not have a name!",
-          file.name,
-          id
+      book.chapters = (0, import_lodash.sortBy)(book.chapters, (c) => c.chapter.number);
+      commentary.books.push(book);
+    }
+    if (parsed.profiles) {
+      for (let profile of parsed.profiles) {
+        const existing = commentary.profiles.find(
+          (p) => p.id === profile.id
         );
-        continue;
+        if (existing) {
+          logger.warn(
+            "[generate] Profile already exists in commentary!",
+            file.name,
+            profile.id
+          );
+          continue;
+        }
+        const datasetProfile = {
+          id: profile.id,
+          subject: profile.subject,
+          reference: profile.reference,
+          content: profile.content
+        };
+        commentary.profiles.push(datasetProfile);
       }
-      const commonName = bookName?.commonName ?? parsed.header ?? parsed.title ?? id;
-      const book = {
-        id,
-        name,
-        commonName,
-        title: parsed.title ?? null,
-        order,
-        chapters: []
-      };
-      for (let content of parsed.content) {
-        if (content.type === "chapter") {
-          book.chapters.push({
-            chapter: {
-              number: content.number,
-              content: content.content,
-              footnotes: content.footnotes
-            },
-            thisChapterAudioLinks: (0, import_audio.getAudioUrlsForChapter)(
-              translation.id,
-              id,
-              content.number
-            )
-          });
+    }
+    commentary.books = (0, import_lodash.sortBy)(commentary.books, (b) => b.order);
+    commentary.profiles = (0, import_lodash.sortBy)(commentary.profiles, (p) => p.id);
+  }
+  function getBookNames(file, metadata, id) {
+    if (!bookMap) {
+      bookMap = import_book_order.bookIdMap.get(metadata.language);
+      if (!bookMap) {
+        if (!unknownLanguages.has(metadata.language)) {
+          logger.warn(
+            "[generate] File does not have a known language!",
+            file.name,
+            metadata.language
+          );
+          unknownLanguages.add(metadata.language);
         }
+        logger.warn(
+          "[generate] Using English book map for unknown language! This might result in outputting english book names."
+        );
+        bookMap = import_book_order.bookIdMap.get("en");
       }
-      book.chapters = (0, import_lodash.sortBy)(book.chapters, (c) => c.chapter.number);
-      const index = (0, import_lodash.sortedIndexBy)(
-        translation.books,
-        book,
-        (b) => b.order
-      );
-      translation.books.splice(index, 0, book);
-    } catch (err) {
-      console.error(
-        `[generate] Error occurred while parsing ${file.name}`,
-        err
+    }
+    const bookName = bookMap?.get(id);
+    if (!!bookMap && !bookName) {
+      logger.warn(
+        "[generate] Book name not found for ID!",
+        file.name,
+        id
       );
+      return null;
     }
+    return {
+      bookName
+    };
   }
   return output;
 }

package/dist/cjs/generation/dataset.cjs.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../generation/dataset.ts"],
-  "sourcesContent": ["import { UsfmParser } from '../parser/usfm-parser.js';\r\nimport { USXParser } from '../parser/usx-parser.js';\r\nimport {\r\n    InputFile,\r\n    Translation,\r\n    TranslationBook,\r\n    TranslationBookChapter,\r\n} from './common-types.js';\r\nimport { bookIdMap, bookOrderMap } from './book-order.js';\r\nimport { omit, sortBy, sortedIndexBy } from 'lodash';\r\nimport { getAudioUrlsForChapter } from './audio.js';\r\nimport { CodexParser } from '../parser/codex-parser.js';\r\n\r\n/**\r\n * Defines an interface that contains generated dataset info.\r\n */\r\nexport interface DatasetOutput {\r\n    /**\r\n     * The list of translations that are available in the dataset.\r\n     */\r\n    translations: DatasetTranslation[];\r\n}\r\n\r\n/**\r\n * Defines a translation that is used in the dataset.\r\n */\r\nexport interface DatasetTranslation extends Translation {\r\n    /**\r\n     * The list of books that are available for the translation.\r\n     */\r\n    books: DatasetTranslationBook[];\r\n}\r\n\r\n/**\r\n * Defines a translation book that is used in the dataset.\r\n */\r\nexport interface DatasetTranslationBook extends TranslationBook {\r\n    /**\r\n     * The list of chapters that are available for the book.\r\n     */\r\n    chapters: TranslationBookChapter[];\r\n}\r\n\r\n/**\r\n * Generates a list of output files from the given list of input files.\r\n * @param file The list of files.\r\n */\r\nexport function generateDataset(\r\n    files: InputFile[],\r\n    parser: DOMParser = new globalThis.DOMParser()\r\n): DatasetOutput {\r\n    let output: DatasetOutput = {\r\n        translations: [],\r\n    };\r\n\r\n    let usfmParser = new UsfmParser();\r\n    let usxParser = new USXParser(parser);\r\n    let codexParser = new CodexParser();\r\n\r\n    let parsedTranslations = new Map<string, DatasetTranslation>();\r\n\r\n    const unknownLanguages = new Set<string>();\r\n    for (let file of files) {\r\n        try {\r\n            let parser: UsfmParser | USXParser | CodexParser;\r\n            if (file.fileType === 'usfm') {\r\n                parser = usfmParser;\r\n            } else if (file.fileType === 'usx') {\r\n                parser = usxParser;\r\n            } else if (file.fileType === 'json') {\r\n                parser = codexParser;\r\n            } else {\r\n                console.warn(\r\n                    '[generate] File does not have a valid type!',\r\n                    file.name\r\n                );\r\n                continue;\r\n            }\r\n\r\n            const parsed = parser.parse(file.content);\r\n            const id = parsed.id;\r\n\r\n            if (!id) {\r\n                console.warn(\r\n                    '[generate] File does not have a valid book ID!',\r\n                    file.name,\r\n                    id\r\n                );\r\n                continue;\r\n            }\r\n\r\n            const order = bookOrderMap.get(id);\r\n\r\n            if (typeof order !== 'number') {\r\n                console.warn('[generate] Book does not have an order!', id);\r\n                continue;\r\n            }\r\n\r\n            const bookMap = bookIdMap.get(file.metadata.translation.language);\r\n\r\n            if (!bookMap) {\r\n                if (!unknownLanguages.has(file.metadata.translation.language)) {\r\n                    console.warn(\r\n                        '[generate] File does not have a known language!',\r\n                        file.name,\r\n                        file.metadata.translation.language\r\n                    );\r\n                    unknownLanguages.add(file.metadata.translation.language);\r\n                }\r\n            }\r\n\r\n            const bookName = bookMap?.get(id);\r\n\r\n            if (!!bookMap && !bookName) {\r\n                console.warn(\r\n                    '[generate] Book name not found for ID!',\r\n                    file.name,\r\n                    id\r\n                );\r\n            }\r\n\r\n            let translation = parsedTranslations.get(\r\n                file.metadata.translation.id\r\n            );\r\n\r\n            if (!translation) {\r\n                translation = {\r\n                    ...omit(file.metadata.translation, 'direction'),\r\n                    textDirection: file.metadata.translation.direction,\r\n                    books: [],\r\n                };\r\n                output.translations.push(translation);\r\n                parsedTranslations.set(\r\n                    file.metadata.translation.id,\r\n                    translation\r\n                );\r\n            }\r\n\r\n            const name = parsed.header ?? bookName?.commonName ?? parsed.title;\r\n\r\n            if (!name) {\r\n                console.warn(\r\n                    '[generate] Book does not have a name!',\r\n                    file.name,\r\n                    id\r\n                );\r\n                continue;\r\n            }\r\n\r\n            const commonName =\r\n                bookName?.commonName ?? parsed.header ?? parsed.title ?? id;\r\n\r\n            const book: DatasetTranslationBook = {\r\n                id: id,\r\n                name,\r\n                commonName,\r\n                title: parsed.title ?? null,\r\n                order,\r\n                chapters: [],\r\n            };\r\n\r\n            for (let content of parsed.content) {\r\n                if (content.type === 'chapter') {\r\n                    book.chapters.push({\r\n                        chapter: {\r\n                            number: content.number,\r\n                            content: content.content,\r\n                            footnotes: content.footnotes,\r\n                        },\r\n                        thisChapterAudioLinks: getAudioUrlsForChapter(\r\n                            translation.id,\r\n                            id,\r\n                            content.number\r\n                        ),\r\n                    });\r\n                }\r\n            }\r\n\r\n            book.chapters = sortBy(book.chapters, (c) => c.chapter.number);\r\n\r\n            const index = sortedIndexBy(\r\n                translation.books,\r\n                book,\r\n                (b) => b.order\r\n            );\r\n            translation.books.splice(index, 0, book);\r\n        } catch (err) {\r\n            console.error(\r\n                `[generate] Error occurred while parsing ${file.name}`,\r\n                err\r\n            );\r\n        }\r\n    }\r\n\r\n    return output;\r\n}\r\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,yBAA2B;AAC3B,wBAA0B;AAO1B,wBAAwC;AACxC,oBAA4C;AAC5C,mBAAuC;AACvC,0BAA4B;AAoCrB,SAAS,gBACZ,OACA,SAAoB,IAAI,WAAW,UAAU,GAChC;AACb,MAAI,SAAwB;AAAA,IACxB,cAAc,CAAC;AAAA,EACnB;AAEA,MAAI,aAAa,IAAI,8BAAW;AAChC,MAAI,YAAY,IAAI,4BAAU,MAAM;AACpC,MAAI,cAAc,IAAI,gCAAY;AAElC,MAAI,qBAAqB,oBAAI,IAAgC;AAE7D,QAAM,mBAAmB,oBAAI,IAAY;AACzC,WAAS,QAAQ,OAAO;AACpB,QAAI;AACA,UAAIA;AACJ,UAAI,KAAK,aAAa,QAAQ;AAC1B,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,OAAO;AAChC,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,QAAQ;AACjC,QAAAA,UAAS;AAAA,MACb,OAAO;AACH,gBAAQ;AAAA,UACJ;AAAA,UACA,KAAK;AAAA,QACT;AACA;AAAA,MACJ;AAEA,YAAM,SAASA,QAAO,MAAM,KAAK,OAAO;AACxC,YAAM,KAAK,OAAO;AAElB,UAAI,CAAC,IAAI;AACL,gBAAQ;AAAA,UACJ;AAAA,UACA,KAAK;AAAA,UACL;AAAA,QACJ;AACA;AAAA,MACJ;AAEA,YAAM,QAAQ,+BAAa,IAAI,EAAE;AAEjC,UAAI,OAAO,UAAU,UAAU;AAC3B,gBAAQ,KAAK,2CAA2C,EAAE;AAC1D;AAAA,MACJ;AAEA,YAAM,UAAU,4BAAU,IAAI,KAAK,SAAS,YAAY,QAAQ;AAEhE,UAAI,CAAC,SAAS;AACV,YAAI,CAAC,iBAAiB,IAAI,KAAK,SAAS,YAAY,QAAQ,GAAG;AAC3D,kBAAQ;AAAA,YACJ;AAAA,YACA,KAAK;AAAA,YACL,KAAK,SAAS,YAAY;AAAA,UAC9B;AACA,2BAAiB,IAAI,KAAK,SAAS,YAAY,QAAQ;AAAA,QAC3D;AAAA,MACJ;AAEA,YAAM,WAAW,SAAS,IAAI,EAAE;AAEhC,UAAI,CAAC,CAAC,WAAW,CAAC,UAAU;AACxB,gBAAQ;AAAA,UACJ;AAAA,UACA,KAAK;AAAA,UACL;AAAA,QACJ;AAAA,MACJ;AAEA,UAAI,cAAc,mBAAmB;AAAA,QACjC,KAAK,SAAS,YAAY;AAAA,MAC9B;AAEA,UAAI,CAAC,aAAa;AACd,sBAAc;AAAA,UACV,OAAG,oBAAK,KAAK,SAAS,aAAa,WAAW;AAAA,UAC9C,eAAe,KAAK,SAAS,YAAY;AAAA,UACzC,OAAO,CAAC;AAAA,QACZ;AACA,eAAO,aAAa,KAAK,WAAW;AACpC,2BAAmB;AAAA,UACf,KAAK,SAAS,YAAY;AAAA,UAC1B;AAAA,QACJ;AAAA,MACJ;AAEA,YAAM,OAAO,OAAO,UAAU,UAAU,cAAc,OAAO;AAE7D,UAAI,CAAC,MAAM;AACP,gBAAQ;AAAA,UACJ;AAAA,UACA,KAAK;AAAA,UACL;AAAA,QACJ;AACA;AAAA,MACJ;AAEA,YAAM,aACF,UAAU,cAAc,OAAO,UAAU,OAAO,SAAS;AAE7D,YAAM,OAA+B;AAAA,QACjC;AAAA,QACA;AAAA,QACA;AAAA,QACA,OAAO,OAAO,SAAS;AAAA,QACvB;AAAA,QACA,UAAU,CAAC;AAAA,MACf;AAEA,eAAS,WAAW,OAAO,SAAS;AAChC,YAAI,QAAQ,SAAS,WAAW;AAC5B,eAAK,SAAS,KAAK;AAAA,YACf,SAAS;AAAA,cACL,QAAQ,QAAQ;AAAA,cAChB,SAAS,QAAQ;AAAA,cACjB,WAAW,QAAQ;AAAA,YACvB;AAAA,YACA,2BAAuB;AAAA,cACnB,YAAY;AAAA,cACZ;AAAA,cACA,QAAQ;AAAA,YACZ;AAAA,UACJ,CAAC;AAAA,QACL;AAAA,MACJ;AAEA,WAAK,eAAW,sBAAO,KAAK,UAAU,CAAC,MAAM,EAAE,QAAQ,MAAM;AAE7D,YAAM,YAAQ;AAAA,QACV,YAAY;AAAA,QACZ;AAAA,QACA,CAAC,MAAM,EAAE;AAAA,MACb;AACA,kBAAY,MAAM,OAAO,OAAO,GAAG,IAAI;AAAA,IAC3C,SAAS,KAAK;AACV,cAAQ;AAAA,QACJ,2CAA2C,KAAK,IAAI;AAAA,QACpD;AAAA,MACJ;AAAA,IACJ;AAAA,EACJ;AAEA,SAAO;AACX;",
-  "names": ["parser"]
+  "sourcesContent": ["import { UsfmParser } from '../parser/usfm-parser.js';\nimport { USXParser } from '../parser/usx-parser.js';\nimport {\n    Commentary,\n    CommentaryBook,\n    CommentaryBookChapter,\n    CommentaryChapterData,\n    CommentaryProfile,\n    InputCommentaryFile,\n    InputFile,\n    InputFileBase,\n    InputTranslationFile,\n    MetadataBase,\n    Translation,\n    TranslationBook,\n    TranslationBookChapter,\n} from './common-types.js';\nimport { bookIdMap as defaultBookIdMap, bookOrderMap } from './book-order.js';\nimport { omit, sortBy, sortedIndexBy } from 'lodash';\nimport { getAudioUrlsForChapter } from './audio.js';\nimport { CodexParser } from '../parser/codex-parser.js';\nimport { CommentaryCsvParser } from '../parser/commentary-csv-parser.js';\nimport { CommentaryParseTree, ParseTree } from '../parser/types.js';\nimport { TyndaleXmlParser } from '../parser/tyndale-xml-parser.js';\nimport { getLogger } from '../log.js';\n\n/**\n * Defines an interface that contains generated dataset info.\n */\nexport interface DatasetOutput {\n    /**\n     * The list of translations that are available in the dataset.\n     */\n    translations: DatasetTranslation[];\n\n    /**\n     * The list of commentaries that are available in the dataset.\n     */\n    commentaries: DatasetCommentary[];\n}\n\n/**\n * Defines a translation that is used in the dataset.\n */\nexport interface DatasetTranslation extends Translation {\n    /**\n     * The list of books that are available for the translation.\n     */\n    books: DatasetTranslationBook[];\n}\n\n/**\n * Defines a commentary that is used in the dataset.\n */\nexport interface DatasetCommentary extends Commentary {\n    /**\n     * The list of books that are available for the commentary.\n     */\n    books: DatasetCommentaryBook[];\n\n    /**\n     * The list of profiles that  are available for the commentary.\n     */\n    profiles: DatasetCommentaryProfile[];\n}\n\n/**\n * Defines a translation book that is used in the dataset.\n */\nexport interface DatasetTranslationBook extends TranslationBook {\n    /**\n     * The list of chapters that are available for the book.\n     */\n    chapters: TranslationBookChapter[];\n}\n\n/**\n * Defines a commentary book that is used in the dataset.\n */\nexport interface DatasetCommentaryBook extends CommentaryBook {\n    /**\n     * The list of chapters that are available for the book.\n     */\n    chapters: CommentaryBookChapter[];\n}\n\nexport interface DatasetCommentaryProfile extends CommentaryProfile {\n    /**\n     * The contents of the profile.\n     */\n    content: string[];\n}\n\n/**\n * Generates a list of output files from the given list of input files.\n * @param file The list of files.\n * @param parser The parser to use for parsing the files.\n * @param bookMap The map of book IDs to names. If undefined, then a default map will be used.\n */\nexport function generateDataset(\n    files: InputFile[],\n    parser: DOMParser = new globalThis.DOMParser(),\n    bookMap?: Map<string, { commonName: string }> | undefined\n): DatasetOutput {\n    const logger = getLogger();\n    let output: DatasetOutput = {\n        translations: [],\n        commentaries: [],\n    };\n\n    let usfmParser = new UsfmParser();\n    let usxParser = new USXParser(parser);\n    let codexParser = new CodexParser();\n    let csvCommentaryParser = new CommentaryCsvParser();\n    let tyndaleXmlParser = new TyndaleXmlParser(parser);\n\n    let parsedTranslations = new Map<string, DatasetTranslation>();\n    let parsedCommentaries = new Map<string, DatasetCommentary>();\n\n    const unknownLanguages = new Set<string>();\n    for (let file of files) {\n        try {\n            let parser:\n                | UsfmParser\n                | USXParser\n                | CodexParser\n                | CommentaryCsvParser\n                | TyndaleXmlParser;\n            if (file.fileType === 'usfm') {\n                parser = usfmParser;\n            } else if (file.fileType === 'usx') {\n                parser = usxParser;\n            } else if (file.fileType === 'json') {\n                parser = codexParser;\n            } else if (file.fileType === 'commentary/csv') {\n                parser = csvCommentaryParser;\n            } else if (file.fileType === 'commentary/tyndale-xml') {\n                parser = tyndaleXmlParser;\n            } else {\n                logger.warn(\n                    '[generate] File does not have a valid type!',\n                    file.name\n                );\n                continue;\n            }\n\n            const parsed = parser.parse(file.content);\n\n            if (parsed.type === 'root') {\n                addTranslationTree(file as InputTranslationFile, parsed);\n            } else {\n                addCommentaryTree(file as InputCommentaryFile, parsed);\n            }\n        } catch (err) {\n            logger.error(\n                `[generate] Error occurred while parsing ${file.name}`,\n                err\n            );\n        }\n    }\n\n    function addTranslationTree(file: InputTranslationFile, parsed: ParseTree) {\n        const id = parsed.id;\n\n        if (!id) {\n            logger.warn(\n                '[generate] File does not have a valid book ID!',\n                file.name,\n                id\n            );\n            return;\n        }\n\n        const order = bookOrderMap.get(id);\n\n        if (typeof order !== 'number') {\n            logger.warn('[generate] Book does not have an order!', id);\n            return;\n        }\n\n        const bookName = getBookNames(file, file.metadata, id);\n\n        let translation = parsedTranslations.get(file.metadata.id);\n\n        if (!translation) {\n            translation = {\n                ...omit(file.metadata, 'direction'),\n                textDirection: file.metadata.direction,\n                books: [],\n            };\n            output.translations.push(translation);\n            parsedTranslations.set(file.metadata.id, translation);\n        }\n\n        const name =\n            parsed.header ?? bookName?.bookName?.commonName ?? parsed.title;\n\n        if (!name) {\n            logger.warn('[generate] Book does not have a name!', file.name, id);\n            return;\n        }\n\n        const commonName =\n            bookName?.bookName?.commonName ??\n            parsed.header ??\n            parsed.title ??\n            id;\n\n        const book: DatasetTranslationBook = {\n            id: id,\n            name,\n            commonName,\n            title: parsed.title ?? null,\n            order,\n            chapters: [],\n        };\n\n        for (let content of parsed.content) {\n            if (content.type === 'chapter') {\n                book.chapters.push({\n                    chapter: {\n                        number: content.number,\n                        content: content.content,\n                        footnotes: content.footnotes,\n                    },\n                    thisChapterAudioLinks: getAudioUrlsForChapter(\n                        translation.id,\n                        id,\n                        content.number\n                    ),\n                });\n            }\n        }\n\n        book.chapters = sortBy(book.chapters, (c) => c.chapter.number);\n\n        const index = sortedIndexBy(translation.books, book, (b) => b.order);\n        translation.books.splice(index, 0, book);\n    }\n\n    function addCommentaryTree(\n        file: InputCommentaryFile,\n        parsed: CommentaryParseTree\n    ) {\n        let commentary = parsedCommentaries.get(file.metadata.id);\n\n        if (!commentary) {\n            commentary = {\n                ...omit(file.metadata, 'direction'),\n                textDirection: file.metadata.direction,\n                books: [],\n                profiles: [],\n            };\n            output.commentaries.push(commentary);\n            parsedCommentaries.set(file.metadata.id, commentary);\n        }\n\n        for (let parsedBook of parsed.books) {\n            let book = commentary.books.find((b) => b.id === parsedBook.book);\n            if (book && book.introduction && parsedBook.introduction) {\n                logger.warn(\n                    '[generate] Book already exists in commentary!',\n                    file.name,\n                    parsedBook.book\n                );\n                continue;\n            }\n\n            if (!book) {\n                const name = getBookNames(file, file.metadata, parsedBook.book);\n                book = {\n                    id: parsedBook.book,\n                    order: bookOrderMap.get(parsedBook.book) ?? -1,\n                    commonName: name?.bookName?.commonName ?? parsedBook.book,\n                    name: name?.bookName?.commonName ?? parsedBook.book,\n                    chapters: [],\n                };\n            }\n\n            if (parsedBook.introduction) {\n                book.introduction = parsedBook.introduction;\n            }\n\n            if (parsedBook.introductionSummary && !book.introductionSummary) {\n                book.introductionSummary = parsedBook.introductionSummary;\n            }\n\n            for (let chapter of parsedBook.chapters) {\n                let data: CommentaryChapterData = {\n                    number: chapter.number,\n                    content: chapter.verses,\n                };\n\n                if (chapter.introduction) {\n                    data.introduction = chapter.introduction;\n                }\n\n                book.chapters.push({\n                    chapter: data,\n                });\n            }\n\n            book.chapters = sortBy(book.chapters, (c) => c.chapter.number);\n            commentary.books.push(book);\n        }\n\n        if (parsed.profiles) {\n            for (let profile of parsed.profiles) {\n                const existing = commentary.profiles.find(\n                    (p) => p.id === profile.id\n                );\n                if (existing) {\n                    logger.warn(\n                        '[generate] Profile already exists in commentary!',\n                        file.name,\n                        profile.id\n                    );\n                    continue;\n                }\n\n                const datasetProfile: DatasetCommentaryProfile = {\n                    id: profile.id,\n                    subject: profile.subject,\n                    reference: profile.reference,\n                    content: profile.content,\n                };\n\n                commentary.profiles.push(datasetProfile);\n            }\n        }\n\n        commentary.books = sortBy(commentary.books, (b) => b.order);\n        commentary.profiles = sortBy(commentary.profiles, (p) => p.id);\n    }\n\n    function getBookNames(\n        file: InputFileBase,\n        metadata: MetadataBase,\n        id: string\n    ) {\n        if (!bookMap) {\n            bookMap = defaultBookIdMap.get(metadata.language);\n\n            if (!bookMap) {\n                if (!unknownLanguages.has(metadata.language)) {\n                    logger.warn(\n                        '[generate] File does not have a known language!',\n                        file.name,\n                        metadata.language\n                    );\n                    unknownLanguages.add(metadata.language);\n                }\n\n                logger.warn(\n                    '[generate] Using English book map for unknown language! This might result in outputting english book names.'\n                );\n                bookMap = defaultBookIdMap.get('en');\n            }\n        }\n\n        const bookName = bookMap?.get(id);\n\n        if (!!bookMap && !bookName) {\n            logger.warn(\n                '[generate] Book name not found for ID!',\n                file.name,\n                id\n            );\n            return null;\n        }\n\n        return {\n            bookName,\n        };\n    }\n\n    return output;\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,yBAA2B;AAC3B,wBAA0B;AAgB1B,wBAA4D;AAC5D,oBAA4C;AAC5C,mBAAuC;AACvC,0BAA4B;AAC5B,mCAAoC;AAEpC,gCAAiC;AACjC,iBAA0B;AA2EnB,SAAS,gBACZ,OACA,SAAoB,IAAI,WAAW,UAAU,GAC7C,SACa;AACb,QAAM,aAAS,sBAAU;AACzB,MAAI,SAAwB;AAAA,IACxB,cAAc,CAAC;AAAA,IACf,cAAc,CAAC;AAAA,EACnB;AAEA,MAAI,aAAa,IAAI,8BAAW;AAChC,MAAI,YAAY,IAAI,4BAAU,MAAM;AACpC,MAAI,cAAc,IAAI,gCAAY;AAClC,MAAI,sBAAsB,IAAI,iDAAoB;AAClD,MAAI,mBAAmB,IAAI,2CAAiB,MAAM;AAElD,MAAI,qBAAqB,oBAAI,IAAgC;AAC7D,MAAI,qBAAqB,oBAAI,IAA+B;AAE5D,QAAM,mBAAmB,oBAAI,IAAY;AACzC,WAAS,QAAQ,OAAO;AACpB,QAAI;AACA,UAAIA;AAMJ,UAAI,KAAK,aAAa,QAAQ;AAC1B,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,OAAO;AAChC,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,QAAQ;AACjC,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,kBAAkB;AAC3C,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,0BAA0B;AACnD,QAAAA,UAAS;AAAA,MACb,OAAO;AACH,eAAO;AAAA,UACH;AAAA,UACA,KAAK;AAAA,QACT;AACA;AAAA,MACJ;AAEA,YAAM,SAASA,QAAO,MAAM,KAAK,OAAO;AAExC,UAAI,OAAO,SAAS,QAAQ;AACxB,2BAAmB,MAA8B,MAAM;AAAA,MAC3D,OAAO;AACH,0BAAkB,MAA6B,MAAM;AAAA,MACzD;AAAA,IACJ,SAAS,KAAK;AACV,aAAO;AAAA,QACH,2CAA2C,KAAK,IAAI;AAAA,QACpD;AAAA,MACJ;AAAA,IACJ;AAAA,EACJ;AAEA,WAAS,mBAAmB,MAA4B,QAAmB;AACvE,UAAM,KAAK,OAAO;AAElB,QAAI,CAAC,IAAI;AACL,aAAO;AAAA,QACH;AAAA,QACA,KAAK;AAAA,QACL;AAAA,MACJ;AACA;AAAA,IACJ;AAEA,UAAM,QAAQ,+BAAa,IAAI,EAAE;AAEjC,QAAI,OAAO,UAAU,UAAU;AAC3B,aAAO,KAAK,2CAA2C,EAAE;AACzD;AAAA,IACJ;AAEA,UAAM,WAAW,aAAa,MAAM,KAAK,UAAU,EAAE;AAErD,QAAI,cAAc,mBAAmB,IAAI,KAAK,SAAS,EAAE;AAEzD,QAAI,CAAC,aAAa;AACd,oBAAc;AAAA,QACV,OAAG,oBAAK,KAAK,UAAU,WAAW;AAAA,QAClC,eAAe,KAAK,SAAS;AAAA,QAC7B,OAAO,CAAC;AAAA,MACZ;AACA,aAAO,aAAa,KAAK,WAAW;AACpC,yBAAmB,IAAI,KAAK,SAAS,IAAI,WAAW;AAAA,IACxD;AAEA,UAAM,OACF,OAAO,UAAU,UAAU,UAAU,cAAc,OAAO;AAE9D,QAAI,CAAC,MAAM;AACP,aAAO,KAAK,yCAAyC,KAAK,MAAM,EAAE;AAClE;AAAA,IACJ;AAEA,UAAM,aACF,UAAU,UAAU,cACpB,OAAO,UACP,OAAO,SACP;AAEJ,UAAM,OAA+B;AAAA,MACjC;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,OAAO,SAAS;AAAA,MACvB;AAAA,MACA,UAAU,CAAC;AAAA,IACf;AAEA,aAAS,WAAW,OAAO,SAAS;AAChC,UAAI,QAAQ,SAAS,WAAW;AAC5B,aAAK,SAAS,KAAK;AAAA,UACf,SAAS;AAAA,YACL,QAAQ,QAAQ;AAAA,YAChB,SAAS,QAAQ;AAAA,YACjB,WAAW,QAAQ;AAAA,UACvB;AAAA,UACA,2BAAuB;AAAA,YACnB,YAAY;AAAA,YACZ;AAAA,YACA,QAAQ;AAAA,UACZ;AAAA,QACJ,CAAC;AAAA,MACL;AAAA,IACJ;AAEA,SAAK,eAAW,sBAAO,KAAK,UAAU,CAAC,MAAM,EAAE,QAAQ,MAAM;AAE7D,UAAM,YAAQ,6BAAc,YAAY,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK;AACnE,gBAAY,MAAM,OAAO,OAAO,GAAG,IAAI;AAAA,EAC3C;AAEA,WAAS,kBACL,MACA,QACF;AACE,QAAI,aAAa,mBAAmB,IAAI,KAAK,SAAS,EAAE;AAExD,QAAI,CAAC,YAAY;AACb,mBAAa;AAAA,QACT,OAAG,oBAAK,KAAK,UAAU,WAAW;AAAA,QAClC,eAAe,KAAK,SAAS;AAAA,QAC7B,OAAO,CAAC;AAAA,QACR,UAAU,CAAC;AAAA,MACf;AACA,aAAO,aAAa,KAAK,UAAU;AACnC,yBAAmB,IAAI,KAAK,SAAS,IAAI,UAAU;AAAA,IACvD;AAEA,aAAS,cAAc,OAAO,OAAO;AACjC,UAAI,OAAO,WAAW,MAAM,KAAK,CAAC,MAAM,EAAE,OAAO,WAAW,IAAI;AAChE,UAAI,QAAQ,KAAK,gBAAgB,WAAW,cAAc;AACtD,eAAO;AAAA,UACH;AAAA,UACA,KAAK;AAAA,UACL,WAAW;AAAA,QACf;AACA;AAAA,MACJ;AAEA,UAAI,CAAC,MAAM;AACP,cAAM,OAAO,aAAa,MAAM,KAAK,UAAU,WAAW,IAAI;AAC9D,eAAO;AAAA,UACH,IAAI,WAAW;AAAA,UACf,OAAO,+BAAa,IAAI,WAAW,IAAI,KAAK;AAAA,UAC5C,YAAY,MAAM,UAAU,cAAc,WAAW;AAAA,UACrD,MAAM,MAAM,UAAU,cAAc,WAAW;AAAA,UAC/C,UAAU,CAAC;AAAA,QACf;AAAA,MACJ;AAEA,UAAI,WAAW,cAAc;AACzB,aAAK,eAAe,WAAW;AAAA,MACnC;AAEA,UAAI,WAAW,uBAAuB,CAAC,KAAK,qBAAqB;AAC7D,aAAK,sBAAsB,WAAW;AAAA,MAC1C;AAEA,eAAS,WAAW,WAAW,UAAU;AACrC,YAAI,OAA8B;AAAA,UAC9B,QAAQ,QAAQ;AAAA,UAChB,SAAS,QAAQ;AAAA,QACrB;AAEA,YAAI,QAAQ,cAAc;AACtB,eAAK,eAAe,QAAQ;AAAA,QAChC;AAEA,aAAK,SAAS,KAAK;AAAA,UACf,SAAS;AAAA,QACb,CAAC;AAAA,MACL;AAEA,WAAK,eAAW,sBAAO,KAAK,UAAU,CAAC,MAAM,EAAE,QAAQ,MAAM;AAC7D,iBAAW,MAAM,KAAK,IAAI;AAAA,IAC9B;AAEA,QAAI,OAAO,UAAU;AACjB,eAAS,WAAW,OAAO,UAAU;AACjC,cAAM,WAAW,WAAW,SAAS;AAAA,UACjC,CAAC,MAAM,EAAE,OAAO,QAAQ;AAAA,QAC5B;AACA,YAAI,UAAU;AACV,iBAAO;AAAA,YACH;AAAA,YACA,KAAK;AAAA,YACL,QAAQ;AAAA,UACZ;AACA;AAAA,QACJ;AAEA,cAAM,iBAA2C;AAAA,UAC7C,IAAI,QAAQ;AAAA,UACZ,SAAS,QAAQ;AAAA,UACjB,WAAW,QAAQ;AAAA,UACnB,SAAS,QAAQ;AAAA,QACrB;AAEA,mBAAW,SAAS,KAAK,cAAc;AAAA,MAC3C;AAAA,IACJ;AAEA,eAAW,YAAQ,sBAAO,WAAW,OAAO,CAAC,MAAM,EAAE,KAAK;AAC1D,eAAW,eAAW,sBAAO,WAAW,UAAU,CAAC,MAAM,EAAE,EAAE;AAAA,EACjE;AAEA,WAAS,aACL,MACA,UACA,IACF;AACE,QAAI,CAAC,SAAS;AACV,gBAAU,kBAAAC,UAAiB,IAAI,SAAS,QAAQ;AAEhD,UAAI,CAAC,SAAS;AACV,YAAI,CAAC,iBAAiB,IAAI,SAAS,QAAQ,GAAG;AAC1C,iBAAO;AAAA,YACH;AAAA,YACA,KAAK;AAAA,YACL,SAAS;AAAA,UACb;AACA,2BAAiB,IAAI,SAAS,QAAQ;AAAA,QAC1C;AAEA,eAAO;AAAA,UACH;AAAA,QACJ;AACA,kBAAU,kBAAAA,UAAiB,IAAI,IAAI;AAAA,MACvC;AAAA,IACJ;AAEA,UAAM,WAAW,SAAS,IAAI,EAAE;AAEhC,QAAI,CAAC,CAAC,WAAW,CAAC,UAAU;AACxB,aAAO;AAAA,QACH;AAAA,QACA,KAAK;AAAA,QACL;AAAA,MACJ;AACA,aAAO;AAAA,IACX;AAEA,WAAO;AAAA,MACH;AAAA,IACJ;AAAA,EACJ;AAEA,SAAO;AACX;",
+  "names": ["parser", "defaultBookIdMap"]
 }

package/dist/cjs/index.cjs

@@ -29,6 +29,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var helloao_tools_exports = {};
 __export(helloao_tools_exports, {
   generation: () => generation,
+  log: () => log,
   parser: () => parser,
   utils: () => utils
 });
@@ -36,9 +37,11 @@ module.exports = __toCommonJS(helloao_tools_exports);
 var utils = __toESM(require("./utils.js"));
 var generation = __toESM(require("./generation/index.js"));
 var parser = __toESM(require("./parser/index.js"));
+var log = __toESM(require("./log.js"));
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   generation,
+  log,
   parser,
   utils
 });

package/dist/cjs/index.cjs.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../index.ts"],
-  "sourcesContent": ["import * as utils from './utils.js';\r\nimport * as generation from './generation/index.js';\r\nimport * as parser from './parser/index.js';\r\n\r\nexport { utils, generation, parser };\r\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,YAAuB;AACvB,iBAA4B;AAC5B,aAAwB;",
+  "sourcesContent": ["import * as utils from './utils.js';\nimport * as generation from './generation/index.js';\nimport * as parser from './parser/index.js';\nimport * as log from './log.js';\n\nexport { utils, generation, parser, log };\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,YAAuB;AACvB,iBAA4B;AAC5B,aAAwB;AACxB,UAAqB;",
   "names": []
 }

package/dist/cjs/log.cjs

@@ -0,0 +1,50 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var log_exports = {};
+__export(log_exports, {
+  consoleLogger: () => consoleLogger,
+  getLogger: () => getLogger,
+  setLogger: () => setLogger
+});
+module.exports = __toCommonJS(log_exports);
+const consoleLogger = {
+  log: (message, ...args) => {
+    console.log(message, ...args);
+  },
+  warn: (message, ...args) => {
+    console.warn(message, ...args);
+  },
+  error: (message, ...args) => {
+    console.error(message, ...args);
+  }
+};
+let logger = consoleLogger;
+function getLogger() {
+  return logger;
+}
+function setLogger(newLogger) {
+  logger = newLogger;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  consoleLogger,
+  getLogger,
+  setLogger
+});
+//# sourceMappingURL=log.cjs.map

package/dist/cjs/log.cjs.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../log.ts"],
+  "sourcesContent": ["/**\n * A simple logging interface that can be used to log messages.\n */\nexport interface Logger {\n    log: (message: any, ...args: any[]) => void;\n    warn: (message: string, ...args: any[]) => void;\n    error: (message: string, ...args: any[]) => void;\n}\n\n/**\n * A logger that logs messages to the console.\n */\nexport const consoleLogger: Logger = {\n    log: (message: string, ...args) => {\n        console.log(message, ...args);\n    },\n    warn: (message: string, ...args) => {\n        console.warn(message, ...args);\n    },\n    error: (message: string, ...args) => {\n        console.error(message, ...args);\n    },\n};\n\n/**\n * The logger that should be used for logging messages.\n */\nlet logger: Logger = consoleLogger;\n\n/**\n * Gets the logger that is currently being used for logging messages.\n * @returns The logger that is currently being used.\n */\nexport function getLogger(): Logger {\n    return logger;\n}\n\n/**\n * Sets the logger to use for logging messages.\n * @param newLogger The new logger to use.\n */\nexport function setLogger(newLogger: Logger) {\n    logger = newLogger;\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAYO,MAAM,gBAAwB;AAAA,EACjC,KAAK,CAAC,YAAoB,SAAS;AAC/B,YAAQ,IAAI,SAAS,GAAG,IAAI;AAAA,EAChC;AAAA,EACA,MAAM,CAAC,YAAoB,SAAS;AAChC,YAAQ,KAAK,SAAS,GAAG,IAAI;AAAA,EACjC;AAAA,EACA,OAAO,CAAC,YAAoB,SAAS;AACjC,YAAQ,MAAM,SAAS,GAAG,IAAI;AAAA,EAClC;AACJ;AAKA,IAAI,SAAiB;AAMd,SAAS,YAAoB;AAChC,SAAO;AACX;AAMO,SAAS,UAAU,WAAmB;AACzC,WAAS;AACb;",
+  "names": []
+}