@helloao/tools 0.0.19 → 0.1.0

package/dist/cjs/generation/dataset.cjs

@@ -38,7 +38,7 @@ function generateDataset(files, parser = new globalThis.DOMParser(), bookMap) {
   };
   let usfmParser = new import_usfm_parser.UsfmParser();
   let usxParser = new import_usx_parser.USXParser(parser);
-  let codexParser = new import_codex_parser.CodexParser();
+  let codexParser = new import_codex_parser.CodexParser(parser);
   let csvCommentaryParser = new import_commentary_csv_parser.CommentaryCsvParser();
   let tyndaleXmlParser = new import_tyndale_xml_parser.TyndaleXmlParser(parser);
   let parsedTranslations = /* @__PURE__ */ new Map();

package/dist/cjs/generation/dataset.cjs.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../generation/dataset.ts"],
-  "sourcesContent": ["import { UsfmParser } from '../parser/usfm-parser.js';\nimport { USXParser } from '../parser/usx-parser.js';\nimport {\n    Commentary,\n    CommentaryBook,\n    CommentaryBookChapter,\n    CommentaryChapterData,\n    CommentaryProfile,\n    InputCommentaryFile,\n    InputFile,\n    InputFileBase,\n    InputTranslationFile,\n    MetadataBase,\n    Translation,\n    TranslationBook,\n    TranslationBookChapter,\n    Dataset as CommonDataset,\n    DatasetBook,\n    DatasetBookChapter,\n} from './common-types.js';\nimport {\n    bookIdMap as defaultBookIdMap,\n    bookOrderMap,\n    apocryphaBooks,\n} from './book-order.js';\nimport { omit, sortBy, sortedIndexBy } from 'lodash';\nimport { getAudioUrlsForChapter } from './audio.js';\nimport { CodexParser } from '../parser/codex-parser.js';\nimport { CommentaryCsvParser } from '../parser/commentary-csv-parser.js';\nimport {\n    CommentaryParseTree,\n    ParseMessage,\n    ParseTree,\n} from '../parser/types.js';\nimport { TyndaleXmlParser } from '../parser/tyndale-xml-parser.js';\nimport { getLogger } from '../log.js';\n\n/**\n * Defines an interface that contains generated dataset info.\n */\nexport interface DatasetOutput {\n    /**\n     * The list of translations that are available in the dataset.\n     */\n    translations: DatasetTranslation[];\n\n    /**\n     * The list of commentaries that are available in the dataset.\n     */\n    commentaries: DatasetCommentary[];\n\n    /**\n     * The list of datasets that are available in the dataset.\n     */\n    datasets?: DatasetDataset[];\n\n    parseMessages?: {\n        [key: string]: ParseMessage[];\n    };\n}\n\n/**\n * Defines a translation that is used in the dataset.\n */\nexport interface DatasetTranslation extends Translation {\n    /**\n     * The list of books that are available for the translation.\n     */\n    books: DatasetTranslationBook[];\n}\n\n/**\n * Defines a commentary that is used in the dataset.\n */\nexport interface DatasetCommentary extends Commentary {\n    /**\n     * The list of books that are available for the commentary.\n     */\n    books: DatasetCommentaryBook[];\n\n    /**\n     * The list of profiles that  are available for the commentary.\n     */\n    profiles: DatasetCommentaryProfile[];\n}\n\n/**\n * Defines a translation book that is used in the dataset.\n */\nexport interface DatasetTranslationBook extends TranslationBook {\n    /**\n     * The list of chapters that are available for the book.\n     */\n    chapters: TranslationBookChapter[];\n}\n\n/**\n * Defines a commentary book that is used in the dataset.\n */\nexport interface DatasetCommentaryBook extends CommentaryBook {\n    /**\n     * The list of chapters that are available for the book.\n     */\n    chapters: CommentaryBookChapter[];\n}\n\nexport interface DatasetCommentaryProfile extends CommentaryProfile {\n    /**\n     * The contents of the profile.\n     */\n    content: string[];\n}\n\nexport interface DatasetDataset extends CommonDataset {\n    books: DatasetDatasetBook[];\n}\n\nexport interface DatasetDatasetBook extends DatasetBook {\n    chapters: DatasetBookChapter[];\n}\n\n/**\n * Generates a list of output files from the given list of input files.\n * @param file The list of files.\n * @param parser The parser to use for parsing the files.\n * @param bookMap The map of book IDs to names. If undefined, then a default map will be used.\n */\nexport function generateDataset(\n    files: InputFile[],\n    parser: DOMParser = new globalThis.DOMParser(),\n    bookMap?: Map<string, { commonName: string }> | undefined\n): DatasetOutput {\n    const logger = getLogger();\n    let output: DatasetOutput = {\n        translations: [],\n        commentaries: [],\n    };\n\n    let usfmParser = new UsfmParser();\n    let usxParser = new USXParser(parser);\n    let codexParser = new CodexParser();\n    let csvCommentaryParser = new CommentaryCsvParser();\n    let tyndaleXmlParser = new TyndaleXmlParser(parser);\n\n    let parsedTranslations = new Map<string, DatasetTranslation>();\n    let parsedCommentaries = new Map<string, DatasetCommentary>();\n\n    const unknownLanguages = new Set<string>();\n    for (let file of files) {\n        try {\n            let parser:\n                | UsfmParser\n                | USXParser\n                | CodexParser\n                | CommentaryCsvParser\n                | TyndaleXmlParser;\n            if (file.fileType === 'usfm') {\n                parser = usfmParser;\n            } else if (file.fileType === 'usx') {\n                parser = usxParser;\n            } else if (file.fileType === 'json') {\n                parser = codexParser;\n            } else if (file.fileType === 'commentary/csv') {\n                parser = csvCommentaryParser;\n            } else if (file.fileType === 'commentary/tyndale-xml') {\n                parser = tyndaleXmlParser;\n            } else {\n                logger.warn(\n                    '[generate] File does not have a valid type!',\n                    file.name\n                );\n                continue;\n            }\n\n            const parsed = parser.parse(file.content);\n\n            if (\n                'parseMessages' in parsed &&\n                parsed.parseMessages &&\n                file.name\n            ) {\n                const messages = (output.parseMessages =\n                    output.parseMessages || {});\n                messages[file.name] = parsed.parseMessages;\n            }\n\n            if (parsed.type === 'root') {\n                addTranslationTree(file as InputTranslationFile, parsed);\n            } else {\n                addCommentaryTree(file as InputCommentaryFile, parsed);\n            }\n        } catch (err) {\n            logger.error(\n                `[generate] Error occurred while parsing ${file.name}`,\n                err\n            );\n        }\n    }\n\n    function addTranslationTree(file: InputTranslationFile, parsed: ParseTree) {\n        const id = parsed.id;\n\n        if (!id) {\n            logger.warn(\n                '[generate] File does not have a valid book ID!',\n                file.name,\n                id\n            );\n            return;\n        }\n\n        const order = bookOrderMap.get(id);\n\n        if (typeof order !== 'number') {\n            logger.warn('[generate] Book does not have an order!', id);\n            return;\n        }\n\n        const isApocryphal = apocryphaBooks.has(id);\n\n        const bookName = getBookNames(file, file.metadata, id);\n\n        let translation = parsedTranslations.get(file.metadata.id);\n\n        if (!translation) {\n            translation = {\n                ...omit(file.metadata, 'direction'),\n                textDirection: file.metadata.direction,\n                books: [],\n            };\n            output.translations.push(translation);\n            parsedTranslations.set(file.metadata.id, translation);\n        }\n\n        const name =\n            parsed.header ??\n            (bookName?.exactMatch ? bookName?.bookName?.commonName : null) ??\n            parsed.title ??\n            bookName?.bookName?.commonName;\n\n        if (!name) {\n            logger.warn('[generate] Book does not have a name!', file.name, id);\n            return;\n        }\n\n        const commonName =\n            (bookName?.exactMatch ? bookName?.bookName?.commonName : null) ??\n            parsed.header ??\n            parsed.title ??\n            bookName?.bookName?.commonName ??\n            id;\n\n        const book: DatasetTranslationBook = {\n            id: id,\n            name,\n            commonName,\n            title: parsed.title ?? null,\n            order,\n            chapters: [],\n        };\n\n        if (isApocryphal) {\n            book.isApocryphal = true;\n        }\n\n        for (let content of parsed.content) {\n            if (content.type === 'chapter') {\n                book.chapters.push({\n                    chapter: {\n                        number: content.number,\n                        content: content.content,\n                        footnotes: content.footnotes,\n                    },\n                    thisChapterAudioLinks: getAudioUrlsForChapter(\n                        translation.id,\n                        id,\n                        content.number\n                    ),\n                });\n            }\n        }\n\n        book.chapters = sortBy(book.chapters, (c) => c.chapter.number);\n\n        const index = sortedIndexBy(translation.books, book, (b) => b.order);\n        translation.books.splice(index, 0, book);\n    }\n\n    function addCommentaryTree(\n        file: InputCommentaryFile,\n        parsed: CommentaryParseTree\n    ) {\n        let commentary = parsedCommentaries.get(file.metadata.id);\n\n        if (!commentary) {\n            commentary = {\n                ...omit(file.metadata, 'direction'),\n                textDirection: file.metadata.direction,\n                books: [],\n                profiles: [],\n            };\n            output.commentaries.push(commentary);\n            parsedCommentaries.set(file.metadata.id, commentary);\n        }\n\n        for (let parsedBook of parsed.books) {\n            let book = commentary.books.find((b) => b.id === parsedBook.book);\n            if (book && book.introduction && parsedBook.introduction) {\n                logger.warn(\n                    '[generate] Book already exists in commentary!',\n                    file.name,\n                    parsedBook.book\n                );\n                continue;\n            }\n\n            if (!book) {\n                const name = getBookNames(file, file.metadata, parsedBook.book);\n                book = {\n                    id: parsedBook.book,\n                    order: bookOrderMap.get(parsedBook.book) ?? -1,\n                    commonName: name?.bookName?.commonName ?? parsedBook.book,\n                    name: name?.bookName?.commonName ?? parsedBook.book,\n                    chapters: [],\n                };\n            }\n\n            if (parsedBook.introduction) {\n                book.introduction = parsedBook.introduction;\n            }\n\n            if (parsedBook.introductionSummary && !book.introductionSummary) {\n                book.introductionSummary = parsedBook.introductionSummary;\n            }\n\n            for (let chapter of parsedBook.chapters) {\n                let data: CommentaryChapterData = {\n                    number: chapter.number,\n                    content: chapter.verses,\n                };\n\n                if (chapter.introduction) {\n                    data.introduction = chapter.introduction;\n                }\n\n                book.chapters.push({\n                    chapter: data,\n                });\n            }\n\n            book.chapters = sortBy(book.chapters, (c) => c.chapter.number);\n            commentary.books.push(book);\n        }\n\n        if (parsed.profiles) {\n            for (let profile of parsed.profiles) {\n                const existing = commentary.profiles.find(\n                    (p) => p.id === profile.id\n                );\n                if (existing) {\n                    logger.warn(\n                        '[generate] Profile already exists in commentary!',\n                        file.name,\n                        profile.id\n                    );\n                    continue;\n                }\n\n                const datasetProfile: DatasetCommentaryProfile = {\n                    id: profile.id,\n                    subject: profile.subject,\n                    reference: profile.reference,\n                    content: profile.content,\n                };\n\n                commentary.profiles.push(datasetProfile);\n            }\n        }\n\n        commentary.books = sortBy(commentary.books, (b) => b.order);\n        commentary.profiles = sortBy(commentary.profiles, (p) => p.id);\n    }\n\n    function getBookNames(\n        file: InputFileBase,\n        metadata: MetadataBase,\n        id: string\n    ) {\n        let exactMatch = true;\n        let map = bookMap;\n        if (!map) {\n            map = defaultBookIdMap.get(metadata.language);\n\n            if (!map) {\n                if (!unknownLanguages.has(metadata.language)) {\n                    logger.warn(\n                        '[generate] File does not have a known language!',\n                        file.name,\n                        metadata.language\n                    );\n                    unknownLanguages.add(metadata.language);\n                }\n\n                logger.warn(\n                    '[generate] Using English book map for unknown language! This might result in outputting english book names.'\n                );\n                exactMatch = false;\n                map = defaultBookIdMap.get('en');\n            }\n        }\n\n        const bookName = map?.get(id);\n\n        if (!!map && !bookName) {\n            logger.warn(\n                '[generate] Book name not found for ID!',\n                file.name,\n                id\n            );\n            return null;\n        }\n\n        return {\n            exactMatch,\n            bookName,\n        };\n    }\n\n    return output;\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,yBAA2B;AAC3B,wBAA0B;AAmB1B,wBAIO;AACP,oBAA4C;AAC5C,mBAAuC;AACvC,0BAA4B;AAC5B,mCAAoC;AAMpC,gCAAiC;AACjC,iBAA0B;AA4FnB,SAAS,gBACZ,OACA,SAAoB,IAAI,WAAW,UAAU,GAC7C,SACa;AACb,QAAM,aAAS,sBAAU;AACzB,MAAI,SAAwB;AAAA,IACxB,cAAc,CAAC;AAAA,IACf,cAAc,CAAC;AAAA,EACnB;AAEA,MAAI,aAAa,IAAI,8BAAW;AAChC,MAAI,YAAY,IAAI,4BAAU,MAAM;AACpC,MAAI,cAAc,IAAI,gCAAY;AAClC,MAAI,sBAAsB,IAAI,iDAAoB;AAClD,MAAI,mBAAmB,IAAI,2CAAiB,MAAM;AAElD,MAAI,qBAAqB,oBAAI,IAAgC;AAC7D,MAAI,qBAAqB,oBAAI,IAA+B;AAE5D,QAAM,mBAAmB,oBAAI,IAAY;AACzC,WAAS,QAAQ,OAAO;AACpB,QAAI;AACA,UAAIA;AAMJ,UAAI,KAAK,aAAa,QAAQ;AAC1B,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,OAAO;AAChC,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,QAAQ;AACjC,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,kBAAkB;AAC3C,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,0BAA0B;AACnD,QAAAA,UAAS;AAAA,MACb,OAAO;AACH,eAAO;AAAA,UACH;AAAA,UACA,KAAK;AAAA,QACT;AACA;AAAA,MACJ;AAEA,YAAM,SAASA,QAAO,MAAM,KAAK,OAAO;AAExC,UACI,mBAAmB,UACnB,OAAO,iBACP,KAAK,MACP;AACE,cAAM,WAAY,OAAO,gBACrB,OAAO,iBAAiB,CAAC;AAC7B,iBAAS,KAAK,IAAI,IAAI,OAAO;AAAA,MACjC;AAEA,UAAI,OAAO,SAAS,QAAQ;AACxB,2BAAmB,MAA8B,MAAM;AAAA,MAC3D,OAAO;AACH,0BAAkB,MAA6B,MAAM;AAAA,MACzD;AAAA,IACJ,SAAS,KAAK;AACV,aAAO;AAAA,QACH,2CAA2C,KAAK,IAAI;AAAA,QACpD;AAAA,MACJ;AAAA,IACJ;AAAA,EACJ;AAEA,WAAS,mBAAmB,MAA4B,QAAmB;AACvE,UAAM,KAAK,OAAO;AAElB,QAAI,CAAC,IAAI;AACL,aAAO;AAAA,QACH;AAAA,QACA,KAAK;AAAA,QACL;AAAA,MACJ;AACA;AAAA,IACJ;AAEA,UAAM,QAAQ,+BAAa,IAAI,EAAE;AAEjC,QAAI,OAAO,UAAU,UAAU;AAC3B,aAAO,KAAK,2CAA2C,EAAE;AACzD;AAAA,IACJ;AAEA,UAAM,eAAe,iCAAe,IAAI,EAAE;AAE1C,UAAM,WAAW,aAAa,MAAM,KAAK,UAAU,EAAE;AAErD,QAAI,cAAc,mBAAmB,IAAI,KAAK,SAAS,EAAE;AAEzD,QAAI,CAAC,aAAa;AACd,oBAAc;AAAA,QACV,OAAG,oBAAK,KAAK,UAAU,WAAW;AAAA,QAClC,eAAe,KAAK,SAAS;AAAA,QAC7B,OAAO,CAAC;AAAA,MACZ;AACA,aAAO,aAAa,KAAK,WAAW;AACpC,yBAAmB,IAAI,KAAK,SAAS,IAAI,WAAW;AAAA,IACxD;AAEA,UAAM,OACF,OAAO,WACN,UAAU,aAAa,UAAU,UAAU,aAAa,SACzD,OAAO,SACP,UAAU,UAAU;AAExB,QAAI,CAAC,MAAM;AACP,aAAO,KAAK,yCAAyC,KAAK,MAAM,EAAE;AAClE;AAAA,IACJ;AAEA,UAAM,cACD,UAAU,aAAa,UAAU,UAAU,aAAa,SACzD,OAAO,UACP,OAAO,SACP,UAAU,UAAU,cACpB;AAEJ,UAAM,OAA+B;AAAA,MACjC;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,OAAO,SAAS;AAAA,MACvB;AAAA,MACA,UAAU,CAAC;AAAA,IACf;AAEA,QAAI,cAAc;AACd,WAAK,eAAe;AAAA,IACxB;AAEA,aAAS,WAAW,OAAO,SAAS;AAChC,UAAI,QAAQ,SAAS,WAAW;AAC5B,aAAK,SAAS,KAAK;AAAA,UACf,SAAS;AAAA,YACL,QAAQ,QAAQ;AAAA,YAChB,SAAS,QAAQ;AAAA,YACjB,WAAW,QAAQ;AAAA,UACvB;AAAA,UACA,2BAAuB;AAAA,YACnB,YAAY;AAAA,YACZ;AAAA,YACA,QAAQ;AAAA,UACZ;AAAA,QACJ,CAAC;AAAA,MACL;AAAA,IACJ;AAEA,SAAK,eAAW,sBAAO,KAAK,UAAU,CAAC,MAAM,EAAE,QAAQ,MAAM;AAE7D,UAAM,YAAQ,6BAAc,YAAY,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK;AACnE,gBAAY,MAAM,OAAO,OAAO,GAAG,IAAI;AAAA,EAC3C;AAEA,WAAS,kBACL,MACA,QACF;AACE,QAAI,aAAa,mBAAmB,IAAI,KAAK,SAAS,EAAE;AAExD,QAAI,CAAC,YAAY;AACb,mBAAa;AAAA,QACT,OAAG,oBAAK,KAAK,UAAU,WAAW;AAAA,QAClC,eAAe,KAAK,SAAS;AAAA,QAC7B,OAAO,CAAC;AAAA,QACR,UAAU,CAAC;AAAA,MACf;AACA,aAAO,aAAa,KAAK,UAAU;AACnC,yBAAmB,IAAI,KAAK,SAAS,IAAI,UAAU;AAAA,IACvD;AAEA,aAAS,cAAc,OAAO,OAAO;AACjC,UAAI,OAAO,WAAW,MAAM,KAAK,CAAC,MAAM,EAAE,OAAO,WAAW,IAAI;AAChE,UAAI,QAAQ,KAAK,gBAAgB,WAAW,cAAc;AACtD,eAAO;AAAA,UACH;AAAA,UACA,KAAK;AAAA,UACL,WAAW;AAAA,QACf;AACA;AAAA,MACJ;AAEA,UAAI,CAAC,MAAM;AACP,cAAM,OAAO,aAAa,MAAM,KAAK,UAAU,WAAW,IAAI;AAC9D,eAAO;AAAA,UACH,IAAI,WAAW;AAAA,UACf,OAAO,+BAAa,IAAI,WAAW,IAAI,KAAK;AAAA,UAC5C,YAAY,MAAM,UAAU,cAAc,WAAW;AAAA,UACrD,MAAM,MAAM,UAAU,cAAc,WAAW;AAAA,UAC/C,UAAU,CAAC;AAAA,QACf;AAAA,MACJ;AAEA,UAAI,WAAW,cAAc;AACzB,aAAK,eAAe,WAAW;AAAA,MACnC;AAEA,UAAI,WAAW,uBAAuB,CAAC,KAAK,qBAAqB;AAC7D,aAAK,sBAAsB,WAAW;AAAA,MAC1C;AAEA,eAAS,WAAW,WAAW,UAAU;AACrC,YAAI,OAA8B;AAAA,UAC9B,QAAQ,QAAQ;AAAA,UAChB,SAAS,QAAQ;AAAA,QACrB;AAEA,YAAI,QAAQ,cAAc;AACtB,eAAK,eAAe,QAAQ;AAAA,QAChC;AAEA,aAAK,SAAS,KAAK;AAAA,UACf,SAAS;AAAA,QACb,CAAC;AAAA,MACL;AAEA,WAAK,eAAW,sBAAO,KAAK,UAAU,CAAC,MAAM,EAAE,QAAQ,MAAM;AAC7D,iBAAW,MAAM,KAAK,IAAI;AAAA,IAC9B;AAEA,QAAI,OAAO,UAAU;AACjB,eAAS,WAAW,OAAO,UAAU;AACjC,cAAM,WAAW,WAAW,SAAS;AAAA,UACjC,CAAC,MAAM,EAAE,OAAO,QAAQ;AAAA,QAC5B;AACA,YAAI,UAAU;AACV,iBAAO;AAAA,YACH;AAAA,YACA,KAAK;AAAA,YACL,QAAQ;AAAA,UACZ;AACA;AAAA,QACJ;AAEA,cAAM,iBAA2C;AAAA,UAC7C,IAAI,QAAQ;AAAA,UACZ,SAAS,QAAQ;AAAA,UACjB,WAAW,QAAQ;AAAA,UACnB,SAAS,QAAQ;AAAA,QACrB;AAEA,mBAAW,SAAS,KAAK,cAAc;AAAA,MAC3C;AAAA,IACJ;AAEA,eAAW,YAAQ,sBAAO,WAAW,OAAO,CAAC,MAAM,EAAE,KAAK;AAC1D,eAAW,eAAW,sBAAO,WAAW,UAAU,CAAC,MAAM,EAAE,EAAE;AAAA,EACjE;AAEA,WAAS,aACL,MACA,UACA,IACF;AACE,QAAI,aAAa;AACjB,QAAI,MAAM;AACV,QAAI,CAAC,KAAK;AACN,YAAM,kBAAAC,UAAiB,IAAI,SAAS,QAAQ;AAE5C,UAAI,CAAC,KAAK;AACN,YAAI,CAAC,iBAAiB,IAAI,SAAS,QAAQ,GAAG;AAC1C,iBAAO;AAAA,YACH;AAAA,YACA,KAAK;AAAA,YACL,SAAS;AAAA,UACb;AACA,2BAAiB,IAAI,SAAS,QAAQ;AAAA,QAC1C;AAEA,eAAO;AAAA,UACH;AAAA,QACJ;AACA,qBAAa;AACb,cAAM,kBAAAA,UAAiB,IAAI,IAAI;AAAA,MACnC;AAAA,IACJ;AAEA,UAAM,WAAW,KAAK,IAAI,EAAE;AAE5B,QAAI,CAAC,CAAC,OAAO,CAAC,UAAU;AACpB,aAAO;AAAA,QACH;AAAA,QACA,KAAK;AAAA,QACL;AAAA,MACJ;AACA,aAAO;AAAA,IACX;AAEA,WAAO;AAAA,MACH;AAAA,MACA;AAAA,IACJ;AAAA,EACJ;AAEA,SAAO;AACX;",
+  "sourcesContent": ["import { UsfmParser } from '../parser/usfm-parser.js';\nimport { USXParser } from '../parser/usx-parser.js';\nimport {\n    Commentary,\n    CommentaryBook,\n    CommentaryBookChapter,\n    CommentaryChapterData,\n    CommentaryProfile,\n    InputCommentaryFile,\n    InputFile,\n    InputFileBase,\n    InputTranslationFile,\n    MetadataBase,\n    Translation,\n    TranslationBook,\n    TranslationBookChapter,\n    Dataset as CommonDataset,\n    DatasetBook,\n    DatasetBookChapter,\n} from './common-types.js';\nimport {\n    bookIdMap as defaultBookIdMap,\n    bookOrderMap,\n    apocryphaBooks,\n} from './book-order.js';\nimport { omit, sortBy, sortedIndexBy } from 'lodash';\nimport { getAudioUrlsForChapter } from './audio.js';\nimport { CodexParser } from '../parser/codex-parser.js';\nimport { CommentaryCsvParser } from '../parser/commentary-csv-parser.js';\nimport {\n    CommentaryParseTree,\n    ParseMessage,\n    ParseTree,\n} from '../parser/types.js';\nimport { TyndaleXmlParser } from '../parser/tyndale-xml-parser.js';\nimport { getLogger } from '../log.js';\n\n/**\n * Defines an interface that contains generated dataset info.\n */\nexport interface DatasetOutput {\n    /**\n     * The list of translations that are available in the dataset.\n     */\n    translations: DatasetTranslation[];\n\n    /**\n     * The list of commentaries that are available in the dataset.\n     */\n    commentaries: DatasetCommentary[];\n\n    /**\n     * The list of datasets that are available in the dataset.\n     */\n    datasets?: DatasetDataset[];\n\n    parseMessages?: {\n        [key: string]: ParseMessage[];\n    };\n}\n\n/**\n * Defines a translation that is used in the dataset.\n */\nexport interface DatasetTranslation extends Translation {\n    /**\n     * The list of books that are available for the translation.\n     */\n    books: DatasetTranslationBook[];\n}\n\n/**\n * Defines a commentary that is used in the dataset.\n */\nexport interface DatasetCommentary extends Commentary {\n    /**\n     * The list of books that are available for the commentary.\n     */\n    books: DatasetCommentaryBook[];\n\n    /**\n     * The list of profiles that  are available for the commentary.\n     */\n    profiles: DatasetCommentaryProfile[];\n}\n\n/**\n * Defines a translation book that is used in the dataset.\n */\nexport interface DatasetTranslationBook extends TranslationBook {\n    /**\n     * The list of chapters that are available for the book.\n     */\n    chapters: TranslationBookChapter[];\n}\n\n/**\n * Defines a commentary book that is used in the dataset.\n */\nexport interface DatasetCommentaryBook extends CommentaryBook {\n    /**\n     * The list of chapters that are available for the book.\n     */\n    chapters: CommentaryBookChapter[];\n}\n\nexport interface DatasetCommentaryProfile extends CommentaryProfile {\n    /**\n     * The contents of the profile.\n     */\n    content: string[];\n}\n\nexport interface DatasetDataset extends CommonDataset {\n    books: DatasetDatasetBook[];\n}\n\nexport interface DatasetDatasetBook extends DatasetBook {\n    chapters: DatasetBookChapter[];\n}\n\n/**\n * Generates a list of output files from the given list of input files.\n * @param file The list of files.\n * @param parser The parser to use for parsing the files.\n * @param bookMap The map of book IDs to names. If undefined, then a default map will be used.\n */\nexport function generateDataset(\n    files: InputFile[],\n    parser: DOMParser = new globalThis.DOMParser(),\n    bookMap?: Map<string, { commonName: string }> | undefined\n): DatasetOutput {\n    const logger = getLogger();\n    let output: DatasetOutput = {\n        translations: [],\n        commentaries: [],\n    };\n\n    let usfmParser = new UsfmParser();\n    let usxParser = new USXParser(parser);\n    let codexParser = new CodexParser(parser);\n    let csvCommentaryParser = new CommentaryCsvParser();\n    let tyndaleXmlParser = new TyndaleXmlParser(parser);\n\n    let parsedTranslations = new Map<string, DatasetTranslation>();\n    let parsedCommentaries = new Map<string, DatasetCommentary>();\n\n    const unknownLanguages = new Set<string>();\n    for (let file of files) {\n        try {\n            let parser:\n                | UsfmParser\n                | USXParser\n                | CodexParser\n                | CommentaryCsvParser\n                | TyndaleXmlParser;\n            if (file.fileType === 'usfm') {\n                parser = usfmParser;\n            } else if (file.fileType === 'usx') {\n                parser = usxParser;\n            } else if (file.fileType === 'json') {\n                parser = codexParser;\n            } else if (file.fileType === 'commentary/csv') {\n                parser = csvCommentaryParser;\n            } else if (file.fileType === 'commentary/tyndale-xml') {\n                parser = tyndaleXmlParser;\n            } else {\n                logger.warn(\n                    '[generate] File does not have a valid type!',\n                    file.name\n                );\n                continue;\n            }\n\n            const parsed = parser.parse(file.content);\n\n            if (\n                'parseMessages' in parsed &&\n                parsed.parseMessages &&\n                file.name\n            ) {\n                const messages = (output.parseMessages =\n                    output.parseMessages || {});\n                messages[file.name] = parsed.parseMessages;\n            }\n\n            if (parsed.type === 'root') {\n                addTranslationTree(file as InputTranslationFile, parsed);\n            } else {\n                addCommentaryTree(file as InputCommentaryFile, parsed);\n            }\n        } catch (err) {\n            logger.error(\n                `[generate] Error occurred while parsing ${file.name}`,\n                err\n            );\n        }\n    }\n\n    function addTranslationTree(file: InputTranslationFile, parsed: ParseTree) {\n        const id = parsed.id;\n\n        if (!id) {\n            logger.warn(\n                '[generate] File does not have a valid book ID!',\n                file.name,\n                id\n            );\n            return;\n        }\n\n        const order = bookOrderMap.get(id);\n\n        if (typeof order !== 'number') {\n            logger.warn('[generate] Book does not have an order!', id);\n            return;\n        }\n\n        const isApocryphal = apocryphaBooks.has(id);\n\n        const bookName = getBookNames(file, file.metadata, id);\n\n        let translation = parsedTranslations.get(file.metadata.id);\n\n        if (!translation) {\n            translation = {\n                ...omit(file.metadata, 'direction'),\n                textDirection: file.metadata.direction,\n                books: [],\n            };\n            output.translations.push(translation);\n            parsedTranslations.set(file.metadata.id, translation);\n        }\n\n        const name =\n            parsed.header ??\n            (bookName?.exactMatch ? bookName?.bookName?.commonName : null) ??\n            parsed.title ??\n            bookName?.bookName?.commonName;\n\n        if (!name) {\n            logger.warn('[generate] Book does not have a name!', file.name, id);\n            return;\n        }\n\n        const commonName =\n            (bookName?.exactMatch ? bookName?.bookName?.commonName : null) ??\n            parsed.header ??\n            parsed.title ??\n            bookName?.bookName?.commonName ??\n            id;\n\n        const book: DatasetTranslationBook = {\n            id: id,\n            name,\n            commonName,\n            title: parsed.title ?? null,\n            order,\n            chapters: [],\n        };\n\n        if (isApocryphal) {\n            book.isApocryphal = true;\n        }\n\n        for (let content of parsed.content) {\n            if (content.type === 'chapter') {\n                book.chapters.push({\n                    chapter: {\n                        number: content.number,\n                        content: content.content,\n                        footnotes: content.footnotes,\n                    },\n                    thisChapterAudioLinks: getAudioUrlsForChapter(\n                        translation.id,\n                        id,\n                        content.number\n                    ),\n                });\n            }\n        }\n\n        book.chapters = sortBy(book.chapters, (c) => c.chapter.number);\n\n        const index = sortedIndexBy(translation.books, book, (b) => b.order);\n        translation.books.splice(index, 0, book);\n    }\n\n    function addCommentaryTree(\n        file: InputCommentaryFile,\n        parsed: CommentaryParseTree\n    ) {\n        let commentary = parsedCommentaries.get(file.metadata.id);\n\n        if (!commentary) {\n            commentary = {\n                ...omit(file.metadata, 'direction'),\n                textDirection: file.metadata.direction,\n                books: [],\n                profiles: [],\n            };\n            output.commentaries.push(commentary);\n            parsedCommentaries.set(file.metadata.id, commentary);\n        }\n\n        for (let parsedBook of parsed.books) {\n            let book = commentary.books.find((b) => b.id === parsedBook.book);\n            if (book && book.introduction && parsedBook.introduction) {\n                logger.warn(\n                    '[generate] Book already exists in commentary!',\n                    file.name,\n                    parsedBook.book\n                );\n                continue;\n            }\n\n            if (!book) {\n                const name = getBookNames(file, file.metadata, parsedBook.book);\n                book = {\n                    id: parsedBook.book,\n                    order: bookOrderMap.get(parsedBook.book) ?? -1,\n                    commonName: name?.bookName?.commonName ?? parsedBook.book,\n                    name: name?.bookName?.commonName ?? parsedBook.book,\n                    chapters: [],\n                };\n            }\n\n            if (parsedBook.introduction) {\n                book.introduction = parsedBook.introduction;\n            }\n\n            if (parsedBook.introductionSummary && !book.introductionSummary) {\n                book.introductionSummary = parsedBook.introductionSummary;\n            }\n\n            for (let chapter of parsedBook.chapters) {\n                let data: CommentaryChapterData = {\n                    number: chapter.number,\n                    content: chapter.verses,\n                };\n\n                if (chapter.introduction) {\n                    data.introduction = chapter.introduction;\n                }\n\n                book.chapters.push({\n                    chapter: data,\n                });\n            }\n\n            book.chapters = sortBy(book.chapters, (c) => c.chapter.number);\n            commentary.books.push(book);\n        }\n\n        if (parsed.profiles) {\n            for (let profile of parsed.profiles) {\n                const existing = commentary.profiles.find(\n                    (p) => p.id === profile.id\n                );\n                if (existing) {\n                    logger.warn(\n                        '[generate] Profile already exists in commentary!',\n                        file.name,\n                        profile.id\n                    );\n                    continue;\n                }\n\n                const datasetProfile: DatasetCommentaryProfile = {\n                    id: profile.id,\n                    subject: profile.subject,\n                    reference: profile.reference,\n                    content: profile.content,\n                };\n\n                commentary.profiles.push(datasetProfile);\n            }\n        }\n\n        commentary.books = sortBy(commentary.books, (b) => b.order);\n        commentary.profiles = sortBy(commentary.profiles, (p) => p.id);\n    }\n\n    function getBookNames(\n        file: InputFileBase,\n        metadata: MetadataBase,\n        id: string\n    ) {\n        let exactMatch = true;\n        let map = bookMap;\n        if (!map) {\n            map = defaultBookIdMap.get(metadata.language);\n\n            if (!map) {\n                if (!unknownLanguages.has(metadata.language)) {\n                    logger.warn(\n                        '[generate] File does not have a known language!',\n                        file.name,\n                        metadata.language\n                    );\n                    unknownLanguages.add(metadata.language);\n                }\n\n                logger.warn(\n                    '[generate] Using English book map for unknown language! This might result in outputting english book names.'\n                );\n                exactMatch = false;\n                map = defaultBookIdMap.get('en');\n            }\n        }\n\n        const bookName = map?.get(id);\n\n        if (!!map && !bookName) {\n            logger.warn(\n                '[generate] Book name not found for ID!',\n                file.name,\n                id\n            );\n            return null;\n        }\n\n        return {\n            exactMatch,\n            bookName,\n        };\n    }\n\n    return output;\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,yBAA2B;AAC3B,wBAA0B;AAmB1B,wBAIO;AACP,oBAA4C;AAC5C,mBAAuC;AACvC,0BAA4B;AAC5B,mCAAoC;AAMpC,gCAAiC;AACjC,iBAA0B;AA4FnB,SAAS,gBACZ,OACA,SAAoB,IAAI,WAAW,UAAU,GAC7C,SACa;AACb,QAAM,aAAS,sBAAU;AACzB,MAAI,SAAwB;AAAA,IACxB,cAAc,CAAC;AAAA,IACf,cAAc,CAAC;AAAA,EACnB;AAEA,MAAI,aAAa,IAAI,8BAAW;AAChC,MAAI,YAAY,IAAI,4BAAU,MAAM;AACpC,MAAI,cAAc,IAAI,gCAAY,MAAM;AACxC,MAAI,sBAAsB,IAAI,iDAAoB;AAClD,MAAI,mBAAmB,IAAI,2CAAiB,MAAM;AAElD,MAAI,qBAAqB,oBAAI,IAAgC;AAC7D,MAAI,qBAAqB,oBAAI,IAA+B;AAE5D,QAAM,mBAAmB,oBAAI,IAAY;AACzC,WAAS,QAAQ,OAAO;AACpB,QAAI;AACA,UAAIA;AAMJ,UAAI,KAAK,aAAa,QAAQ;AAC1B,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,OAAO;AAChC,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,QAAQ;AACjC,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,kBAAkB;AAC3C,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,0BAA0B;AACnD,QAAAA,UAAS;AAAA,MACb,OAAO;AACH,eAAO;AAAA,UACH;AAAA,UACA,KAAK;AAAA,QACT;AACA;AAAA,MACJ;AAEA,YAAM,SAASA,QAAO,MAAM,KAAK,OAAO;AAExC,UACI,mBAAmB,UACnB,OAAO,iBACP,KAAK,MACP;AACE,cAAM,WAAY,OAAO,gBACrB,OAAO,iBAAiB,CAAC;AAC7B,iBAAS,KAAK,IAAI,IAAI,OAAO;AAAA,MACjC;AAEA,UAAI,OAAO,SAAS,QAAQ;AACxB,2BAAmB,MAA8B,MAAM;AAAA,MAC3D,OAAO;AACH,0BAAkB,MAA6B,MAAM;AAAA,MACzD;AAAA,IACJ,SAAS,KAAK;AACV,aAAO;AAAA,QACH,2CAA2C,KAAK,IAAI;AAAA,QACpD;AAAA,MACJ;AAAA,IACJ;AAAA,EACJ;AAEA,WAAS,mBAAmB,MAA4B,QAAmB;AACvE,UAAM,KAAK,OAAO;AAElB,QAAI,CAAC,IAAI;AACL,aAAO;AAAA,QACH;AAAA,QACA,KAAK;AAAA,QACL;AAAA,MACJ;AACA;AAAA,IACJ;AAEA,UAAM,QAAQ,+BAAa,IAAI,EAAE;AAEjC,QAAI,OAAO,UAAU,UAAU;AAC3B,aAAO,KAAK,2CAA2C,EAAE;AACzD;AAAA,IACJ;AAEA,UAAM,eAAe,iCAAe,IAAI,EAAE;AAE1C,UAAM,WAAW,aAAa,MAAM,KAAK,UAAU,EAAE;AAErD,QAAI,cAAc,mBAAmB,IAAI,KAAK,SAAS,EAAE;AAEzD,QAAI,CAAC,aAAa;AACd,oBAAc;AAAA,QACV,OAAG,oBAAK,KAAK,UAAU,WAAW;AAAA,QAClC,eAAe,KAAK,SAAS;AAAA,QAC7B,OAAO,CAAC;AAAA,MACZ;AACA,aAAO,aAAa,KAAK,WAAW;AACpC,yBAAmB,IAAI,KAAK,SAAS,IAAI,WAAW;AAAA,IACxD;AAEA,UAAM,OACF,OAAO,WACN,UAAU,aAAa,UAAU,UAAU,aAAa,SACzD,OAAO,SACP,UAAU,UAAU;AAExB,QAAI,CAAC,MAAM;AACP,aAAO,KAAK,yCAAyC,KAAK,MAAM,EAAE;AAClE;AAAA,IACJ;AAEA,UAAM,cACD,UAAU,aAAa,UAAU,UAAU,aAAa,SACzD,OAAO,UACP,OAAO,SACP,UAAU,UAAU,cACpB;AAEJ,UAAM,OAA+B;AAAA,MACjC;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,OAAO,SAAS;AAAA,MACvB;AAAA,MACA,UAAU,CAAC;AAAA,IACf;AAEA,QAAI,cAAc;AACd,WAAK,eAAe;AAAA,IACxB;AAEA,aAAS,WAAW,OAAO,SAAS;AAChC,UAAI,QAAQ,SAAS,WAAW;AAC5B,aAAK,SAAS,KAAK;AAAA,UACf,SAAS;AAAA,YACL,QAAQ,QAAQ;AAAA,YAChB,SAAS,QAAQ;AAAA,YACjB,WAAW,QAAQ;AAAA,UACvB;AAAA,UACA,2BAAuB;AAAA,YACnB,YAAY;AAAA,YACZ;AAAA,YACA,QAAQ;AAAA,UACZ;AAAA,QACJ,CAAC;AAAA,MACL;AAAA,IACJ;AAEA,SAAK,eAAW,sBAAO,KAAK,UAAU,CAAC,MAAM,EAAE,QAAQ,MAAM;AAE7D,UAAM,YAAQ,6BAAc,YAAY,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK;AACnE,gBAAY,MAAM,OAAO,OAAO,GAAG,IAAI;AAAA,EAC3C;AAEA,WAAS,kBACL,MACA,QACF;AACE,QAAI,aAAa,mBAAmB,IAAI,KAAK,SAAS,EAAE;AAExD,QAAI,CAAC,YAAY;AACb,mBAAa;AAAA,QACT,OAAG,oBAAK,KAAK,UAAU,WAAW;AAAA,QAClC,eAAe,KAAK,SAAS;AAAA,QAC7B,OAAO,CAAC;AAAA,QACR,UAAU,CAAC;AAAA,MACf;AACA,aAAO,aAAa,KAAK,UAAU;AACnC,yBAAmB,IAAI,KAAK,SAAS,IAAI,UAAU;AAAA,IACvD;AAEA,aAAS,cAAc,OAAO,OAAO;AACjC,UAAI,OAAO,WAAW,MAAM,KAAK,CAAC,MAAM,EAAE,OAAO,WAAW,IAAI;AAChE,UAAI,QAAQ,KAAK,gBAAgB,WAAW,cAAc;AACtD,eAAO;AAAA,UACH;AAAA,UACA,KAAK;AAAA,UACL,WAAW;AAAA,QACf;AACA;AAAA,MACJ;AAEA,UAAI,CAAC,MAAM;AACP,cAAM,OAAO,aAAa,MAAM,KAAK,UAAU,WAAW,IAAI;AAC9D,eAAO;AAAA,UACH,IAAI,WAAW;AAAA,UACf,OAAO,+BAAa,IAAI,WAAW,IAAI,KAAK;AAAA,UAC5C,YAAY,MAAM,UAAU,cAAc,WAAW;AAAA,UACrD,MAAM,MAAM,UAAU,cAAc,WAAW;AAAA,UAC/C,UAAU,CAAC;AAAA,QACf;AAAA,MACJ;AAEA,UAAI,WAAW,cAAc;AACzB,aAAK,eAAe,WAAW;AAAA,MACnC;AAEA,UAAI,WAAW,uBAAuB,CAAC,KAAK,qBAAqB;AAC7D,aAAK,sBAAsB,WAAW;AAAA,MAC1C;AAEA,eAAS,WAAW,WAAW,UAAU;AACrC,YAAI,OAA8B;AAAA,UAC9B,QAAQ,QAAQ;AAAA,UAChB,SAAS,QAAQ;AAAA,QACrB;AAEA,YAAI,QAAQ,cAAc;AACtB,eAAK,eAAe,QAAQ;AAAA,QAChC;AAEA,aAAK,SAAS,KAAK;AAAA,UACf,SAAS;AAAA,QACb,CAAC;AAAA,MACL;AAEA,WAAK,eAAW,sBAAO,KAAK,UAAU,CAAC,MAAM,EAAE,QAAQ,MAAM;AAC7D,iBAAW,MAAM,KAAK,IAAI;AAAA,IAC9B;AAEA,QAAI,OAAO,UAAU;AACjB,eAAS,WAAW,OAAO,UAAU;AACjC,cAAM,WAAW,WAAW,SAAS;AAAA,UACjC,CAAC,MAAM,EAAE,OAAO,QAAQ;AAAA,QAC5B;AACA,YAAI,UAAU;AACV,iBAAO;AAAA,YACH;AAAA,YACA,KAAK;AAAA,YACL,QAAQ;AAAA,UACZ;AACA;AAAA,QACJ;AAEA,cAAM,iBAA2C;AAAA,UAC7C,IAAI,QAAQ;AAAA,UACZ,SAAS,QAAQ;AAAA,UACjB,WAAW,QAAQ;AAAA,UACnB,SAAS,QAAQ;AAAA,QACrB;AAEA,mBAAW,SAAS,KAAK,cAAc;AAAA,MAC3C;AAAA,IACJ;AAEA,eAAW,YAAQ,sBAAO,WAAW,OAAO,CAAC,MAAM,EAAE,KAAK;AAC1D,eAAW,eAAW,sBAAO,WAAW,UAAU,CAAC,MAAM,EAAE,EAAE;AAAA,EACjE;AAEA,WAAS,aACL,MACA,UACA,IACF;AACE,QAAI,aAAa;AACjB,QAAI,MAAM;AACV,QAAI,CAAC,KAAK;AACN,YAAM,kBAAAC,UAAiB,IAAI,SAAS,QAAQ;AAE5C,UAAI,CAAC,KAAK;AACN,YAAI,CAAC,iBAAiB,IAAI,SAAS,QAAQ,GAAG;AAC1C,iBAAO;AAAA,YACH;AAAA,YACA,KAAK;AAAA,YACL,SAAS;AAAA,UACb;AACA,2BAAiB,IAAI,SAAS,QAAQ;AAAA,QAC1C;AAEA,eAAO;AAAA,UACH;AAAA,QACJ;AACA,qBAAa;AACb,cAAM,kBAAAA,UAAiB,IAAI,IAAI;AAAA,MACnC;AAAA,IACJ;AAEA,UAAM,WAAW,KAAK,IAAI,EAAE;AAE5B,QAAI,CAAC,CAAC,OAAO,CAAC,UAAU;AACpB,aAAO;AAAA,QACH;AAAA,QACA,KAAK;AAAA,QACL;AAAA,MACJ;AACA,aAAO;AAAA,IACX;AAEA,WAAO;AAAA,MACH;AAAA,MACA;AAAA,IACJ;AAAA,EACJ;AAEA,SAAO;AACX;",
   "names": ["parser", "defaultBookIdMap"]
 }

package/dist/cjs/parser/codex-parser.cjs

@@ -39,7 +39,10 @@ const verseSchema = import_zod.z.object({
   id: import_zod.z.string(),
   bookCode: import_zod.z.string().optional(),
   chapter: import_zod.z.number().optional(),
-  verse: import_zod.z.number().optional()
+  verse: import_zod.z.number().optional(),
+  data: import_zod.z.object({
+    globalReferences: import_zod.z.array(import_zod.z.string()).optional()
+  }).optional()
 });
 const paratextSchema = import_zod.z.object({
   type: import_zod.z.literal("paratext"),
@@ -69,6 +72,10 @@ class CodexParser {
   //  */
   // preserveMarkdown: boolean = true;
   _noteCounter = 0;
+  _parser;
+  constructor(parser) {
+    this._parser = parser;
+  }
   /**
    * Parses the specified codex content.
    *
@@ -84,7 +91,7 @@ class CodexParser {
     };
     let chapters = /* @__PURE__ */ new Map();
     let lastChapter = null;
-    function addChapterContent(chapterNumber, content) {
+    function ensureChapter(chapterNumber) {
       let chapter = chapters.get(chapterNumber);
       if (!chapter) {
         chapter = {
@@ -96,8 +103,23 @@ class CodexParser {
         lastChapter = chapter;
         chapters.set(chapterNumber, chapter);
       }
+      return chapter;
+    }
+    function getDocumentContent(doc) {
+      let content = "";
+      for (let element of doc.childNodes) {
+        content += element.textContent;
+      }
+      return content;
+    }
+    function addChapterContent(chapterNumber, content) {
+      const chapter = ensureChapter(chapterNumber);
       chapter.content.push(...content);
     }
+    function addChapterFootnote(chapterNumber, footnote) {
+      const chapter = ensureChapter(chapterNumber);
+      chapter.footnotes.push(footnote);
+    }
     function addReference(ref2, content) {
       addChapterContent(ref2.chapter, [
         {
@@ -119,8 +141,12 @@ class CodexParser {
     function addLineBreaks(lines2) {
       return lines2.reduce(
         (prev, current) => {
-          if (prev.length === 0) return [current];
-          else
+          if (typeof current === "string" && current.trim() === "") {
+            return prev;
+          }
+          if (prev.length === 0) {
+            return [current];
+          } else {
             return [
               ...prev,
               {
@@ -128,6 +154,7 @@ class CodexParser {
               },
               current
             ];
+          }
         },
         []
       );
@@ -172,6 +199,15 @@ class CodexParser {
                   "Could not find verse reference in metadata."
                 );
               }
+            } else if (metadata.data?.globalReferences?.length) {
+              reference = (0, import_utils.parseVerseReference)(
+                metadata.data.globalReferences[0]
+              );
+              if (!reference) {
+                throw new Error(
+                  "Could not find verse reference in globalReferences metadata."
+                );
+              }
             } else {
               throw new Error(
                 "Could not find verse reference in metadata."
@@ -184,13 +220,49 @@ class CodexParser {
           }
           previousReference = reference;
           if (!root.id) root.id = reference.book;
-          const content = stripHTML(cell.value);
-          const newLines = content.split("\n");
-          lines.push(...newLines);
+          const doc = this._parser.parseFromString(
+            cell.value,
+            "text/html"
+          );
+          for (let element of doc.children) {
+            const allFootnotes = element.querySelectorAll(
+              "sup.footnote-marker[data-footnote]"
+            );
+            for (const footnote of allFootnotes) {
+              const footnoteData = footnote.getAttribute("data-footnote");
+              if (footnoteData) {
+                const footnoteDoc = this._parser.parseFromString(
+                  footnoteData,
+                  "text/html"
+                );
+                const footnoteText = getDocumentContent(footnoteDoc);
+                if (footnoteText) {
+                  addChapterFootnote(reference.chapter, {
+                    noteId: this._noteCounter++,
+                    text: footnoteText,
+                    caller: null
+                  });
+                }
+              }
+              footnote.remove();
+            }
+          }
+          const content = getDocumentContent(doc);
+          if (content) {
+            const newLines = content.split("\n");
+            lines.push(...newLines);
+          }
         } else if (metadata.type === "paratext") {
           const reference = (0, import_utils.parseVerseReference)(`${metadata.id}:1`);
           if (reference) {
-            const content = stripHTML(cell.value);
+            const doc = this._parser.parseFromString(
+              cell.value,
+              "text/html"
+            );
+            const content = getDocumentContent(doc);
+            if (!content) {
+              continue;
+            }
             const lines2 = content.split("\n").reduce((prev, current) => {
               if (prev.length === 0)
                 return [toHeading(current)];
@@ -207,8 +279,12 @@ class CodexParser {
               (line) => addChapterContent(reference.chapter, [line])
             );
           } else {
-            const content = stripHTML(cell.value);
-            if (!cell.metadata) {
+            const doc = this._parser.parseFromString(
+              cell.value,
+              "text/html"
+            );
+            const content = getDocumentContent(doc);
+            if (content && !cell.metadata) {
               if (lastChapter) {
                 lastChapter.footnotes.push({
                   noteId: this._noteCounter++,

package/dist/cjs/parser/codex-parser.cjs.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../parser/codex-parser.ts"],
-  "sourcesContent": ["import { z } from 'zod';\nimport {\n    Chapter,\n    ChapterContent,\n    ParseTree,\n    Verse,\n    VerseContent,\n} from './types.js';\nimport { getBookId, parseVerseReference, VerseRef } from '../utils.js';\nimport { ref } from 'process';\n\nexport const chapterHeadingSchema = z.object({\n    type: z.literal('chapter-heading'),\n    data: z.object({\n        chapter: z.coerce.number().int(),\n    }),\n});\n\nexport const verseSchema = z.object({\n    type: z.literal('text'),\n    id: z.string(),\n    bookCode: z.string().optional(),\n    chapter: z.number().optional(),\n    verse: z.number().optional(),\n});\n\nexport const paratextSchema = z.object({\n    type: z.literal('paratext'),\n    id: z.string(),\n});\n\nexport const metadataSchema = z.discriminatedUnion('type', [\n    chapterHeadingSchema,\n    verseSchema,\n    paratextSchema,\n]);\n\nexport const codexSchema = z.object({\n    cells: z.array(\n        z.object({\n            kind: z.number(),\n            languageId: z.string(),\n            value: z.string(),\n            metadata: z\n                .object({\n                    type: z.string(),\n                })\n                .passthrough()\n                .nullable()\n                .optional(),\n        })\n    ),\n});\n\n/**\n * Defines a parser for codex files.\n */\nexport class CodexParser {\n    // TODO:\n    // /**\n    //  * Whether to preserve markdown in the parsed content.\n    //  */\n    // preserveMarkdown: boolean = true;\n\n    private _noteCounter: number = 0;\n\n    /**\n     * Parses the specified codex content.\n     *\n     * @param codex The codex content to parse.\n     * @returns The parse tree that was generated.\n     */\n    public parse(codex: string): ParseTree {\n        const json = JSON.parse(codex);\n        const data = codexSchema.parse(json);\n\n        let root: ParseTree = {\n            type: 'root',\n            content: [],\n        };\n\n        let chapters: Map<number, Chapter> = new Map();\n        let lastChapter: Chapter | null = null;\n\n        function addChapterContent(\n            chapterNumber: number,\n            content: ChapterContent[]\n        ) {\n            let chapter = chapters.get(chapterNumber);\n            if (!chapter) {\n                chapter = {\n                    type: 'chapter',\n                    number: chapterNumber,\n                    content: [],\n                    footnotes: [],\n                };\n                lastChapter = chapter;\n                chapters.set(chapterNumber, chapter);\n            }\n\n            chapter.content.push(...content);\n        }\n\n        function addReference(ref: VerseRef, content: Verse['content']) {\n            addChapterContent(ref.chapter, [\n                {\n                    type: 'verse',\n                    number: ref.verse,\n                    content: content,\n                },\n            ]);\n        }\n\n        function stripHTML(value: string) {\n            return value.replace(/<[^>]*>/g, '');\n        }\n\n        function toHeading(content: string) {\n            return {\n                type: 'heading',\n                content: [content],\n            } satisfies ChapterContent;\n        }\n\n        function addLineBreaks(lines: Verse['content']) {\n            return lines.reduce(\n                (prev, current) => {\n                    if (prev.length === 0) return [current];\n                    else\n                        return [\n                            ...prev,\n                            {\n                                lineBreak: true,\n                            },\n                            current,\n                        ] satisfies Verse['content'];\n                },\n                [] as Verse['content']\n            );\n        }\n\n        let previousReference: VerseRef | null = null;\n        let lines: Verse['content'] = [];\n\n        for (let cell of data.cells) {\n            if (cell.languageId === 'html' && cell.value) {\n                if (!cell.metadata) continue; // ignore html cells without metadata\n\n                const metadataResult = metadataSchema.safeParse(cell.metadata);\n\n                if (!metadataResult.success) continue; // ignore cells with invalid/unknown metadata\n                const metadata = metadataResult.data;\n                if (metadata.type === 'text') {\n                    let reference = parseVerseReference(metadata.id);\n                    if (!reference) {\n                        if (metadata.bookCode && metadata.chapter) {\n                            const bookId = getBookId(metadata.bookCode);\n                            if (!bookId) {\n                                throw new Error(\n                                    'Could not find book ID for code: ' +\n                                        metadata.bookCode\n                                );\n                            }\n\n                            if (metadata.verse) {\n                                reference = {\n                                    book: bookId,\n                                    chapter: metadata.chapter,\n                                    verse: metadata.verse,\n                                };\n                            } else if (previousReference?.verse) {\n                                if (\n                                    previousReference.chapter !==\n                                        metadata.chapter ||\n                                    previousReference.book !== bookId\n                                ) {\n                                    throw new Error(\n                                        'Cannot infer verse number for non-consecutive verse reference.'\n                                    );\n                                }\n\n                                // Sometimes codex doesn't include verse numbers for every cell, so we infer it from the previous reference\n                                reference = {\n                                    book: bookId,\n                                    chapter: metadata.chapter,\n                                    verse: previousReference.verse,\n                                };\n                            } else {\n                                throw new Error(\n                                    'Could not find verse reference in metadata.'\n                                );\n                            }\n                        } else {\n                            throw new Error(\n                                'Could not find verse reference in metadata.'\n                            );\n                        }\n                    }\n\n                    if (\n                        previousReference &&\n                        (reference.book !== previousReference.book ||\n                            reference.chapter !== previousReference.chapter ||\n                            reference.verse !== previousReference.verse)\n                    ) {\n                        addReference(previousReference, addLineBreaks(lines));\n                        lines = [];\n                    }\n\n                    previousReference = reference;\n\n                    if (!root.id) root.id = reference.book; // set book id if not already set\n\n                    const content = stripHTML(cell.value);\n\n                    // add line breaks in heading if there are multiple lines\n                    const newLines = content.split('\\n');\n\n                    lines.push(...newLines);\n                } else if (metadata.type === 'paratext') {\n                    const reference = parseVerseReference(`${metadata.id}:1`); // chapter headings do not have a verse reference always default to 1\n                    if (reference) {\n                        const content = stripHTML(cell.value);\n\n                        // add line breaks in heading if there are multiple lines\n                        const lines = content\n                            .split('\\n')\n                            .reduce<ChapterContent[]>((prev, current) => {\n                                if (prev.length === 0)\n                                    return [toHeading(current)];\n                                else\n                                    return [\n                                        ...prev,\n                                        {\n                                            type: 'line_break',\n                                        },\n                                        toHeading(current),\n                                    ] satisfies ChapterContent[];\n                            }, []);\n\n                        lines.forEach((line) =>\n                            addChapterContent(reference.chapter, [line])\n                        );\n                    } else {\n                        // parse paratext that isn't a chapter reference as footnote\n                        const content = stripHTML(cell.value);\n                        if (!cell.metadata) {\n                            if (lastChapter) {\n                                (lastChapter as Chapter).footnotes.push({\n                                    noteId: this._noteCounter++,\n                                    text: content,\n                                    caller: null,\n                                });\n                            }\n                        }\n                    }\n                }\n            }\n        }\n\n        if (previousReference) {\n            addReference(previousReference, addLineBreaks(lines));\n        }\n\n        for (let chapter of chapters.values()) {\n            root.content.push(chapter);\n        }\n\n        return root;\n    }\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBAAkB;AAQlB,mBAAyD;AAGlD,MAAM,uBAAuB,aAAE,OAAO;AAAA,EACzC,MAAM,aAAE,QAAQ,iBAAiB;AAAA,EACjC,MAAM,aAAE,OAAO;AAAA,IACX,SAAS,aAAE,OAAO,OAAO,EAAE,IAAI;AAAA,EACnC,CAAC;AACL,CAAC;AAEM,MAAM,cAAc,aAAE,OAAO;AAAA,EAChC,MAAM,aAAE,QAAQ,MAAM;AAAA,EACtB,IAAI,aAAE,OAAO;AAAA,EACb,UAAU,aAAE,OAAO,EAAE,SAAS;AAAA,EAC9B,SAAS,aAAE,OAAO,EAAE,SAAS;AAAA,EAC7B,OAAO,aAAE,OAAO,EAAE,SAAS;AAC/B,CAAC;AAEM,MAAM,iBAAiB,aAAE,OAAO;AAAA,EACnC,MAAM,aAAE,QAAQ,UAAU;AAAA,EAC1B,IAAI,aAAE,OAAO;AACjB,CAAC;AAEM,MAAM,iBAAiB,aAAE,mBAAmB,QAAQ;AAAA,EACvD;AAAA,EACA;AAAA,EACA;AACJ,CAAC;AAEM,MAAM,cAAc,aAAE,OAAO;AAAA,EAChC,OAAO,aAAE;AAAA,IACL,aAAE,OAAO;AAAA,MACL,MAAM,aAAE,OAAO;AAAA,MACf,YAAY,aAAE,OAAO;AAAA,MACrB,OAAO,aAAE,OAAO;AAAA,MAChB,UAAU,aACL,OAAO;AAAA,QACJ,MAAM,aAAE,OAAO;AAAA,MACnB,CAAC,EACA,YAAY,EACZ,SAAS,EACT,SAAS;AAAA,IAClB,CAAC;AAAA,EACL;AACJ,CAAC;AAKM,MAAM,YAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOb,eAAuB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQxB,MAAM,OAA0B;AACnC,UAAM,OAAO,KAAK,MAAM,KAAK;AAC7B,UAAM,OAAO,YAAY,MAAM,IAAI;AAEnC,QAAI,OAAkB;AAAA,MAClB,MAAM;AAAA,MACN,SAAS,CAAC;AAAA,IACd;AAEA,QAAI,WAAiC,oBAAI,IAAI;AAC7C,QAAI,cAA8B;AAElC,aAAS,kBACL,eACA,SACF;AACE,UAAI,UAAU,SAAS,IAAI,aAAa;AACxC,UAAI,CAAC,SAAS;AACV,kBAAU;AAAA,UACN,MAAM;AAAA,UACN,QAAQ;AAAA,UACR,SAAS,CAAC;AAAA,UACV,WAAW,CAAC;AAAA,QAChB;AACA,sBAAc;AACd,iBAAS,IAAI,eAAe,OAAO;AAAA,MACvC;AAEA,cAAQ,QAAQ,KAAK,GAAG,OAAO;AAAA,IACnC;AAEA,aAAS,aAAaA,MAAe,SAA2B;AAC5D,wBAAkBA,KAAI,SAAS;AAAA,QAC3B;AAAA,UACI,MAAM;AAAA,UACN,QAAQA,KAAI;AAAA,UACZ;AAAA,QACJ;AAAA,MACJ,CAAC;AAAA,IACL;AAEA,aAAS,UAAU,OAAe;AAC9B,aAAO,MAAM,QAAQ,YAAY,EAAE;AAAA,IACvC;AAEA,aAAS,UAAU,SAAiB;AAChC,aAAO;AAAA,QACH,MAAM;AAAA,QACN,SAAS,CAAC,OAAO;AAAA,MACrB;AAAA,IACJ;AAEA,aAAS,cAAcC,QAAyB;AAC5C,aAAOA,OAAM;AAAA,QACT,CAAC,MAAM,YAAY;AACf,cAAI,KAAK,WAAW,EAAG,QAAO,CAAC,OAAO;AAAA;AAElC,mBAAO;AAAA,cACH,GAAG;AAAA,cACH;AAAA,gBACI,WAAW;AAAA,cACf;AAAA,cACA;AAAA,YACJ;AAAA,QACR;AAAA,QACA,CAAC;AAAA,MACL;AAAA,IACJ;AAEA,QAAI,oBAAqC;AACzC,QAAI,QAA0B,CAAC;AAE/B,aAAS,QAAQ,KAAK,OAAO;AACzB,UAAI,KAAK,eAAe,UAAU,KAAK,OAAO;AAC1C,YAAI,CAAC,KAAK,SAAU;AAEpB,cAAM,iBAAiB,eAAe,UAAU,KAAK,QAAQ;AAE7D,YAAI,CAAC,eAAe,QAAS;AAC7B,cAAM,WAAW,eAAe;AAChC,YAAI,SAAS,SAAS,QAAQ;AAC1B,cAAI,gBAAY,kCAAoB,SAAS,EAAE;AAC/C,cAAI,CAAC,WAAW;AACZ,gBAAI,SAAS,YAAY,SAAS,SAAS;AACvC,oBAAM,aAAS,wBAAU,SAAS,QAAQ;AAC1C,kBAAI,CAAC,QAAQ;AACT,sBAAM,IAAI;AAAA,kBACN,sCACI,SAAS;AAAA,gBACjB;AAAA,cACJ;AAEA,kBAAI,SAAS,OAAO;AAChB,4BAAY;AAAA,kBACR,MAAM;AAAA,kBACN,SAAS,SAAS;AAAA,kBAClB,OAAO,SAAS;AAAA,gBACpB;AAAA,cACJ,WAAW,mBAAmB,OAAO;AACjC,oBACI,kBAAkB,YACd,SAAS,WACb,kBAAkB,SAAS,QAC7B;AACE,wBAAM,IAAI;AAAA,oBACN;AAAA,kBACJ;AAAA,gBACJ;AAGA,4BAAY;AAAA,kBACR,MAAM;AAAA,kBACN,SAAS,SAAS;AAAA,kBAClB,OAAO,kBAAkB;AAAA,gBAC7B;AAAA,cACJ,OAAO;AACH,sBAAM,IAAI;AAAA,kBACN;AAAA,gBACJ;AAAA,cACJ;AAAA,YACJ,OAAO;AACH,oBAAM,IAAI;AAAA,gBACN;AAAA,cACJ;AAAA,YACJ;AAAA,UACJ;AAEA,cACI,sBACC,UAAU,SAAS,kBAAkB,QAClC,UAAU,YAAY,kBAAkB,WACxC,UAAU,UAAU,kBAAkB,QAC5C;AACE,yBAAa,mBAAmB,cAAc,KAAK,CAAC;AACpD,oBAAQ,CAAC;AAAA,UACb;AAEA,8BAAoB;AAEpB,cAAI,CAAC,KAAK,GAAI,MAAK,KAAK,UAAU;AAElC,gBAAM,UAAU,UAAU,KAAK,KAAK;AAGpC,gBAAM,WAAW,QAAQ,MAAM,IAAI;AAEnC,gBAAM,KAAK,GAAG,QAAQ;AAAA,QAC1B,WAAW,SAAS,SAAS,YAAY;AACrC,gBAAM,gBAAY,kCAAoB,GAAG,SAAS,EAAE,IAAI;AACxD,cAAI,WAAW;AACX,kBAAM,UAAU,UAAU,KAAK,KAAK;AAGpC,kBAAMA,SAAQ,QACT,MAAM,IAAI,EACV,OAAyB,CAAC,MAAM,YAAY;AACzC,kBAAI,KAAK,WAAW;AAChB,uBAAO,CAAC,UAAU,OAAO,CAAC;AAAA;AAE1B,uBAAO;AAAA,kBACH,GAAG;AAAA,kBACH;AAAA,oBACI,MAAM;AAAA,kBACV;AAAA,kBACA,UAAU,OAAO;AAAA,gBACrB;AAAA,YACR,GAAG,CAAC,CAAC;AAET,YAAAA,OAAM;AAAA,cAAQ,CAAC,SACX,kBAAkB,UAAU,SAAS,CAAC,IAAI,CAAC;AAAA,YAC/C;AAAA,UACJ,OAAO;AAEH,kBAAM,UAAU,UAAU,KAAK,KAAK;AACpC,gBAAI,CAAC,KAAK,UAAU;AAChB,kBAAI,aAAa;AACb,gBAAC,YAAwB,UAAU,KAAK;AAAA,kBACpC,QAAQ,KAAK;AAAA,kBACb,MAAM;AAAA,kBACN,QAAQ;AAAA,gBACZ,CAAC;AAAA,cACL;AAAA,YACJ;AAAA,UACJ;AAAA,QACJ;AAAA,MACJ;AAAA,IACJ;AAEA,QAAI,mBAAmB;AACnB,mBAAa,mBAAmB,cAAc,KAAK,CAAC;AAAA,IACxD;AAEA,aAAS,WAAW,SAAS,OAAO,GAAG;AACnC,WAAK,QAAQ,KAAK,OAAO;AAAA,IAC7B;AAEA,WAAO;AAAA,EACX;AACJ;",
+  "sourcesContent": ["import { z } from 'zod';\nimport {\n    Chapter,\n    ChapterContent,\n    ParseTree,\n    Verse,\n    VerseContent,\n} from './types.js';\nimport { getBookId, parseVerseReference, VerseRef } from '../utils.js';\nimport { ref } from 'process';\n\nexport const chapterHeadingSchema = z.object({\n    type: z.literal('chapter-heading'),\n    data: z.object({\n        chapter: z.coerce.number().int(),\n    }),\n});\n\nexport const verseSchema = z.object({\n    type: z.literal('text'),\n    id: z.string(),\n    bookCode: z.string().optional(),\n    chapter: z.number().optional(),\n    verse: z.number().optional(),\n    data: z\n        .object({\n            globalReferences: z.array(z.string()).optional(),\n        })\n        .optional(),\n});\n\nexport const paratextSchema = z.object({\n    type: z.literal('paratext'),\n    id: z.string(),\n});\n\nexport const metadataSchema = z.discriminatedUnion('type', [\n    chapterHeadingSchema,\n    verseSchema,\n    paratextSchema,\n]);\n\nexport const codexSchema = z.object({\n    cells: z.array(\n        z.object({\n            kind: z.number(),\n            languageId: z.string(),\n            value: z.string(),\n            metadata: z\n                .object({\n                    type: z.string(),\n                })\n                .passthrough()\n                .nullable()\n                .optional(),\n        })\n    ),\n});\n\n/**\n * Defines a parser for codex files.\n */\nexport class CodexParser {\n    // TODO:\n    // /**\n    //  * Whether to preserve markdown in the parsed content.\n    //  */\n    // preserveMarkdown: boolean = true;\n\n    private _noteCounter: number = 0;\n\n    private _parser: DOMParser;\n\n    constructor(parser: DOMParser) {\n        this._parser = parser;\n    }\n\n    /**\n     * Parses the specified codex content.\n     *\n     * @param codex The codex content to parse.\n     * @returns The parse tree that was generated.\n     */\n    public parse(codex: string): ParseTree {\n        const json = JSON.parse(codex);\n        const data = codexSchema.parse(json);\n\n        let root: ParseTree = {\n            type: 'root',\n            content: [],\n        };\n\n        let chapters: Map<number, Chapter> = new Map();\n        let lastChapter: Chapter | null = null;\n\n        function ensureChapter(chapterNumber: number) {\n            let chapter = chapters.get(chapterNumber);\n            if (!chapter) {\n                chapter = {\n                    type: 'chapter',\n                    number: chapterNumber,\n                    content: [],\n                    footnotes: [],\n                };\n                lastChapter = chapter;\n                chapters.set(chapterNumber, chapter);\n            }\n\n            return chapter;\n        }\n\n        function getDocumentContent(doc: Document): string {\n            let content = '';\n            for (let element of doc.childNodes) {\n                content += element.textContent;\n            }\n            return content;\n        }\n\n        function addChapterContent(\n            chapterNumber: number,\n            content: ChapterContent[]\n        ) {\n            const chapter = ensureChapter(chapterNumber);\n            chapter.content.push(...content);\n        }\n\n        function addChapterFootnote(\n            chapterNumber: number,\n            footnote: Chapter['footnotes'][0]\n        ) {\n            const chapter = ensureChapter(chapterNumber);\n            chapter.footnotes.push(footnote);\n        }\n\n        function addReference(ref: VerseRef, content: Verse['content']) {\n            addChapterContent(ref.chapter, [\n                {\n                    type: 'verse',\n                    number: ref.verse,\n                    content: content,\n                },\n            ]);\n        }\n\n        function stripHTML(value: string) {\n            return value.replace(/<[^>]*>/g, '');\n        }\n\n        function toHeading(content: string) {\n            return {\n                type: 'heading',\n                content: [content],\n            } satisfies ChapterContent;\n        }\n\n        function addLineBreaks(lines: Verse['content']) {\n            return lines.reduce(\n                (prev, current) => {\n                    if (typeof current === 'string' && current.trim() === '') {\n                        return prev; // skip empty lines\n                    }\n                    if (prev.length === 0) {\n                        return [current];\n                    } else {\n                        return [\n                            ...prev,\n                            {\n                                lineBreak: true,\n                            },\n                            current,\n                        ] satisfies Verse['content'];\n                    }\n                },\n                [] as Verse['content']\n            );\n        }\n\n        let previousReference: VerseRef | null = null;\n        let lines: Verse['content'] = [];\n\n        for (let cell of data.cells) {\n            if (cell.languageId === 'html' && cell.value) {\n                if (!cell.metadata) continue; // ignore html cells without metadata\n\n                const metadataResult = metadataSchema.safeParse(cell.metadata);\n\n                if (!metadataResult.success) continue; // ignore cells with invalid/unknown metadata\n                const metadata = metadataResult.data;\n                if (metadata.type === 'text') {\n                    let reference = parseVerseReference(metadata.id);\n                    if (!reference) {\n                        if (metadata.bookCode && metadata.chapter) {\n                            const bookId = getBookId(metadata.bookCode);\n                            if (!bookId) {\n                                throw new Error(\n                                    'Could not find book ID for code: ' +\n                                        metadata.bookCode\n                                );\n                            }\n\n                            if (metadata.verse) {\n                                reference = {\n                                    book: bookId,\n                                    chapter: metadata.chapter,\n                                    verse: metadata.verse,\n                                };\n                            } else if (previousReference?.verse) {\n                                if (\n                                    previousReference.chapter !==\n                                        metadata.chapter ||\n                                    previousReference.book !== bookId\n                                ) {\n                                    throw new Error(\n                                        'Cannot infer verse number for non-consecutive verse reference.'\n                                    );\n                                }\n\n                                // Sometimes codex doesn't include verse numbers for every cell, so we infer it from the previous reference\n                                reference = {\n                                    book: bookId,\n                                    chapter: metadata.chapter,\n                                    verse: previousReference.verse,\n                                };\n                            } else {\n                                throw new Error(\n                                    'Could not find verse reference in metadata.'\n                                );\n                            }\n                        } else if (metadata.data?.globalReferences?.length) {\n                            reference = parseVerseReference(\n                                metadata.data.globalReferences[0]\n                            );\n                            if (!reference) {\n                                throw new Error(\n                                    'Could not find verse reference in globalReferences metadata.'\n                                );\n                            }\n                        } else {\n                            throw new Error(\n                                'Could not find verse reference in metadata.'\n                            );\n                        }\n                    }\n\n                    if (\n                        previousReference &&\n                        (reference.book !== previousReference.book ||\n                            reference.chapter !== previousReference.chapter ||\n                            reference.verse !== previousReference.verse)\n                    ) {\n                        addReference(previousReference, addLineBreaks(lines));\n                        lines = [];\n                    }\n\n                    previousReference = reference;\n\n                    if (!root.id) root.id = reference.book; // set book id if not already set\n\n                    const doc = this._parser.parseFromString(\n                        cell.value,\n                        'text/html'\n                    );\n\n                    for (let element of doc.children) {\n                        const allFootnotes = element.querySelectorAll(\n                            'sup.footnote-marker[data-footnote]'\n                        );\n                        for (const footnote of allFootnotes) {\n                            const footnoteData =\n                                footnote.getAttribute('data-footnote');\n                            if (footnoteData) {\n                                const footnoteDoc =\n                                    this._parser.parseFromString(\n                                        footnoteData,\n                                        'text/html'\n                                    );\n                                const footnoteText =\n                                    getDocumentContent(footnoteDoc);\n                                if (footnoteText) {\n                                    addChapterFootnote(reference.chapter, {\n                                        noteId: this._noteCounter++,\n                                        text: footnoteText,\n                                        caller: null,\n                                    });\n                                }\n                            }\n                            footnote.remove();\n                        }\n                    }\n\n                    const content = getDocumentContent(doc);\n\n                    if (content) {\n                        // add line breaks in heading if there are multiple lines\n                        const newLines = content.split('\\n');\n\n                        lines.push(...newLines);\n                    }\n                } else if (metadata.type === 'paratext') {\n                    const reference = parseVerseReference(`${metadata.id}:1`); // chapter headings do not have a verse reference always default to 1\n                    if (reference) {\n                        const doc = this._parser.parseFromString(\n                            cell.value,\n                            'text/html'\n                        );\n                        const content = getDocumentContent(doc);\n\n                        if (!content) {\n                            continue;\n                        }\n\n                        // add line breaks in heading if there are multiple lines\n                        const lines = content\n                            .split('\\n')\n                            .reduce<ChapterContent[]>((prev, current) => {\n                                if (prev.length === 0)\n                                    return [toHeading(current)];\n                                else\n                                    return [\n                                        ...prev,\n                                        {\n                                            type: 'line_break',\n                                        },\n                                        toHeading(current),\n                                    ] satisfies ChapterContent[];\n                            }, []);\n\n                        lines.forEach((line) =>\n                            addChapterContent(reference.chapter, [line])\n                        );\n                    } else {\n                        const doc = this._parser.parseFromString(\n                            cell.value,\n                            'text/html'\n                        );\n                        const content = getDocumentContent(doc);\n                        // parse paratext that isn't a chapter reference as footnote\n                        // const content = stripHTML(cell.value);\n                        if (content && !cell.metadata) {\n                            if (lastChapter) {\n                                (lastChapter as Chapter).footnotes.push({\n                                    noteId: this._noteCounter++,\n                                    text: content,\n                                    caller: null,\n                                });\n                            }\n                        }\n                    }\n                }\n            }\n        }\n\n        if (previousReference) {\n            addReference(previousReference, addLineBreaks(lines));\n        }\n\n        for (let chapter of chapters.values()) {\n            root.content.push(chapter);\n        }\n\n        return root;\n    }\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iBAAkB;AAQlB,mBAAyD;AAGlD,MAAM,uBAAuB,aAAE,OAAO;AAAA,EACzC,MAAM,aAAE,QAAQ,iBAAiB;AAAA,EACjC,MAAM,aAAE,OAAO;AAAA,IACX,SAAS,aAAE,OAAO,OAAO,EAAE,IAAI;AAAA,EACnC,CAAC;AACL,CAAC;AAEM,MAAM,cAAc,aAAE,OAAO;AAAA,EAChC,MAAM,aAAE,QAAQ,MAAM;AAAA,EACtB,IAAI,aAAE,OAAO;AAAA,EACb,UAAU,aAAE,OAAO,EAAE,SAAS;AAAA,EAC9B,SAAS,aAAE,OAAO,EAAE,SAAS;AAAA,EAC7B,OAAO,aAAE,OAAO,EAAE,SAAS;AAAA,EAC3B,MAAM,aACD,OAAO;AAAA,IACJ,kBAAkB,aAAE,MAAM,aAAE,OAAO,CAAC,EAAE,SAAS;AAAA,EACnD,CAAC,EACA,SAAS;AAClB,CAAC;AAEM,MAAM,iBAAiB,aAAE,OAAO;AAAA,EACnC,MAAM,aAAE,QAAQ,UAAU;AAAA,EAC1B,IAAI,aAAE,OAAO;AACjB,CAAC;AAEM,MAAM,iBAAiB,aAAE,mBAAmB,QAAQ;AAAA,EACvD;AAAA,EACA;AAAA,EACA;AACJ,CAAC;AAEM,MAAM,cAAc,aAAE,OAAO;AAAA,EAChC,OAAO,aAAE;AAAA,IACL,aAAE,OAAO;AAAA,MACL,MAAM,aAAE,OAAO;AAAA,MACf,YAAY,aAAE,OAAO;AAAA,MACrB,OAAO,aAAE,OAAO;AAAA,MAChB,UAAU,aACL,OAAO;AAAA,QACJ,MAAM,aAAE,OAAO;AAAA,MACnB,CAAC,EACA,YAAY,EACZ,SAAS,EACT,SAAS;AAAA,IAClB,CAAC;AAAA,EACL;AACJ,CAAC;AAKM,MAAM,YAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOb,eAAuB;AAAA,EAEvB;AAAA,EAER,YAAY,QAAmB;AAC3B,SAAK,UAAU;AAAA,EACnB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQO,MAAM,OAA0B;AACnC,UAAM,OAAO,KAAK,MAAM,KAAK;AAC7B,UAAM,OAAO,YAAY,MAAM,IAAI;AAEnC,QAAI,OAAkB;AAAA,MAClB,MAAM;AAAA,MACN,SAAS,CAAC;AAAA,IACd;AAEA,QAAI,WAAiC,oBAAI,IAAI;AAC7C,QAAI,cAA8B;AAElC,aAAS,cAAc,eAAuB;AAC1C,UAAI,UAAU,SAAS,IAAI,aAAa;AACxC,UAAI,CAAC,SAAS;AACV,kBAAU;AAAA,UACN,MAAM;AAAA,UACN,QAAQ;AAAA,UACR,SAAS,CAAC;AAAA,UACV,WAAW,CAAC;AAAA,QAChB;AACA,sBAAc;AACd,iBAAS,IAAI,eAAe,OAAO;AAAA,MACvC;AAEA,aAAO;AAAA,IACX;AAEA,aAAS,mBAAmB,KAAuB;AAC/C,UAAI,UAAU;AACd,eAAS,WAAW,IAAI,YAAY;AAChC,mBAAW,QAAQ;AAAA,MACvB;AACA,aAAO;AAAA,IACX;AAEA,aAAS,kBACL,eACA,SACF;AACE,YAAM,UAAU,cAAc,aAAa;AAC3C,cAAQ,QAAQ,KAAK,GAAG,OAAO;AAAA,IACnC;AAEA,aAAS,mBACL,eACA,UACF;AACE,YAAM,UAAU,cAAc,aAAa;AAC3C,cAAQ,UAAU,KAAK,QAAQ;AAAA,IACnC;AAEA,aAAS,aAAaA,MAAe,SAA2B;AAC5D,wBAAkBA,KAAI,SAAS;AAAA,QAC3B;AAAA,UACI,MAAM;AAAA,UACN,QAAQA,KAAI;AAAA,UACZ;AAAA,QACJ;AAAA,MACJ,CAAC;AAAA,IACL;AAEA,aAAS,UAAU,OAAe;AAC9B,aAAO,MAAM,QAAQ,YAAY,EAAE;AAAA,IACvC;AAEA,aAAS,UAAU,SAAiB;AAChC,aAAO;AAAA,QACH,MAAM;AAAA,QACN,SAAS,CAAC,OAAO;AAAA,MACrB;AAAA,IACJ;AAEA,aAAS,cAAcC,QAAyB;AAC5C,aAAOA,OAAM;AAAA,QACT,CAAC,MAAM,YAAY;AACf,cAAI,OAAO,YAAY,YAAY,QAAQ,KAAK,MAAM,IAAI;AACtD,mBAAO;AAAA,UACX;AACA,cAAI,KAAK,WAAW,GAAG;AACnB,mBAAO,CAAC,OAAO;AAAA,UACnB,OAAO;AACH,mBAAO;AAAA,cACH,GAAG;AAAA,cACH;AAAA,gBACI,WAAW;AAAA,cACf;AAAA,cACA;AAAA,YACJ;AAAA,UACJ;AAAA,QACJ;AAAA,QACA,CAAC;AAAA,MACL;AAAA,IACJ;AAEA,QAAI,oBAAqC;AACzC,QAAI,QAA0B,CAAC;AAE/B,aAAS,QAAQ,KAAK,OAAO;AACzB,UAAI,KAAK,eAAe,UAAU,KAAK,OAAO;AAC1C,YAAI,CAAC,KAAK,SAAU;AAEpB,cAAM,iBAAiB,eAAe,UAAU,KAAK,QAAQ;AAE7D,YAAI,CAAC,eAAe,QAAS;AAC7B,cAAM,WAAW,eAAe;AAChC,YAAI,SAAS,SAAS,QAAQ;AAC1B,cAAI,gBAAY,kCAAoB,SAAS,EAAE;AAC/C,cAAI,CAAC,WAAW;AACZ,gBAAI,SAAS,YAAY,SAAS,SAAS;AACvC,oBAAM,aAAS,wBAAU,SAAS,QAAQ;AAC1C,kBAAI,CAAC,QAAQ;AACT,sBAAM,IAAI;AAAA,kBACN,sCACI,SAAS;AAAA,gBACjB;AAAA,cACJ;AAEA,kBAAI,SAAS,OAAO;AAChB,4BAAY;AAAA,kBACR,MAAM;AAAA,kBACN,SAAS,SAAS;AAAA,kBAClB,OAAO,SAAS;AAAA,gBACpB;AAAA,cACJ,WAAW,mBAAmB,OAAO;AACjC,oBACI,kBAAkB,YACd,SAAS,WACb,kBAAkB,SAAS,QAC7B;AACE,wBAAM,IAAI;AAAA,oBACN;AAAA,kBACJ;AAAA,gBACJ;AAGA,4BAAY;AAAA,kBACR,MAAM;AAAA,kBACN,SAAS,SAAS;AAAA,kBAClB,OAAO,kBAAkB;AAAA,gBAC7B;AAAA,cACJ,OAAO;AACH,sBAAM,IAAI;AAAA,kBACN;AAAA,gBACJ;AAAA,cACJ;AAAA,YACJ,WAAW,SAAS,MAAM,kBAAkB,QAAQ;AAChD,8BAAY;AAAA,gBACR,SAAS,KAAK,iBAAiB,CAAC;AAAA,cACpC;AACA,kBAAI,CAAC,WAAW;AACZ,sBAAM,IAAI;AAAA,kBACN;AAAA,gBACJ;AAAA,cACJ;AAAA,YACJ,OAAO;AACH,oBAAM,IAAI;AAAA,gBACN;AAAA,cACJ;AAAA,YACJ;AAAA,UACJ;AAEA,cACI,sBACC,UAAU,SAAS,kBAAkB,QAClC,UAAU,YAAY,kBAAkB,WACxC,UAAU,UAAU,kBAAkB,QAC5C;AACE,yBAAa,mBAAmB,cAAc,KAAK,CAAC;AACpD,oBAAQ,CAAC;AAAA,UACb;AAEA,8BAAoB;AAEpB,cAAI,CAAC,KAAK,GAAI,MAAK,KAAK,UAAU;AAElC,gBAAM,MAAM,KAAK,QAAQ;AAAA,YACrB,KAAK;AAAA,YACL;AAAA,UACJ;AAEA,mBAAS,WAAW,IAAI,UAAU;AAC9B,kBAAM,eAAe,QAAQ;AAAA,cACzB;AAAA,YACJ;AACA,uBAAW,YAAY,cAAc;AACjC,oBAAM,eACF,SAAS,aAAa,eAAe;AACzC,kBAAI,cAAc;AACd,sBAAM,cACF,KAAK,QAAQ;AAAA,kBACT;AAAA,kBACA;AAAA,gBACJ;AACJ,sBAAM,eACF,mBAAmB,WAAW;AAClC,oBAAI,cAAc;AACd,qCAAmB,UAAU,SAAS;AAAA,oBAClC,QAAQ,KAAK;AAAA,oBACb,MAAM;AAAA,oBACN,QAAQ;AAAA,kBACZ,CAAC;AAAA,gBACL;AAAA,cACJ;AACA,uBAAS,OAAO;AAAA,YACpB;AAAA,UACJ;AAEA,gBAAM,UAAU,mBAAmB,GAAG;AAEtC,cAAI,SAAS;AAET,kBAAM,WAAW,QAAQ,MAAM,IAAI;AAEnC,kBAAM,KAAK,GAAG,QAAQ;AAAA,UAC1B;AAAA,QACJ,WAAW,SAAS,SAAS,YAAY;AACrC,gBAAM,gBAAY,kCAAoB,GAAG,SAAS,EAAE,IAAI;AACxD,cAAI,WAAW;AACX,kBAAM,MAAM,KAAK,QAAQ;AAAA,cACrB,KAAK;AAAA,cACL;AAAA,YACJ;AACA,kBAAM,UAAU,mBAAmB,GAAG;AAEtC,gBAAI,CAAC,SAAS;AACV;AAAA,YACJ;AAGA,kBAAMA,SAAQ,QACT,MAAM,IAAI,EACV,OAAyB,CAAC,MAAM,YAAY;AACzC,kBAAI,KAAK,WAAW;AAChB,uBAAO,CAAC,UAAU,OAAO,CAAC;AAAA;AAE1B,uBAAO;AAAA,kBACH,GAAG;AAAA,kBACH;AAAA,oBACI,MAAM;AAAA,kBACV;AAAA,kBACA,UAAU,OAAO;AAAA,gBACrB;AAAA,YACR,GAAG,CAAC,CAAC;AAET,YAAAA,OAAM;AAAA,cAAQ,CAAC,SACX,kBAAkB,UAAU,SAAS,CAAC,IAAI,CAAC;AAAA,YAC/C;AAAA,UACJ,OAAO;AACH,kBAAM,MAAM,KAAK,QAAQ;AAAA,cACrB,KAAK;AAAA,cACL;AAAA,YACJ;AACA,kBAAM,UAAU,mBAAmB,GAAG;AAGtC,gBAAI,WAAW,CAAC,KAAK,UAAU;AAC3B,kBAAI,aAAa;AACb,gBAAC,YAAwB,UAAU,KAAK;AAAA,kBACpC,QAAQ,KAAK;AAAA,kBACb,MAAM;AAAA,kBACN,QAAQ;AAAA,gBACZ,CAAC;AAAA,cACL;AAAA,YACJ;AAAA,UACJ;AAAA,QACJ;AAAA,MACJ;AAAA,IACJ;AAEA,QAAI,mBAAmB;AACnB,mBAAa,mBAAmB,cAAc,KAAK,CAAC;AAAA,IACxD;AAEA,aAAS,WAAW,SAAS,OAAO,GAAG;AACnC,WAAK,QAAQ,KAAK,OAAO;AAAA,IAC7B;AAEA,WAAO;AAAA,EACX;AACJ;",
   "names": ["ref", "lines"]
 }

package/dist/esm/generation/dataset.js

@@ -20,7 +20,7 @@ export function generateDataset(files, parser = new globalThis.DOMParser(), book
   };
   let usfmParser = new UsfmParser();
   let usxParser = new USXParser(parser);
-  let codexParser = new CodexParser();
+  let codexParser = new CodexParser(parser);
   let csvCommentaryParser = new CommentaryCsvParser();
   let tyndaleXmlParser = new TyndaleXmlParser(parser);
   let parsedTranslations = /* @__PURE__ */ new Map();

package/dist/esm/generation/dataset.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../generation/dataset.ts"],
-  "sourcesContent": ["import { UsfmParser } from '../parser/usfm-parser.js';\nimport { USXParser } from '../parser/usx-parser.js';\nimport {\n    Commentary,\n    CommentaryBook,\n    CommentaryBookChapter,\n    CommentaryChapterData,\n    CommentaryProfile,\n    InputCommentaryFile,\n    InputFile,\n    InputFileBase,\n    InputTranslationFile,\n    MetadataBase,\n    Translation,\n    TranslationBook,\n    TranslationBookChapter,\n    Dataset as CommonDataset,\n    DatasetBook,\n    DatasetBookChapter,\n} from './common-types.js';\nimport {\n    bookIdMap as defaultBookIdMap,\n    bookOrderMap,\n    apocryphaBooks,\n} from './book-order.js';\nimport { omit, sortBy, sortedIndexBy } from 'lodash';\nimport { getAudioUrlsForChapter } from './audio.js';\nimport { CodexParser } from '../parser/codex-parser.js';\nimport { CommentaryCsvParser } from '../parser/commentary-csv-parser.js';\nimport {\n    CommentaryParseTree,\n    ParseMessage,\n    ParseTree,\n} from '../parser/types.js';\nimport { TyndaleXmlParser } from '../parser/tyndale-xml-parser.js';\nimport { getLogger } from '../log.js';\n\n/**\n * Defines an interface that contains generated dataset info.\n */\nexport interface DatasetOutput {\n    /**\n     * The list of translations that are available in the dataset.\n     */\n    translations: DatasetTranslation[];\n\n    /**\n     * The list of commentaries that are available in the dataset.\n     */\n    commentaries: DatasetCommentary[];\n\n    /**\n     * The list of datasets that are available in the dataset.\n     */\n    datasets?: DatasetDataset[];\n\n    parseMessages?: {\n        [key: string]: ParseMessage[];\n    };\n}\n\n/**\n * Defines a translation that is used in the dataset.\n */\nexport interface DatasetTranslation extends Translation {\n    /**\n     * The list of books that are available for the translation.\n     */\n    books: DatasetTranslationBook[];\n}\n\n/**\n * Defines a commentary that is used in the dataset.\n */\nexport interface DatasetCommentary extends Commentary {\n    /**\n     * The list of books that are available for the commentary.\n     */\n    books: DatasetCommentaryBook[];\n\n    /**\n     * The list of profiles that  are available for the commentary.\n     */\n    profiles: DatasetCommentaryProfile[];\n}\n\n/**\n * Defines a translation book that is used in the dataset.\n */\nexport interface DatasetTranslationBook extends TranslationBook {\n    /**\n     * The list of chapters that are available for the book.\n     */\n    chapters: TranslationBookChapter[];\n}\n\n/**\n * Defines a commentary book that is used in the dataset.\n */\nexport interface DatasetCommentaryBook extends CommentaryBook {\n    /**\n     * The list of chapters that are available for the book.\n     */\n    chapters: CommentaryBookChapter[];\n}\n\nexport interface DatasetCommentaryProfile extends CommentaryProfile {\n    /**\n     * The contents of the profile.\n     */\n    content: string[];\n}\n\nexport interface DatasetDataset extends CommonDataset {\n    books: DatasetDatasetBook[];\n}\n\nexport interface DatasetDatasetBook extends DatasetBook {\n    chapters: DatasetBookChapter[];\n}\n\n/**\n * Generates a list of output files from the given list of input files.\n * @param file The list of files.\n * @param parser The parser to use for parsing the files.\n * @param bookMap The map of book IDs to names. If undefined, then a default map will be used.\n */\nexport function generateDataset(\n    files: InputFile[],\n    parser: DOMParser = new globalThis.DOMParser(),\n    bookMap?: Map<string, { commonName: string }> | undefined\n): DatasetOutput {\n    const logger = getLogger();\n    let output: DatasetOutput = {\n        translations: [],\n        commentaries: [],\n    };\n\n    let usfmParser = new UsfmParser();\n    let usxParser = new USXParser(parser);\n    let codexParser = new CodexParser();\n    let csvCommentaryParser = new CommentaryCsvParser();\n    let tyndaleXmlParser = new TyndaleXmlParser(parser);\n\n    let parsedTranslations = new Map<string, DatasetTranslation>();\n    let parsedCommentaries = new Map<string, DatasetCommentary>();\n\n    const unknownLanguages = new Set<string>();\n    for (let file of files) {\n        try {\n            let parser:\n                | UsfmParser\n                | USXParser\n                | CodexParser\n                | CommentaryCsvParser\n                | TyndaleXmlParser;\n            if (file.fileType === 'usfm') {\n                parser = usfmParser;\n            } else if (file.fileType === 'usx') {\n                parser = usxParser;\n            } else if (file.fileType === 'json') {\n                parser = codexParser;\n            } else if (file.fileType === 'commentary/csv') {\n                parser = csvCommentaryParser;\n            } else if (file.fileType === 'commentary/tyndale-xml') {\n                parser = tyndaleXmlParser;\n            } else {\n                logger.warn(\n                    '[generate] File does not have a valid type!',\n                    file.name\n                );\n                continue;\n            }\n\n            const parsed = parser.parse(file.content);\n\n            if (\n                'parseMessages' in parsed &&\n                parsed.parseMessages &&\n                file.name\n            ) {\n                const messages = (output.parseMessages =\n                    output.parseMessages || {});\n                messages[file.name] = parsed.parseMessages;\n            }\n\n            if (parsed.type === 'root') {\n                addTranslationTree(file as InputTranslationFile, parsed);\n            } else {\n                addCommentaryTree(file as InputCommentaryFile, parsed);\n            }\n        } catch (err) {\n            logger.error(\n                `[generate] Error occurred while parsing ${file.name}`,\n                err\n            );\n        }\n    }\n\n    function addTranslationTree(file: InputTranslationFile, parsed: ParseTree) {\n        const id = parsed.id;\n\n        if (!id) {\n            logger.warn(\n                '[generate] File does not have a valid book ID!',\n                file.name,\n                id\n            );\n            return;\n        }\n\n        const order = bookOrderMap.get(id);\n\n        if (typeof order !== 'number') {\n            logger.warn('[generate] Book does not have an order!', id);\n            return;\n        }\n\n        const isApocryphal = apocryphaBooks.has(id);\n\n        const bookName = getBookNames(file, file.metadata, id);\n\n        let translation = parsedTranslations.get(file.metadata.id);\n\n        if (!translation) {\n            translation = {\n                ...omit(file.metadata, 'direction'),\n                textDirection: file.metadata.direction,\n                books: [],\n            };\n            output.translations.push(translation);\n            parsedTranslations.set(file.metadata.id, translation);\n        }\n\n        const name =\n            parsed.header ??\n            (bookName?.exactMatch ? bookName?.bookName?.commonName : null) ??\n            parsed.title ??\n            bookName?.bookName?.commonName;\n\n        if (!name) {\n            logger.warn('[generate] Book does not have a name!', file.name, id);\n            return;\n        }\n\n        const commonName =\n            (bookName?.exactMatch ? bookName?.bookName?.commonName : null) ??\n            parsed.header ??\n            parsed.title ??\n            bookName?.bookName?.commonName ??\n            id;\n\n        const book: DatasetTranslationBook = {\n            id: id,\n            name,\n            commonName,\n            title: parsed.title ?? null,\n            order,\n            chapters: [],\n        };\n\n        if (isApocryphal) {\n            book.isApocryphal = true;\n        }\n\n        for (let content of parsed.content) {\n            if (content.type === 'chapter') {\n                book.chapters.push({\n                    chapter: {\n                        number: content.number,\n                        content: content.content,\n                        footnotes: content.footnotes,\n                    },\n                    thisChapterAudioLinks: getAudioUrlsForChapter(\n                        translation.id,\n                        id,\n                        content.number\n                    ),\n                });\n            }\n        }\n\n        book.chapters = sortBy(book.chapters, (c) => c.chapter.number);\n\n        const index = sortedIndexBy(translation.books, book, (b) => b.order);\n        translation.books.splice(index, 0, book);\n    }\n\n    function addCommentaryTree(\n        file: InputCommentaryFile,\n        parsed: CommentaryParseTree\n    ) {\n        let commentary = parsedCommentaries.get(file.metadata.id);\n\n        if (!commentary) {\n            commentary = {\n                ...omit(file.metadata, 'direction'),\n                textDirection: file.metadata.direction,\n                books: [],\n                profiles: [],\n            };\n            output.commentaries.push(commentary);\n            parsedCommentaries.set(file.metadata.id, commentary);\n        }\n\n        for (let parsedBook of parsed.books) {\n            let book = commentary.books.find((b) => b.id === parsedBook.book);\n            if (book && book.introduction && parsedBook.introduction) {\n                logger.warn(\n                    '[generate] Book already exists in commentary!',\n                    file.name,\n                    parsedBook.book\n                );\n                continue;\n            }\n\n            if (!book) {\n                const name = getBookNames(file, file.metadata, parsedBook.book);\n                book = {\n                    id: parsedBook.book,\n                    order: bookOrderMap.get(parsedBook.book) ?? -1,\n                    commonName: name?.bookName?.commonName ?? parsedBook.book,\n                    name: name?.bookName?.commonName ?? parsedBook.book,\n                    chapters: [],\n                };\n            }\n\n            if (parsedBook.introduction) {\n                book.introduction = parsedBook.introduction;\n            }\n\n            if (parsedBook.introductionSummary && !book.introductionSummary) {\n                book.introductionSummary = parsedBook.introductionSummary;\n            }\n\n            for (let chapter of parsedBook.chapters) {\n                let data: CommentaryChapterData = {\n                    number: chapter.number,\n                    content: chapter.verses,\n                };\n\n                if (chapter.introduction) {\n                    data.introduction = chapter.introduction;\n                }\n\n                book.chapters.push({\n                    chapter: data,\n                });\n            }\n\n            book.chapters = sortBy(book.chapters, (c) => c.chapter.number);\n            commentary.books.push(book);\n        }\n\n        if (parsed.profiles) {\n            for (let profile of parsed.profiles) {\n                const existing = commentary.profiles.find(\n                    (p) => p.id === profile.id\n                );\n                if (existing) {\n                    logger.warn(\n                        '[generate] Profile already exists in commentary!',\n                        file.name,\n                        profile.id\n                    );\n                    continue;\n                }\n\n                const datasetProfile: DatasetCommentaryProfile = {\n                    id: profile.id,\n                    subject: profile.subject,\n                    reference: profile.reference,\n                    content: profile.content,\n                };\n\n                commentary.profiles.push(datasetProfile);\n            }\n        }\n\n        commentary.books = sortBy(commentary.books, (b) => b.order);\n        commentary.profiles = sortBy(commentary.profiles, (p) => p.id);\n    }\n\n    function getBookNames(\n        file: InputFileBase,\n        metadata: MetadataBase,\n        id: string\n    ) {\n        let exactMatch = true;\n        let map = bookMap;\n        if (!map) {\n            map = defaultBookIdMap.get(metadata.language);\n\n            if (!map) {\n                if (!unknownLanguages.has(metadata.language)) {\n                    logger.warn(\n                        '[generate] File does not have a known language!',\n                        file.name,\n                        metadata.language\n                    );\n                    unknownLanguages.add(metadata.language);\n                }\n\n                logger.warn(\n                    '[generate] Using English book map for unknown language! This might result in outputting english book names.'\n                );\n                exactMatch = false;\n                map = defaultBookIdMap.get('en');\n            }\n        }\n\n        const bookName = map?.get(id);\n\n        if (!!map && !bookName) {\n            logger.warn(\n                '[generate] Book name not found for ID!',\n                file.name,\n                id\n            );\n            return null;\n        }\n\n        return {\n            exactMatch,\n            bookName,\n        };\n    }\n\n    return output;\n}\n"],
-  "mappings": ";AAAA,SAAS,kBAAkB;AAC3B,SAAS,iBAAiB;AAmB1B;AAAA,EACI,aAAa;AAAA,EACb;AAAA,EACA;AAAA,OACG;AACP,SAAS,MAAM,QAAQ,qBAAqB;AAC5C,SAAS,8BAA8B;AACvC,SAAS,mBAAmB;AAC5B,SAAS,2BAA2B;AAMpC,SAAS,wBAAwB;AACjC,SAAS,iBAAiB;AA4FnB,gBAAS,gBACZ,OACA,SAAoB,IAAI,WAAW,UAAU,GAC7C,SACa;AACb,QAAM,SAAS,UAAU;AACzB,MAAI,SAAwB;AAAA,IACxB,cAAc,CAAC;AAAA,IACf,cAAc,CAAC;AAAA,EACnB;AAEA,MAAI,aAAa,IAAI,WAAW;AAChC,MAAI,YAAY,IAAI,UAAU,MAAM;AACpC,MAAI,cAAc,IAAI,YAAY;AAClC,MAAI,sBAAsB,IAAI,oBAAoB;AAClD,MAAI,mBAAmB,IAAI,iBAAiB,MAAM;AAElD,MAAI,qBAAqB,oBAAI,IAAgC;AAC7D,MAAI,qBAAqB,oBAAI,IAA+B;AAE5D,QAAM,mBAAmB,oBAAI,IAAY;AACzC,WAAS,QAAQ,OAAO;AACpB,QAAI;AACA,UAAIA;AAMJ,UAAI,KAAK,aAAa,QAAQ;AAC1B,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,OAAO;AAChC,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,QAAQ;AACjC,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,kBAAkB;AAC3C,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,0BAA0B;AACnD,QAAAA,UAAS;AAAA,MACb,OAAO;AACH,eAAO;AAAA,UACH;AAAA,UACA,KAAK;AAAA,QACT;AACA;AAAA,MACJ;AAEA,YAAM,SAASA,QAAO,MAAM,KAAK,OAAO;AAExC,UACI,mBAAmB,UACnB,OAAO,iBACP,KAAK,MACP;AACE,cAAM,WAAY,OAAO,gBACrB,OAAO,iBAAiB,CAAC;AAC7B,iBAAS,KAAK,IAAI,IAAI,OAAO;AAAA,MACjC;AAEA,UAAI,OAAO,SAAS,QAAQ;AACxB,2BAAmB,MAA8B,MAAM;AAAA,MAC3D,OAAO;AACH,0BAAkB,MAA6B,MAAM;AAAA,MACzD;AAAA,IACJ,SAAS,KAAK;AACV,aAAO;AAAA,QACH,2CAA2C,KAAK,IAAI;AAAA,QACpD;AAAA,MACJ;AAAA,IACJ;AAAA,EACJ;AAEA,WAAS,mBAAmB,MAA4B,QAAmB;AACvE,UAAM,KAAK,OAAO;AAElB,QAAI,CAAC,IAAI;AACL,aAAO;AAAA,QACH;AAAA,QACA,KAAK;AAAA,QACL;AAAA,MACJ;AACA;AAAA,IACJ;AAEA,UAAM,QAAQ,aAAa,IAAI,EAAE;AAEjC,QAAI,OAAO,UAAU,UAAU;AAC3B,aAAO,KAAK,2CAA2C,EAAE;AACzD;AAAA,IACJ;AAEA,UAAM,eAAe,eAAe,IAAI,EAAE;AAE1C,UAAM,WAAW,aAAa,MAAM,KAAK,UAAU,EAAE;AAErD,QAAI,cAAc,mBAAmB,IAAI,KAAK,SAAS,EAAE;AAEzD,QAAI,CAAC,aAAa;AACd,oBAAc;AAAA,QACV,GAAG,KAAK,KAAK,UAAU,WAAW;AAAA,QAClC,eAAe,KAAK,SAAS;AAAA,QAC7B,OAAO,CAAC;AAAA,MACZ;AACA,aAAO,aAAa,KAAK,WAAW;AACpC,yBAAmB,IAAI,KAAK,SAAS,IAAI,WAAW;AAAA,IACxD;AAEA,UAAM,OACF,OAAO,WACN,UAAU,aAAa,UAAU,UAAU,aAAa,SACzD,OAAO,SACP,UAAU,UAAU;AAExB,QAAI,CAAC,MAAM;AACP,aAAO,KAAK,yCAAyC,KAAK,MAAM,EAAE;AAClE;AAAA,IACJ;AAEA,UAAM,cACD,UAAU,aAAa,UAAU,UAAU,aAAa,SACzD,OAAO,UACP,OAAO,SACP,UAAU,UAAU,cACpB;AAEJ,UAAM,OAA+B;AAAA,MACjC;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,OAAO,SAAS;AAAA,MACvB;AAAA,MACA,UAAU,CAAC;AAAA,IACf;AAEA,QAAI,cAAc;AACd,WAAK,eAAe;AAAA,IACxB;AAEA,aAAS,WAAW,OAAO,SAAS;AAChC,UAAI,QAAQ,SAAS,WAAW;AAC5B,aAAK,SAAS,KAAK;AAAA,UACf,SAAS;AAAA,YACL,QAAQ,QAAQ;AAAA,YAChB,SAAS,QAAQ;AAAA,YACjB,WAAW,QAAQ;AAAA,UACvB;AAAA,UACA,uBAAuB;AAAA,YACnB,YAAY;AAAA,YACZ;AAAA,YACA,QAAQ;AAAA,UACZ;AAAA,QACJ,CAAC;AAAA,MACL;AAAA,IACJ;AAEA,SAAK,WAAW,OAAO,KAAK,UAAU,CAAC,MAAM,EAAE,QAAQ,MAAM;AAE7D,UAAM,QAAQ,cAAc,YAAY,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK;AACnE,gBAAY,MAAM,OAAO,OAAO,GAAG,IAAI;AAAA,EAC3C;AAEA,WAAS,kBACL,MACA,QACF;AACE,QAAI,aAAa,mBAAmB,IAAI,KAAK,SAAS,EAAE;AAExD,QAAI,CAAC,YAAY;AACb,mBAAa;AAAA,QACT,GAAG,KAAK,KAAK,UAAU,WAAW;AAAA,QAClC,eAAe,KAAK,SAAS;AAAA,QAC7B,OAAO,CAAC;AAAA,QACR,UAAU,CAAC;AAAA,MACf;AACA,aAAO,aAAa,KAAK,UAAU;AACnC,yBAAmB,IAAI,KAAK,SAAS,IAAI,UAAU;AAAA,IACvD;AAEA,aAAS,cAAc,OAAO,OAAO;AACjC,UAAI,OAAO,WAAW,MAAM,KAAK,CAAC,MAAM,EAAE,OAAO,WAAW,IAAI;AAChE,UAAI,QAAQ,KAAK,gBAAgB,WAAW,cAAc;AACtD,eAAO;AAAA,UACH;AAAA,UACA,KAAK;AAAA,UACL,WAAW;AAAA,QACf;AACA;AAAA,MACJ;AAEA,UAAI,CAAC,MAAM;AACP,cAAM,OAAO,aAAa,MAAM,KAAK,UAAU,WAAW,IAAI;AAC9D,eAAO;AAAA,UACH,IAAI,WAAW;AAAA,UACf,OAAO,aAAa,IAAI,WAAW,IAAI,KAAK;AAAA,UAC5C,YAAY,MAAM,UAAU,cAAc,WAAW;AAAA,UACrD,MAAM,MAAM,UAAU,cAAc,WAAW;AAAA,UAC/C,UAAU,CAAC;AAAA,QACf;AAAA,MACJ;AAEA,UAAI,WAAW,cAAc;AACzB,aAAK,eAAe,WAAW;AAAA,MACnC;AAEA,UAAI,WAAW,uBAAuB,CAAC,KAAK,qBAAqB;AAC7D,aAAK,sBAAsB,WAAW;AAAA,MAC1C;AAEA,eAAS,WAAW,WAAW,UAAU;AACrC,YAAI,OAA8B;AAAA,UAC9B,QAAQ,QAAQ;AAAA,UAChB,SAAS,QAAQ;AAAA,QACrB;AAEA,YAAI,QAAQ,cAAc;AACtB,eAAK,eAAe,QAAQ;AAAA,QAChC;AAEA,aAAK,SAAS,KAAK;AAAA,UACf,SAAS;AAAA,QACb,CAAC;AAAA,MACL;AAEA,WAAK,WAAW,OAAO,KAAK,UAAU,CAAC,MAAM,EAAE,QAAQ,MAAM;AAC7D,iBAAW,MAAM,KAAK,IAAI;AAAA,IAC9B;AAEA,QAAI,OAAO,UAAU;AACjB,eAAS,WAAW,OAAO,UAAU;AACjC,cAAM,WAAW,WAAW,SAAS;AAAA,UACjC,CAAC,MAAM,EAAE,OAAO,QAAQ;AAAA,QAC5B;AACA,YAAI,UAAU;AACV,iBAAO;AAAA,YACH;AAAA,YACA,KAAK;AAAA,YACL,QAAQ;AAAA,UACZ;AACA;AAAA,QACJ;AAEA,cAAM,iBAA2C;AAAA,UAC7C,IAAI,QAAQ;AAAA,UACZ,SAAS,QAAQ;AAAA,UACjB,WAAW,QAAQ;AAAA,UACnB,SAAS,QAAQ;AAAA,QACrB;AAEA,mBAAW,SAAS,KAAK,cAAc;AAAA,MAC3C;AAAA,IACJ;AAEA,eAAW,QAAQ,OAAO,WAAW,OAAO,CAAC,MAAM,EAAE,KAAK;AAC1D,eAAW,WAAW,OAAO,WAAW,UAAU,CAAC,MAAM,EAAE,EAAE;AAAA,EACjE;AAEA,WAAS,aACL,MACA,UACA,IACF;AACE,QAAI,aAAa;AACjB,QAAI,MAAM;AACV,QAAI,CAAC,KAAK;AACN,YAAM,iBAAiB,IAAI,SAAS,QAAQ;AAE5C,UAAI,CAAC,KAAK;AACN,YAAI,CAAC,iBAAiB,IAAI,SAAS,QAAQ,GAAG;AAC1C,iBAAO;AAAA,YACH;AAAA,YACA,KAAK;AAAA,YACL,SAAS;AAAA,UACb;AACA,2BAAiB,IAAI,SAAS,QAAQ;AAAA,QAC1C;AAEA,eAAO;AAAA,UACH;AAAA,QACJ;AACA,qBAAa;AACb,cAAM,iBAAiB,IAAI,IAAI;AAAA,MACnC;AAAA,IACJ;AAEA,UAAM,WAAW,KAAK,IAAI,EAAE;AAE5B,QAAI,CAAC,CAAC,OAAO,CAAC,UAAU;AACpB,aAAO;AAAA,QACH;AAAA,QACA,KAAK;AAAA,QACL;AAAA,MACJ;AACA,aAAO;AAAA,IACX;AAEA,WAAO;AAAA,MACH;AAAA,MACA;AAAA,IACJ;AAAA,EACJ;AAEA,SAAO;AACX;",
+  "sourcesContent": ["import { UsfmParser } from '../parser/usfm-parser.js';\nimport { USXParser } from '../parser/usx-parser.js';\nimport {\n    Commentary,\n    CommentaryBook,\n    CommentaryBookChapter,\n    CommentaryChapterData,\n    CommentaryProfile,\n    InputCommentaryFile,\n    InputFile,\n    InputFileBase,\n    InputTranslationFile,\n    MetadataBase,\n    Translation,\n    TranslationBook,\n    TranslationBookChapter,\n    Dataset as CommonDataset,\n    DatasetBook,\n    DatasetBookChapter,\n} from './common-types.js';\nimport {\n    bookIdMap as defaultBookIdMap,\n    bookOrderMap,\n    apocryphaBooks,\n} from './book-order.js';\nimport { omit, sortBy, sortedIndexBy } from 'lodash';\nimport { getAudioUrlsForChapter } from './audio.js';\nimport { CodexParser } from '../parser/codex-parser.js';\nimport { CommentaryCsvParser } from '../parser/commentary-csv-parser.js';\nimport {\n    CommentaryParseTree,\n    ParseMessage,\n    ParseTree,\n} from '../parser/types.js';\nimport { TyndaleXmlParser } from '../parser/tyndale-xml-parser.js';\nimport { getLogger } from '../log.js';\n\n/**\n * Defines an interface that contains generated dataset info.\n */\nexport interface DatasetOutput {\n    /**\n     * The list of translations that are available in the dataset.\n     */\n    translations: DatasetTranslation[];\n\n    /**\n     * The list of commentaries that are available in the dataset.\n     */\n    commentaries: DatasetCommentary[];\n\n    /**\n     * The list of datasets that are available in the dataset.\n     */\n    datasets?: DatasetDataset[];\n\n    parseMessages?: {\n        [key: string]: ParseMessage[];\n    };\n}\n\n/**\n * Defines a translation that is used in the dataset.\n */\nexport interface DatasetTranslation extends Translation {\n    /**\n     * The list of books that are available for the translation.\n     */\n    books: DatasetTranslationBook[];\n}\n\n/**\n * Defines a commentary that is used in the dataset.\n */\nexport interface DatasetCommentary extends Commentary {\n    /**\n     * The list of books that are available for the commentary.\n     */\n    books: DatasetCommentaryBook[];\n\n    /**\n     * The list of profiles that  are available for the commentary.\n     */\n    profiles: DatasetCommentaryProfile[];\n}\n\n/**\n * Defines a translation book that is used in the dataset.\n */\nexport interface DatasetTranslationBook extends TranslationBook {\n    /**\n     * The list of chapters that are available for the book.\n     */\n    chapters: TranslationBookChapter[];\n}\n\n/**\n * Defines a commentary book that is used in the dataset.\n */\nexport interface DatasetCommentaryBook extends CommentaryBook {\n    /**\n     * The list of chapters that are available for the book.\n     */\n    chapters: CommentaryBookChapter[];\n}\n\nexport interface DatasetCommentaryProfile extends CommentaryProfile {\n    /**\n     * The contents of the profile.\n     */\n    content: string[];\n}\n\nexport interface DatasetDataset extends CommonDataset {\n    books: DatasetDatasetBook[];\n}\n\nexport interface DatasetDatasetBook extends DatasetBook {\n    chapters: DatasetBookChapter[];\n}\n\n/**\n * Generates a list of output files from the given list of input files.\n * @param file The list of files.\n * @param parser The parser to use for parsing the files.\n * @param bookMap The map of book IDs to names. If undefined, then a default map will be used.\n */\nexport function generateDataset(\n    files: InputFile[],\n    parser: DOMParser = new globalThis.DOMParser(),\n    bookMap?: Map<string, { commonName: string }> | undefined\n): DatasetOutput {\n    const logger = getLogger();\n    let output: DatasetOutput = {\n        translations: [],\n        commentaries: [],\n    };\n\n    let usfmParser = new UsfmParser();\n    let usxParser = new USXParser(parser);\n    let codexParser = new CodexParser(parser);\n    let csvCommentaryParser = new CommentaryCsvParser();\n    let tyndaleXmlParser = new TyndaleXmlParser(parser);\n\n    let parsedTranslations = new Map<string, DatasetTranslation>();\n    let parsedCommentaries = new Map<string, DatasetCommentary>();\n\n    const unknownLanguages = new Set<string>();\n    for (let file of files) {\n        try {\n            let parser:\n                | UsfmParser\n                | USXParser\n                | CodexParser\n                | CommentaryCsvParser\n                | TyndaleXmlParser;\n            if (file.fileType === 'usfm') {\n                parser = usfmParser;\n            } else if (file.fileType === 'usx') {\n                parser = usxParser;\n            } else if (file.fileType === 'json') {\n                parser = codexParser;\n            } else if (file.fileType === 'commentary/csv') {\n                parser = csvCommentaryParser;\n            } else if (file.fileType === 'commentary/tyndale-xml') {\n                parser = tyndaleXmlParser;\n            } else {\n                logger.warn(\n                    '[generate] File does not have a valid type!',\n                    file.name\n                );\n                continue;\n            }\n\n            const parsed = parser.parse(file.content);\n\n            if (\n                'parseMessages' in parsed &&\n                parsed.parseMessages &&\n                file.name\n            ) {\n                const messages = (output.parseMessages =\n                    output.parseMessages || {});\n                messages[file.name] = parsed.parseMessages;\n            }\n\n            if (parsed.type === 'root') {\n                addTranslationTree(file as InputTranslationFile, parsed);\n            } else {\n                addCommentaryTree(file as InputCommentaryFile, parsed);\n            }\n        } catch (err) {\n            logger.error(\n                `[generate] Error occurred while parsing ${file.name}`,\n                err\n            );\n        }\n    }\n\n    function addTranslationTree(file: InputTranslationFile, parsed: ParseTree) {\n        const id = parsed.id;\n\n        if (!id) {\n            logger.warn(\n                '[generate] File does not have a valid book ID!',\n                file.name,\n                id\n            );\n            return;\n        }\n\n        const order = bookOrderMap.get(id);\n\n        if (typeof order !== 'number') {\n            logger.warn('[generate] Book does not have an order!', id);\n            return;\n        }\n\n        const isApocryphal = apocryphaBooks.has(id);\n\n        const bookName = getBookNames(file, file.metadata, id);\n\n        let translation = parsedTranslations.get(file.metadata.id);\n\n        if (!translation) {\n            translation = {\n                ...omit(file.metadata, 'direction'),\n                textDirection: file.metadata.direction,\n                books: [],\n            };\n            output.translations.push(translation);\n            parsedTranslations.set(file.metadata.id, translation);\n        }\n\n        const name =\n            parsed.header ??\n            (bookName?.exactMatch ? bookName?.bookName?.commonName : null) ??\n            parsed.title ??\n            bookName?.bookName?.commonName;\n\n        if (!name) {\n            logger.warn('[generate] Book does not have a name!', file.name, id);\n            return;\n        }\n\n        const commonName =\n            (bookName?.exactMatch ? bookName?.bookName?.commonName : null) ??\n            parsed.header ??\n            parsed.title ??\n            bookName?.bookName?.commonName ??\n            id;\n\n        const book: DatasetTranslationBook = {\n            id: id,\n            name,\n            commonName,\n            title: parsed.title ?? null,\n            order,\n            chapters: [],\n        };\n\n        if (isApocryphal) {\n            book.isApocryphal = true;\n        }\n\n        for (let content of parsed.content) {\n            if (content.type === 'chapter') {\n                book.chapters.push({\n                    chapter: {\n                        number: content.number,\n                        content: content.content,\n                        footnotes: content.footnotes,\n                    },\n                    thisChapterAudioLinks: getAudioUrlsForChapter(\n                        translation.id,\n                        id,\n                        content.number\n                    ),\n                });\n            }\n        }\n\n        book.chapters = sortBy(book.chapters, (c) => c.chapter.number);\n\n        const index = sortedIndexBy(translation.books, book, (b) => b.order);\n        translation.books.splice(index, 0, book);\n    }\n\n    function addCommentaryTree(\n        file: InputCommentaryFile,\n        parsed: CommentaryParseTree\n    ) {\n        let commentary = parsedCommentaries.get(file.metadata.id);\n\n        if (!commentary) {\n            commentary = {\n                ...omit(file.metadata, 'direction'),\n                textDirection: file.metadata.direction,\n                books: [],\n                profiles: [],\n            };\n            output.commentaries.push(commentary);\n            parsedCommentaries.set(file.metadata.id, commentary);\n        }\n\n        for (let parsedBook of parsed.books) {\n            let book = commentary.books.find((b) => b.id === parsedBook.book);\n            if (book && book.introduction && parsedBook.introduction) {\n                logger.warn(\n                    '[generate] Book already exists in commentary!',\n                    file.name,\n                    parsedBook.book\n                );\n                continue;\n            }\n\n            if (!book) {\n                const name = getBookNames(file, file.metadata, parsedBook.book);\n                book = {\n                    id: parsedBook.book,\n                    order: bookOrderMap.get(parsedBook.book) ?? -1,\n                    commonName: name?.bookName?.commonName ?? parsedBook.book,\n                    name: name?.bookName?.commonName ?? parsedBook.book,\n                    chapters: [],\n                };\n            }\n\n            if (parsedBook.introduction) {\n                book.introduction = parsedBook.introduction;\n            }\n\n            if (parsedBook.introductionSummary && !book.introductionSummary) {\n                book.introductionSummary = parsedBook.introductionSummary;\n            }\n\n            for (let chapter of parsedBook.chapters) {\n                let data: CommentaryChapterData = {\n                    number: chapter.number,\n                    content: chapter.verses,\n                };\n\n                if (chapter.introduction) {\n                    data.introduction = chapter.introduction;\n                }\n\n                book.chapters.push({\n                    chapter: data,\n                });\n            }\n\n            book.chapters = sortBy(book.chapters, (c) => c.chapter.number);\n            commentary.books.push(book);\n        }\n\n        if (parsed.profiles) {\n            for (let profile of parsed.profiles) {\n                const existing = commentary.profiles.find(\n                    (p) => p.id === profile.id\n                );\n                if (existing) {\n                    logger.warn(\n                        '[generate] Profile already exists in commentary!',\n                        file.name,\n                        profile.id\n                    );\n                    continue;\n                }\n\n                const datasetProfile: DatasetCommentaryProfile = {\n                    id: profile.id,\n                    subject: profile.subject,\n                    reference: profile.reference,\n                    content: profile.content,\n                };\n\n                commentary.profiles.push(datasetProfile);\n            }\n        }\n\n        commentary.books = sortBy(commentary.books, (b) => b.order);\n        commentary.profiles = sortBy(commentary.profiles, (p) => p.id);\n    }\n\n    function getBookNames(\n        file: InputFileBase,\n        metadata: MetadataBase,\n        id: string\n    ) {\n        let exactMatch = true;\n        let map = bookMap;\n        if (!map) {\n            map = defaultBookIdMap.get(metadata.language);\n\n            if (!map) {\n                if (!unknownLanguages.has(metadata.language)) {\n                    logger.warn(\n                        '[generate] File does not have a known language!',\n                        file.name,\n                        metadata.language\n                    );\n                    unknownLanguages.add(metadata.language);\n                }\n\n                logger.warn(\n                    '[generate] Using English book map for unknown language! This might result in outputting english book names.'\n                );\n                exactMatch = false;\n                map = defaultBookIdMap.get('en');\n            }\n        }\n\n        const bookName = map?.get(id);\n\n        if (!!map && !bookName) {\n            logger.warn(\n                '[generate] Book name not found for ID!',\n                file.name,\n                id\n            );\n            return null;\n        }\n\n        return {\n            exactMatch,\n            bookName,\n        };\n    }\n\n    return output;\n}\n"],
+  "mappings": ";AAAA,SAAS,kBAAkB;AAC3B,SAAS,iBAAiB;AAmB1B;AAAA,EACI,aAAa;AAAA,EACb;AAAA,EACA;AAAA,OACG;AACP,SAAS,MAAM,QAAQ,qBAAqB;AAC5C,SAAS,8BAA8B;AACvC,SAAS,mBAAmB;AAC5B,SAAS,2BAA2B;AAMpC,SAAS,wBAAwB;AACjC,SAAS,iBAAiB;AA4FnB,gBAAS,gBACZ,OACA,SAAoB,IAAI,WAAW,UAAU,GAC7C,SACa;AACb,QAAM,SAAS,UAAU;AACzB,MAAI,SAAwB;AAAA,IACxB,cAAc,CAAC;AAAA,IACf,cAAc,CAAC;AAAA,EACnB;AAEA,MAAI,aAAa,IAAI,WAAW;AAChC,MAAI,YAAY,IAAI,UAAU,MAAM;AACpC,MAAI,cAAc,IAAI,YAAY,MAAM;AACxC,MAAI,sBAAsB,IAAI,oBAAoB;AAClD,MAAI,mBAAmB,IAAI,iBAAiB,MAAM;AAElD,MAAI,qBAAqB,oBAAI,IAAgC;AAC7D,MAAI,qBAAqB,oBAAI,IAA+B;AAE5D,QAAM,mBAAmB,oBAAI,IAAY;AACzC,WAAS,QAAQ,OAAO;AACpB,QAAI;AACA,UAAIA;AAMJ,UAAI,KAAK,aAAa,QAAQ;AAC1B,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,OAAO;AAChC,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,QAAQ;AACjC,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,kBAAkB;AAC3C,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,0BAA0B;AACnD,QAAAA,UAAS;AAAA,MACb,OAAO;AACH,eAAO;AAAA,UACH;AAAA,UACA,KAAK;AAAA,QACT;AACA;AAAA,MACJ;AAEA,YAAM,SAASA,QAAO,MAAM,KAAK,OAAO;AAExC,UACI,mBAAmB,UACnB,OAAO,iBACP,KAAK,MACP;AACE,cAAM,WAAY,OAAO,gBACrB,OAAO,iBAAiB,CAAC;AAC7B,iBAAS,KAAK,IAAI,IAAI,OAAO;AAAA,MACjC;AAEA,UAAI,OAAO,SAAS,QAAQ;AACxB,2BAAmB,MAA8B,MAAM;AAAA,MAC3D,OAAO;AACH,0BAAkB,MAA6B,MAAM;AAAA,MACzD;AAAA,IACJ,SAAS,KAAK;AACV,aAAO;AAAA,QACH,2CAA2C,KAAK,IAAI;AAAA,QACpD;AAAA,MACJ;AAAA,IACJ;AAAA,EACJ;AAEA,WAAS,mBAAmB,MAA4B,QAAmB;AACvE,UAAM,KAAK,OAAO;AAElB,QAAI,CAAC,IAAI;AACL,aAAO;AAAA,QACH;AAAA,QACA,KAAK;AAAA,QACL;AAAA,MACJ;AACA;AAAA,IACJ;AAEA,UAAM,QAAQ,aAAa,IAAI,EAAE;AAEjC,QAAI,OAAO,UAAU,UAAU;AAC3B,aAAO,KAAK,2CAA2C,EAAE;AACzD;AAAA,IACJ;AAEA,UAAM,eAAe,eAAe,IAAI,EAAE;AAE1C,UAAM,WAAW,aAAa,MAAM,KAAK,UAAU,EAAE;AAErD,QAAI,cAAc,mBAAmB,IAAI,KAAK,SAAS,EAAE;AAEzD,QAAI,CAAC,aAAa;AACd,oBAAc;AAAA,QACV,GAAG,KAAK,KAAK,UAAU,WAAW;AAAA,QAClC,eAAe,KAAK,SAAS;AAAA,QAC7B,OAAO,CAAC;AAAA,MACZ;AACA,aAAO,aAAa,KAAK,WAAW;AACpC,yBAAmB,IAAI,KAAK,SAAS,IAAI,WAAW;AAAA,IACxD;AAEA,UAAM,OACF,OAAO,WACN,UAAU,aAAa,UAAU,UAAU,aAAa,SACzD,OAAO,SACP,UAAU,UAAU;AAExB,QAAI,CAAC,MAAM;AACP,aAAO,KAAK,yCAAyC,KAAK,MAAM,EAAE;AAClE;AAAA,IACJ;AAEA,UAAM,cACD,UAAU,aAAa,UAAU,UAAU,aAAa,SACzD,OAAO,UACP,OAAO,SACP,UAAU,UAAU,cACpB;AAEJ,UAAM,OAA+B;AAAA,MACjC;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,OAAO,SAAS;AAAA,MACvB;AAAA,MACA,UAAU,CAAC;AAAA,IACf;AAEA,QAAI,cAAc;AACd,WAAK,eAAe;AAAA,IACxB;AAEA,aAAS,WAAW,OAAO,SAAS;AAChC,UAAI,QAAQ,SAAS,WAAW;AAC5B,aAAK,SAAS,KAAK;AAAA,UACf,SAAS;AAAA,YACL,QAAQ,QAAQ;AAAA,YAChB,SAAS,QAAQ;AAAA,YACjB,WAAW,QAAQ;AAAA,UACvB;AAAA,UACA,uBAAuB;AAAA,YACnB,YAAY;AAAA,YACZ;AAAA,YACA,QAAQ;AAAA,UACZ;AAAA,QACJ,CAAC;AAAA,MACL;AAAA,IACJ;AAEA,SAAK,WAAW,OAAO,KAAK,UAAU,CAAC,MAAM,EAAE,QAAQ,MAAM;AAE7D,UAAM,QAAQ,cAAc,YAAY,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK;AACnE,gBAAY,MAAM,OAAO,OAAO,GAAG,IAAI;AAAA,EAC3C;AAEA,WAAS,kBACL,MACA,QACF;AACE,QAAI,aAAa,mBAAmB,IAAI,KAAK,SAAS,EAAE;AAExD,QAAI,CAAC,YAAY;AACb,mBAAa;AAAA,QACT,GAAG,KAAK,KAAK,UAAU,WAAW;AAAA,QAClC,eAAe,KAAK,SAAS;AAAA,QAC7B,OAAO,CAAC;AAAA,QACR,UAAU,CAAC;AAAA,MACf;AACA,aAAO,aAAa,KAAK,UAAU;AACnC,yBAAmB,IAAI,KAAK,SAAS,IAAI,UAAU;AAAA,IACvD;AAEA,aAAS,cAAc,OAAO,OAAO;AACjC,UAAI,OAAO,WAAW,MAAM,KAAK,CAAC,MAAM,EAAE,OAAO,WAAW,IAAI;AAChE,UAAI,QAAQ,KAAK,gBAAgB,WAAW,cAAc;AACtD,eAAO;AAAA,UACH;AAAA,UACA,KAAK;AAAA,UACL,WAAW;AAAA,QACf;AACA;AAAA,MACJ;AAEA,UAAI,CAAC,MAAM;AACP,cAAM,OAAO,aAAa,MAAM,KAAK,UAAU,WAAW,IAAI;AAC9D,eAAO;AAAA,UACH,IAAI,WAAW;AAAA,UACf,OAAO,aAAa,IAAI,WAAW,IAAI,KAAK;AAAA,UAC5C,YAAY,MAAM,UAAU,cAAc,WAAW;AAAA,UACrD,MAAM,MAAM,UAAU,cAAc,WAAW;AAAA,UAC/C,UAAU,CAAC;AAAA,QACf;AAAA,MACJ;AAEA,UAAI,WAAW,cAAc;AACzB,aAAK,eAAe,WAAW;AAAA,MACnC;AAEA,UAAI,WAAW,uBAAuB,CAAC,KAAK,qBAAqB;AAC7D,aAAK,sBAAsB,WAAW;AAAA,MAC1C;AAEA,eAAS,WAAW,WAAW,UAAU;AACrC,YAAI,OAA8B;AAAA,UAC9B,QAAQ,QAAQ;AAAA,UAChB,SAAS,QAAQ;AAAA,QACrB;AAEA,YAAI,QAAQ,cAAc;AACtB,eAAK,eAAe,QAAQ;AAAA,QAChC;AAEA,aAAK,SAAS,KAAK;AAAA,UACf,SAAS;AAAA,QACb,CAAC;AAAA,MACL;AAEA,WAAK,WAAW,OAAO,KAAK,UAAU,CAAC,MAAM,EAAE,QAAQ,MAAM;AAC7D,iBAAW,MAAM,KAAK,IAAI;AAAA,IAC9B;AAEA,QAAI,OAAO,UAAU;AACjB,eAAS,WAAW,OAAO,UAAU;AACjC,cAAM,WAAW,WAAW,SAAS;AAAA,UACjC,CAAC,MAAM,EAAE,OAAO,QAAQ;AAAA,QAC5B;AACA,YAAI,UAAU;AACV,iBAAO;AAAA,YACH;AAAA,YACA,KAAK;AAAA,YACL,QAAQ;AAAA,UACZ;AACA;AAAA,QACJ;AAEA,cAAM,iBAA2C;AAAA,UAC7C,IAAI,QAAQ;AAAA,UACZ,SAAS,QAAQ;AAAA,UACjB,WAAW,QAAQ;AAAA,UACnB,SAAS,QAAQ;AAAA,QACrB;AAEA,mBAAW,SAAS,KAAK,cAAc;AAAA,MAC3C;AAAA,IACJ;AAEA,eAAW,QAAQ,OAAO,WAAW,OAAO,CAAC,MAAM,EAAE,KAAK;AAC1D,eAAW,WAAW,OAAO,WAAW,UAAU,CAAC,MAAM,EAAE,EAAE;AAAA,EACjE;AAEA,WAAS,aACL,MACA,UACA,IACF;AACE,QAAI,aAAa;AACjB,QAAI,MAAM;AACV,QAAI,CAAC,KAAK;AACN,YAAM,iBAAiB,IAAI,SAAS,QAAQ;AAE5C,UAAI,CAAC,KAAK;AACN,YAAI,CAAC,iBAAiB,IAAI,SAAS,QAAQ,GAAG;AAC1C,iBAAO;AAAA,YACH;AAAA,YACA,KAAK;AAAA,YACL,SAAS;AAAA,UACb;AACA,2BAAiB,IAAI,SAAS,QAAQ;AAAA,QAC1C;AAEA,eAAO;AAAA,UACH;AAAA,QACJ;AACA,qBAAa;AACb,cAAM,iBAAiB,IAAI,IAAI;AAAA,MACnC;AAAA,IACJ;AAEA,UAAM,WAAW,KAAK,IAAI,EAAE;AAE5B,QAAI,CAAC,CAAC,OAAO,CAAC,UAAU;AACpB,aAAO;AAAA,QACH;AAAA,QACA,KAAK;AAAA,QACL;AAAA,MACJ;AACA,aAAO;AAAA,IACX;AAEA,WAAO;AAAA,MACH;AAAA,MACA;AAAA,IACJ;AAAA,EACJ;AAEA,SAAO;AACX;",
   "names": ["parser"]
 }

package/dist/esm/parser/codex-parser.js

@@ -12,7 +12,10 @@ export const verseSchema = z.object({
   id: z.string(),
   bookCode: z.string().optional(),
   chapter: z.number().optional(),
-  verse: z.number().optional()
+  verse: z.number().optional(),
+  data: z.object({
+    globalReferences: z.array(z.string()).optional()
+  }).optional()
 });
 export const paratextSchema = z.object({
   type: z.literal("paratext"),
@@ -42,6 +45,10 @@ export class CodexParser {
   //  */
   // preserveMarkdown: boolean = true;
   _noteCounter = 0;
+  _parser;
+  constructor(parser) {
+    this._parser = parser;
+  }
   /**
    * Parses the specified codex content.
    *
@@ -57,7 +64,7 @@ export class CodexParser {
     };
     let chapters = /* @__PURE__ */ new Map();
     let lastChapter = null;
-    function addChapterContent(chapterNumber, content) {
+    function ensureChapter(chapterNumber) {
       let chapter = chapters.get(chapterNumber);
       if (!chapter) {
         chapter = {
@@ -69,8 +76,23 @@ export class CodexParser {
         lastChapter = chapter;
         chapters.set(chapterNumber, chapter);
       }
+      return chapter;
+    }
+    function getDocumentContent(doc) {
+      let content = "";
+      for (let element of doc.childNodes) {
+        content += element.textContent;
+      }
+      return content;
+    }
+    function addChapterContent(chapterNumber, content) {
+      const chapter = ensureChapter(chapterNumber);
       chapter.content.push(...content);
     }
+    function addChapterFootnote(chapterNumber, footnote) {
+      const chapter = ensureChapter(chapterNumber);
+      chapter.footnotes.push(footnote);
+    }
     function addReference(ref2, content) {
       addChapterContent(ref2.chapter, [
         {
@@ -92,8 +114,12 @@ export class CodexParser {
     function addLineBreaks(lines2) {
       return lines2.reduce(
         (prev, current) => {
-          if (prev.length === 0) return [current];
-          else
+          if (typeof current === "string" && current.trim() === "") {
+            return prev;
+          }
+          if (prev.length === 0) {
+            return [current];
+          } else {
             return [
               ...prev,
               {
@@ -101,6 +127,7 @@ export class CodexParser {
               },
               current
             ];
+          }
         },
         []
       );
@@ -145,6 +172,15 @@ export class CodexParser {
                   "Could not find verse reference in metadata."
                 );
               }
+            } else if (metadata.data?.globalReferences?.length) {
+              reference = parseVerseReference(
+                metadata.data.globalReferences[0]
+              );
+              if (!reference) {
+                throw new Error(
+                  "Could not find verse reference in globalReferences metadata."
+                );
+              }
             } else {
               throw new Error(
                 "Could not find verse reference in metadata."
@@ -157,13 +193,49 @@ export class CodexParser {
           }
           previousReference = reference;
           if (!root.id) root.id = reference.book;
-          const content = stripHTML(cell.value);
-          const newLines = content.split("\n");
-          lines.push(...newLines);
+          const doc = this._parser.parseFromString(
+            cell.value,
+            "text/html"
+          );
+          for (let element of doc.children) {
+            const allFootnotes = element.querySelectorAll(
+              "sup.footnote-marker[data-footnote]"
+            );
+            for (const footnote of allFootnotes) {
+              const footnoteData = footnote.getAttribute("data-footnote");
+              if (footnoteData) {
+                const footnoteDoc = this._parser.parseFromString(
+                  footnoteData,
+                  "text/html"
+                );
+                const footnoteText = getDocumentContent(footnoteDoc);
+                if (footnoteText) {
+                  addChapterFootnote(reference.chapter, {
+                    noteId: this._noteCounter++,
+                    text: footnoteText,
+                    caller: null
+                  });
+                }
+              }
+              footnote.remove();
+            }
+          }
+          const content = getDocumentContent(doc);
+          if (content) {
+            const newLines = content.split("\n");
+            lines.push(...newLines);
+          }
         } else if (metadata.type === "paratext") {
           const reference = parseVerseReference(`${metadata.id}:1`);
           if (reference) {
-            const content = stripHTML(cell.value);
+            const doc = this._parser.parseFromString(
+              cell.value,
+              "text/html"
+            );
+            const content = getDocumentContent(doc);
+            if (!content) {
+              continue;
+            }
             const lines2 = content.split("\n").reduce((prev, current) => {
               if (prev.length === 0)
                 return [toHeading(current)];
@@ -180,8 +252,12 @@ export class CodexParser {
               (line) => addChapterContent(reference.chapter, [line])
             );
           } else {
-            const content = stripHTML(cell.value);
-            if (!cell.metadata) {
+            const doc = this._parser.parseFromString(
+              cell.value,
+              "text/html"
+            );
+            const content = getDocumentContent(doc);
+            if (content && !cell.metadata) {
               if (lastChapter) {
                 lastChapter.footnotes.push({
                   noteId: this._noteCounter++,

package/dist/esm/parser/codex-parser.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../parser/codex-parser.ts"],
-  "sourcesContent": ["import { z } from 'zod';\nimport {\n    Chapter,\n    ChapterContent,\n    ParseTree,\n    Verse,\n    VerseContent,\n} from './types.js';\nimport { getBookId, parseVerseReference, VerseRef } from '../utils.js';\nimport { ref } from 'process';\n\nexport const chapterHeadingSchema = z.object({\n    type: z.literal('chapter-heading'),\n    data: z.object({\n        chapter: z.coerce.number().int(),\n    }),\n});\n\nexport const verseSchema = z.object({\n    type: z.literal('text'),\n    id: z.string(),\n    bookCode: z.string().optional(),\n    chapter: z.number().optional(),\n    verse: z.number().optional(),\n});\n\nexport const paratextSchema = z.object({\n    type: z.literal('paratext'),\n    id: z.string(),\n});\n\nexport const metadataSchema = z.discriminatedUnion('type', [\n    chapterHeadingSchema,\n    verseSchema,\n    paratextSchema,\n]);\n\nexport const codexSchema = z.object({\n    cells: z.array(\n        z.object({\n            kind: z.number(),\n            languageId: z.string(),\n            value: z.string(),\n            metadata: z\n                .object({\n                    type: z.string(),\n                })\n                .passthrough()\n                .nullable()\n                .optional(),\n        })\n    ),\n});\n\n/**\n * Defines a parser for codex files.\n */\nexport class CodexParser {\n    // TODO:\n    // /**\n    //  * Whether to preserve markdown in the parsed content.\n    //  */\n    // preserveMarkdown: boolean = true;\n\n    private _noteCounter: number = 0;\n\n    /**\n     * Parses the specified codex content.\n     *\n     * @param codex The codex content to parse.\n     * @returns The parse tree that was generated.\n     */\n    public parse(codex: string): ParseTree {\n        const json = JSON.parse(codex);\n        const data = codexSchema.parse(json);\n\n        let root: ParseTree = {\n            type: 'root',\n            content: [],\n        };\n\n        let chapters: Map<number, Chapter> = new Map();\n        let lastChapter: Chapter | null = null;\n\n        function addChapterContent(\n            chapterNumber: number,\n            content: ChapterContent[]\n        ) {\n            let chapter = chapters.get(chapterNumber);\n            if (!chapter) {\n                chapter = {\n                    type: 'chapter',\n                    number: chapterNumber,\n                    content: [],\n                    footnotes: [],\n                };\n                lastChapter = chapter;\n                chapters.set(chapterNumber, chapter);\n            }\n\n            chapter.content.push(...content);\n        }\n\n        function addReference(ref: VerseRef, content: Verse['content']) {\n            addChapterContent(ref.chapter, [\n                {\n                    type: 'verse',\n                    number: ref.verse,\n                    content: content,\n                },\n            ]);\n        }\n\n        function stripHTML(value: string) {\n            return value.replace(/<[^>]*>/g, '');\n        }\n\n        function toHeading(content: string) {\n            return {\n                type: 'heading',\n                content: [content],\n            } satisfies ChapterContent;\n        }\n\n        function addLineBreaks(lines: Verse['content']) {\n            return lines.reduce(\n                (prev, current) => {\n                    if (prev.length === 0) return [current];\n                    else\n                        return [\n                            ...prev,\n                            {\n                                lineBreak: true,\n                            },\n                            current,\n                        ] satisfies Verse['content'];\n                },\n                [] as Verse['content']\n            );\n        }\n\n        let previousReference: VerseRef | null = null;\n        let lines: Verse['content'] = [];\n\n        for (let cell of data.cells) {\n            if (cell.languageId === 'html' && cell.value) {\n                if (!cell.metadata) continue; // ignore html cells without metadata\n\n                const metadataResult = metadataSchema.safeParse(cell.metadata);\n\n                if (!metadataResult.success) continue; // ignore cells with invalid/unknown metadata\n                const metadata = metadataResult.data;\n                if (metadata.type === 'text') {\n                    let reference = parseVerseReference(metadata.id);\n                    if (!reference) {\n                        if (metadata.bookCode && metadata.chapter) {\n                            const bookId = getBookId(metadata.bookCode);\n                            if (!bookId) {\n                                throw new Error(\n                                    'Could not find book ID for code: ' +\n                                        metadata.bookCode\n                                );\n                            }\n\n                            if (metadata.verse) {\n                                reference = {\n                                    book: bookId,\n                                    chapter: metadata.chapter,\n                                    verse: metadata.verse,\n                                };\n                            } else if (previousReference?.verse) {\n                                if (\n                                    previousReference.chapter !==\n                                        metadata.chapter ||\n                                    previousReference.book !== bookId\n                                ) {\n                                    throw new Error(\n                                        'Cannot infer verse number for non-consecutive verse reference.'\n                                    );\n                                }\n\n                                // Sometimes codex doesn't include verse numbers for every cell, so we infer it from the previous reference\n                                reference = {\n                                    book: bookId,\n                                    chapter: metadata.chapter,\n                                    verse: previousReference.verse,\n                                };\n                            } else {\n                                throw new Error(\n                                    'Could not find verse reference in metadata.'\n                                );\n                            }\n                        } else {\n                            throw new Error(\n                                'Could not find verse reference in metadata.'\n                            );\n                        }\n                    }\n\n                    if (\n                        previousReference &&\n                        (reference.book !== previousReference.book ||\n                            reference.chapter !== previousReference.chapter ||\n                            reference.verse !== previousReference.verse)\n                    ) {\n                        addReference(previousReference, addLineBreaks(lines));\n                        lines = [];\n                    }\n\n                    previousReference = reference;\n\n                    if (!root.id) root.id = reference.book; // set book id if not already set\n\n                    const content = stripHTML(cell.value);\n\n                    // add line breaks in heading if there are multiple lines\n                    const newLines = content.split('\\n');\n\n                    lines.push(...newLines);\n                } else if (metadata.type === 'paratext') {\n                    const reference = parseVerseReference(`${metadata.id}:1`); // chapter headings do not have a verse reference always default to 1\n                    if (reference) {\n                        const content = stripHTML(cell.value);\n\n                        // add line breaks in heading if there are multiple lines\n                        const lines = content\n                            .split('\\n')\n                            .reduce<ChapterContent[]>((prev, current) => {\n                                if (prev.length === 0)\n                                    return [toHeading(current)];\n                                else\n                                    return [\n                                        ...prev,\n                                        {\n                                            type: 'line_break',\n                                        },\n                                        toHeading(current),\n                                    ] satisfies ChapterContent[];\n                            }, []);\n\n                        lines.forEach((line) =>\n                            addChapterContent(reference.chapter, [line])\n                        );\n                    } else {\n                        // parse paratext that isn't a chapter reference as footnote\n                        const content = stripHTML(cell.value);\n                        if (!cell.metadata) {\n                            if (lastChapter) {\n                                (lastChapter as Chapter).footnotes.push({\n                                    noteId: this._noteCounter++,\n                                    text: content,\n                                    caller: null,\n                                });\n                            }\n                        }\n                    }\n                }\n            }\n        }\n\n        if (previousReference) {\n            addReference(previousReference, addLineBreaks(lines));\n        }\n\n        for (let chapter of chapters.values()) {\n            root.content.push(chapter);\n        }\n\n        return root;\n    }\n}\n"],
-  "mappings": ";AAAA,SAAS,SAAS;AAQlB,SAAS,WAAW,2BAAqC;AAGlD,aAAM,uBAAuB,EAAE,OAAO;AAAA,EACzC,MAAM,EAAE,QAAQ,iBAAiB;AAAA,EACjC,MAAM,EAAE,OAAO;AAAA,IACX,SAAS,EAAE,OAAO,OAAO,EAAE,IAAI;AAAA,EACnC,CAAC;AACL,CAAC;AAEM,aAAM,cAAc,EAAE,OAAO;AAAA,EAChC,MAAM,EAAE,QAAQ,MAAM;AAAA,EACtB,IAAI,EAAE,OAAO;AAAA,EACb,UAAU,EAAE,OAAO,EAAE,SAAS;AAAA,EAC9B,SAAS,EAAE,OAAO,EAAE,SAAS;AAAA,EAC7B,OAAO,EAAE,OAAO,EAAE,SAAS;AAC/B,CAAC;AAEM,aAAM,iBAAiB,EAAE,OAAO;AAAA,EACnC,MAAM,EAAE,QAAQ,UAAU;AAAA,EAC1B,IAAI,EAAE,OAAO;AACjB,CAAC;AAEM,aAAM,iBAAiB,EAAE,mBAAmB,QAAQ;AAAA,EACvD;AAAA,EACA;AAAA,EACA;AACJ,CAAC;AAEM,aAAM,cAAc,EAAE,OAAO;AAAA,EAChC,OAAO,EAAE;AAAA,IACL,EAAE,OAAO;AAAA,MACL,MAAM,EAAE,OAAO;AAAA,MACf,YAAY,EAAE,OAAO;AAAA,MACrB,OAAO,EAAE,OAAO;AAAA,MAChB,UAAU,EACL,OAAO;AAAA,QACJ,MAAM,EAAE,OAAO;AAAA,MACnB,CAAC,EACA,YAAY,EACZ,SAAS,EACT,SAAS;AAAA,IAClB,CAAC;AAAA,EACL;AACJ,CAAC;AAKM,aAAM,YAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOb,eAAuB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQxB,MAAM,OAA0B;AACnC,UAAM,OAAO,KAAK,MAAM,KAAK;AAC7B,UAAM,OAAO,YAAY,MAAM,IAAI;AAEnC,QAAI,OAAkB;AAAA,MAClB,MAAM;AAAA,MACN,SAAS,CAAC;AAAA,IACd;AAEA,QAAI,WAAiC,oBAAI,IAAI;AAC7C,QAAI,cAA8B;AAElC,aAAS,kBACL,eACA,SACF;AACE,UAAI,UAAU,SAAS,IAAI,aAAa;AACxC,UAAI,CAAC,SAAS;AACV,kBAAU;AAAA,UACN,MAAM;AAAA,UACN,QAAQ;AAAA,UACR,SAAS,CAAC;AAAA,UACV,WAAW,CAAC;AAAA,QAChB;AACA,sBAAc;AACd,iBAAS,IAAI,eAAe,OAAO;AAAA,MACvC;AAEA,cAAQ,QAAQ,KAAK,GAAG,OAAO;AAAA,IACnC;AAEA,aAAS,aAAaA,MAAe,SAA2B;AAC5D,wBAAkBA,KAAI,SAAS;AAAA,QAC3B;AAAA,UACI,MAAM;AAAA,UACN,QAAQA,KAAI;AAAA,UACZ;AAAA,QACJ;AAAA,MACJ,CAAC;AAAA,IACL;AAEA,aAAS,UAAU,OAAe;AAC9B,aAAO,MAAM,QAAQ,YAAY,EAAE;AAAA,IACvC;AAEA,aAAS,UAAU,SAAiB;AAChC,aAAO;AAAA,QACH,MAAM;AAAA,QACN,SAAS,CAAC,OAAO;AAAA,MACrB;AAAA,IACJ;AAEA,aAAS,cAAcC,QAAyB;AAC5C,aAAOA,OAAM;AAAA,QACT,CAAC,MAAM,YAAY;AACf,cAAI,KAAK,WAAW,EAAG,QAAO,CAAC,OAAO;AAAA;AAElC,mBAAO;AAAA,cACH,GAAG;AAAA,cACH;AAAA,gBACI,WAAW;AAAA,cACf;AAAA,cACA;AAAA,YACJ;AAAA,QACR;AAAA,QACA,CAAC;AAAA,MACL;AAAA,IACJ;AAEA,QAAI,oBAAqC;AACzC,QAAI,QAA0B,CAAC;AAE/B,aAAS,QAAQ,KAAK,OAAO;AACzB,UAAI,KAAK,eAAe,UAAU,KAAK,OAAO;AAC1C,YAAI,CAAC,KAAK,SAAU;AAEpB,cAAM,iBAAiB,eAAe,UAAU,KAAK,QAAQ;AAE7D,YAAI,CAAC,eAAe,QAAS;AAC7B,cAAM,WAAW,eAAe;AAChC,YAAI,SAAS,SAAS,QAAQ;AAC1B,cAAI,YAAY,oBAAoB,SAAS,EAAE;AAC/C,cAAI,CAAC,WAAW;AACZ,gBAAI,SAAS,YAAY,SAAS,SAAS;AACvC,oBAAM,SAAS,UAAU,SAAS,QAAQ;AAC1C,kBAAI,CAAC,QAAQ;AACT,sBAAM,IAAI;AAAA,kBACN,sCACI,SAAS;AAAA,gBACjB;AAAA,cACJ;AAEA,kBAAI,SAAS,OAAO;AAChB,4BAAY;AAAA,kBACR,MAAM;AAAA,kBACN,SAAS,SAAS;AAAA,kBAClB,OAAO,SAAS;AAAA,gBACpB;AAAA,cACJ,WAAW,mBAAmB,OAAO;AACjC,oBACI,kBAAkB,YACd,SAAS,WACb,kBAAkB,SAAS,QAC7B;AACE,wBAAM,IAAI;AAAA,oBACN;AAAA,kBACJ;AAAA,gBACJ;AAGA,4BAAY;AAAA,kBACR,MAAM;AAAA,kBACN,SAAS,SAAS;AAAA,kBAClB,OAAO,kBAAkB;AAAA,gBAC7B;AAAA,cACJ,OAAO;AACH,sBAAM,IAAI;AAAA,kBACN;AAAA,gBACJ;AAAA,cACJ;AAAA,YACJ,OAAO;AACH,oBAAM,IAAI;AAAA,gBACN;AAAA,cACJ;AAAA,YACJ;AAAA,UACJ;AAEA,cACI,sBACC,UAAU,SAAS,kBAAkB,QAClC,UAAU,YAAY,kBAAkB,WACxC,UAAU,UAAU,kBAAkB,QAC5C;AACE,yBAAa,mBAAmB,cAAc,KAAK,CAAC;AACpD,oBAAQ,CAAC;AAAA,UACb;AAEA,8BAAoB;AAEpB,cAAI,CAAC,KAAK,GAAI,MAAK,KAAK,UAAU;AAElC,gBAAM,UAAU,UAAU,KAAK,KAAK;AAGpC,gBAAM,WAAW,QAAQ,MAAM,IAAI;AAEnC,gBAAM,KAAK,GAAG,QAAQ;AAAA,QAC1B,WAAW,SAAS,SAAS,YAAY;AACrC,gBAAM,YAAY,oBAAoB,GAAG,SAAS,EAAE,IAAI;AACxD,cAAI,WAAW;AACX,kBAAM,UAAU,UAAU,KAAK,KAAK;AAGpC,kBAAMA,SAAQ,QACT,MAAM,IAAI,EACV,OAAyB,CAAC,MAAM,YAAY;AACzC,kBAAI,KAAK,WAAW;AAChB,uBAAO,CAAC,UAAU,OAAO,CAAC;AAAA;AAE1B,uBAAO;AAAA,kBACH,GAAG;AAAA,kBACH;AAAA,oBACI,MAAM;AAAA,kBACV;AAAA,kBACA,UAAU,OAAO;AAAA,gBACrB;AAAA,YACR,GAAG,CAAC,CAAC;AAET,YAAAA,OAAM;AAAA,cAAQ,CAAC,SACX,kBAAkB,UAAU,SAAS,CAAC,IAAI,CAAC;AAAA,YAC/C;AAAA,UACJ,OAAO;AAEH,kBAAM,UAAU,UAAU,KAAK,KAAK;AACpC,gBAAI,CAAC,KAAK,UAAU;AAChB,kBAAI,aAAa;AACb,gBAAC,YAAwB,UAAU,KAAK;AAAA,kBACpC,QAAQ,KAAK;AAAA,kBACb,MAAM;AAAA,kBACN,QAAQ;AAAA,gBACZ,CAAC;AAAA,cACL;AAAA,YACJ;AAAA,UACJ;AAAA,QACJ;AAAA,MACJ;AAAA,IACJ;AAEA,QAAI,mBAAmB;AACnB,mBAAa,mBAAmB,cAAc,KAAK,CAAC;AAAA,IACxD;AAEA,aAAS,WAAW,SAAS,OAAO,GAAG;AACnC,WAAK,QAAQ,KAAK,OAAO;AAAA,IAC7B;AAEA,WAAO;AAAA,EACX;AACJ;",
+  "sourcesContent": ["import { z } from 'zod';\nimport {\n    Chapter,\n    ChapterContent,\n    ParseTree,\n    Verse,\n    VerseContent,\n} from './types.js';\nimport { getBookId, parseVerseReference, VerseRef } from '../utils.js';\nimport { ref } from 'process';\n\nexport const chapterHeadingSchema = z.object({\n    type: z.literal('chapter-heading'),\n    data: z.object({\n        chapter: z.coerce.number().int(),\n    }),\n});\n\nexport const verseSchema = z.object({\n    type: z.literal('text'),\n    id: z.string(),\n    bookCode: z.string().optional(),\n    chapter: z.number().optional(),\n    verse: z.number().optional(),\n    data: z\n        .object({\n            globalReferences: z.array(z.string()).optional(),\n        })\n        .optional(),\n});\n\nexport const paratextSchema = z.object({\n    type: z.literal('paratext'),\n    id: z.string(),\n});\n\nexport const metadataSchema = z.discriminatedUnion('type', [\n    chapterHeadingSchema,\n    verseSchema,\n    paratextSchema,\n]);\n\nexport const codexSchema = z.object({\n    cells: z.array(\n        z.object({\n            kind: z.number(),\n            languageId: z.string(),\n            value: z.string(),\n            metadata: z\n                .object({\n                    type: z.string(),\n                })\n                .passthrough()\n                .nullable()\n                .optional(),\n        })\n    ),\n});\n\n/**\n * Defines a parser for codex files.\n */\nexport class CodexParser {\n    // TODO:\n    // /**\n    //  * Whether to preserve markdown in the parsed content.\n    //  */\n    // preserveMarkdown: boolean = true;\n\n    private _noteCounter: number = 0;\n\n    private _parser: DOMParser;\n\n    constructor(parser: DOMParser) {\n        this._parser = parser;\n    }\n\n    /**\n     * Parses the specified codex content.\n     *\n     * @param codex The codex content to parse.\n     * @returns The parse tree that was generated.\n     */\n    public parse(codex: string): ParseTree {\n        const json = JSON.parse(codex);\n        const data = codexSchema.parse(json);\n\n        let root: ParseTree = {\n            type: 'root',\n            content: [],\n        };\n\n        let chapters: Map<number, Chapter> = new Map();\n        let lastChapter: Chapter | null = null;\n\n        function ensureChapter(chapterNumber: number) {\n            let chapter = chapters.get(chapterNumber);\n            if (!chapter) {\n                chapter = {\n                    type: 'chapter',\n                    number: chapterNumber,\n                    content: [],\n                    footnotes: [],\n                };\n                lastChapter = chapter;\n                chapters.set(chapterNumber, chapter);\n            }\n\n            return chapter;\n        }\n\n        function getDocumentContent(doc: Document): string {\n            let content = '';\n            for (let element of doc.childNodes) {\n                content += element.textContent;\n            }\n            return content;\n        }\n\n        function addChapterContent(\n            chapterNumber: number,\n            content: ChapterContent[]\n        ) {\n            const chapter = ensureChapter(chapterNumber);\n            chapter.content.push(...content);\n        }\n\n        function addChapterFootnote(\n            chapterNumber: number,\n            footnote: Chapter['footnotes'][0]\n        ) {\n            const chapter = ensureChapter(chapterNumber);\n            chapter.footnotes.push(footnote);\n        }\n\n        function addReference(ref: VerseRef, content: Verse['content']) {\n            addChapterContent(ref.chapter, [\n                {\n                    type: 'verse',\n                    number: ref.verse,\n                    content: content,\n                },\n            ]);\n        }\n\n        function stripHTML(value: string) {\n            return value.replace(/<[^>]*>/g, '');\n        }\n\n        function toHeading(content: string) {\n            return {\n                type: 'heading',\n                content: [content],\n            } satisfies ChapterContent;\n        }\n\n        function addLineBreaks(lines: Verse['content']) {\n            return lines.reduce(\n                (prev, current) => {\n                    if (typeof current === 'string' && current.trim() === '') {\n                        return prev; // skip empty lines\n                    }\n                    if (prev.length === 0) {\n                        return [current];\n                    } else {\n                        return [\n                            ...prev,\n                            {\n                                lineBreak: true,\n                            },\n                            current,\n                        ] satisfies Verse['content'];\n                    }\n                },\n                [] as Verse['content']\n            );\n        }\n\n        let previousReference: VerseRef | null = null;\n        let lines: Verse['content'] = [];\n\n        for (let cell of data.cells) {\n            if (cell.languageId === 'html' && cell.value) {\n                if (!cell.metadata) continue; // ignore html cells without metadata\n\n                const metadataResult = metadataSchema.safeParse(cell.metadata);\n\n                if (!metadataResult.success) continue; // ignore cells with invalid/unknown metadata\n                const metadata = metadataResult.data;\n                if (metadata.type === 'text') {\n                    let reference = parseVerseReference(metadata.id);\n                    if (!reference) {\n                        if (metadata.bookCode && metadata.chapter) {\n                            const bookId = getBookId(metadata.bookCode);\n                            if (!bookId) {\n                                throw new Error(\n                                    'Could not find book ID for code: ' +\n                                        metadata.bookCode\n                                );\n                            }\n\n                            if (metadata.verse) {\n                                reference = {\n                                    book: bookId,\n                                    chapter: metadata.chapter,\n                                    verse: metadata.verse,\n                                };\n                            } else if (previousReference?.verse) {\n                                if (\n                                    previousReference.chapter !==\n                                        metadata.chapter ||\n                                    previousReference.book !== bookId\n                                ) {\n                                    throw new Error(\n                                        'Cannot infer verse number for non-consecutive verse reference.'\n                                    );\n                                }\n\n                                // Sometimes codex doesn't include verse numbers for every cell, so we infer it from the previous reference\n                                reference = {\n                                    book: bookId,\n                                    chapter: metadata.chapter,\n                                    verse: previousReference.verse,\n                                };\n                            } else {\n                                throw new Error(\n                                    'Could not find verse reference in metadata.'\n                                );\n                            }\n                        } else if (metadata.data?.globalReferences?.length) {\n                            reference = parseVerseReference(\n                                metadata.data.globalReferences[0]\n                            );\n                            if (!reference) {\n                                throw new Error(\n                                    'Could not find verse reference in globalReferences metadata.'\n                                );\n                            }\n                        } else {\n                            throw new Error(\n                                'Could not find verse reference in metadata.'\n                            );\n                        }\n                    }\n\n                    if (\n                        previousReference &&\n                        (reference.book !== previousReference.book ||\n                            reference.chapter !== previousReference.chapter ||\n                            reference.verse !== previousReference.verse)\n                    ) {\n                        addReference(previousReference, addLineBreaks(lines));\n                        lines = [];\n                    }\n\n                    previousReference = reference;\n\n                    if (!root.id) root.id = reference.book; // set book id if not already set\n\n                    const doc = this._parser.parseFromString(\n                        cell.value,\n                        'text/html'\n                    );\n\n                    for (let element of doc.children) {\n                        const allFootnotes = element.querySelectorAll(\n                            'sup.footnote-marker[data-footnote]'\n                        );\n                        for (const footnote of allFootnotes) {\n                            const footnoteData =\n                                footnote.getAttribute('data-footnote');\n                            if (footnoteData) {\n                                const footnoteDoc =\n                                    this._parser.parseFromString(\n                                        footnoteData,\n                                        'text/html'\n                                    );\n                                const footnoteText =\n                                    getDocumentContent(footnoteDoc);\n                                if (footnoteText) {\n                                    addChapterFootnote(reference.chapter, {\n                                        noteId: this._noteCounter++,\n                                        text: footnoteText,\n                                        caller: null,\n                                    });\n                                }\n                            }\n                            footnote.remove();\n                        }\n                    }\n\n                    const content = getDocumentContent(doc);\n\n                    if (content) {\n                        // add line breaks in heading if there are multiple lines\n                        const newLines = content.split('\\n');\n\n                        lines.push(...newLines);\n                    }\n                } else if (metadata.type === 'paratext') {\n                    const reference = parseVerseReference(`${metadata.id}:1`); // chapter headings do not have a verse reference always default to 1\n                    if (reference) {\n                        const doc = this._parser.parseFromString(\n                            cell.value,\n                            'text/html'\n                        );\n                        const content = getDocumentContent(doc);\n\n                        if (!content) {\n                            continue;\n                        }\n\n                        // add line breaks in heading if there are multiple lines\n                        const lines = content\n                            .split('\\n')\n                            .reduce<ChapterContent[]>((prev, current) => {\n                                if (prev.length === 0)\n                                    return [toHeading(current)];\n                                else\n                                    return [\n                                        ...prev,\n                                        {\n                                            type: 'line_break',\n                                        },\n                                        toHeading(current),\n                                    ] satisfies ChapterContent[];\n                            }, []);\n\n                        lines.forEach((line) =>\n                            addChapterContent(reference.chapter, [line])\n                        );\n                    } else {\n                        const doc = this._parser.parseFromString(\n                            cell.value,\n                            'text/html'\n                        );\n                        const content = getDocumentContent(doc);\n                        // parse paratext that isn't a chapter reference as footnote\n                        // const content = stripHTML(cell.value);\n                        if (content && !cell.metadata) {\n                            if (lastChapter) {\n                                (lastChapter as Chapter).footnotes.push({\n                                    noteId: this._noteCounter++,\n                                    text: content,\n                                    caller: null,\n                                });\n                            }\n                        }\n                    }\n                }\n            }\n        }\n\n        if (previousReference) {\n            addReference(previousReference, addLineBreaks(lines));\n        }\n\n        for (let chapter of chapters.values()) {\n            root.content.push(chapter);\n        }\n\n        return root;\n    }\n}\n"],
+  "mappings": ";AAAA,SAAS,SAAS;AAQlB,SAAS,WAAW,2BAAqC;AAGlD,aAAM,uBAAuB,EAAE,OAAO;AAAA,EACzC,MAAM,EAAE,QAAQ,iBAAiB;AAAA,EACjC,MAAM,EAAE,OAAO;AAAA,IACX,SAAS,EAAE,OAAO,OAAO,EAAE,IAAI;AAAA,EACnC,CAAC;AACL,CAAC;AAEM,aAAM,cAAc,EAAE,OAAO;AAAA,EAChC,MAAM,EAAE,QAAQ,MAAM;AAAA,EACtB,IAAI,EAAE,OAAO;AAAA,EACb,UAAU,EAAE,OAAO,EAAE,SAAS;AAAA,EAC9B,SAAS,EAAE,OAAO,EAAE,SAAS;AAAA,EAC7B,OAAO,EAAE,OAAO,EAAE,SAAS;AAAA,EAC3B,MAAM,EACD,OAAO;AAAA,IACJ,kBAAkB,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,SAAS;AAAA,EACnD,CAAC,EACA,SAAS;AAClB,CAAC;AAEM,aAAM,iBAAiB,EAAE,OAAO;AAAA,EACnC,MAAM,EAAE,QAAQ,UAAU;AAAA,EAC1B,IAAI,EAAE,OAAO;AACjB,CAAC;AAEM,aAAM,iBAAiB,EAAE,mBAAmB,QAAQ;AAAA,EACvD;AAAA,EACA;AAAA,EACA;AACJ,CAAC;AAEM,aAAM,cAAc,EAAE,OAAO;AAAA,EAChC,OAAO,EAAE;AAAA,IACL,EAAE,OAAO;AAAA,MACL,MAAM,EAAE,OAAO;AAAA,MACf,YAAY,EAAE,OAAO;AAAA,MACrB,OAAO,EAAE,OAAO;AAAA,MAChB,UAAU,EACL,OAAO;AAAA,QACJ,MAAM,EAAE,OAAO;AAAA,MACnB,CAAC,EACA,YAAY,EACZ,SAAS,EACT,SAAS;AAAA,IAClB,CAAC;AAAA,EACL;AACJ,CAAC;AAKM,aAAM,YAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOb,eAAuB;AAAA,EAEvB;AAAA,EAER,YAAY,QAAmB;AAC3B,SAAK,UAAU;AAAA,EACnB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQO,MAAM,OAA0B;AACnC,UAAM,OAAO,KAAK,MAAM,KAAK;AAC7B,UAAM,OAAO,YAAY,MAAM,IAAI;AAEnC,QAAI,OAAkB;AAAA,MAClB,MAAM;AAAA,MACN,SAAS,CAAC;AAAA,IACd;AAEA,QAAI,WAAiC,oBAAI,IAAI;AAC7C,QAAI,cAA8B;AAElC,aAAS,cAAc,eAAuB;AAC1C,UAAI,UAAU,SAAS,IAAI,aAAa;AACxC,UAAI,CAAC,SAAS;AACV,kBAAU;AAAA,UACN,MAAM;AAAA,UACN,QAAQ;AAAA,UACR,SAAS,CAAC;AAAA,UACV,WAAW,CAAC;AAAA,QAChB;AACA,sBAAc;AACd,iBAAS,IAAI,eAAe,OAAO;AAAA,MACvC;AAEA,aAAO;AAAA,IACX;AAEA,aAAS,mBAAmB,KAAuB;AAC/C,UAAI,UAAU;AACd,eAAS,WAAW,IAAI,YAAY;AAChC,mBAAW,QAAQ;AAAA,MACvB;AACA,aAAO;AAAA,IACX;AAEA,aAAS,kBACL,eACA,SACF;AACE,YAAM,UAAU,cAAc,aAAa;AAC3C,cAAQ,QAAQ,KAAK,GAAG,OAAO;AAAA,IACnC;AAEA,aAAS,mBACL,eACA,UACF;AACE,YAAM,UAAU,cAAc,aAAa;AAC3C,cAAQ,UAAU,KAAK,QAAQ;AAAA,IACnC;AAEA,aAAS,aAAaA,MAAe,SAA2B;AAC5D,wBAAkBA,KAAI,SAAS;AAAA,QAC3B;AAAA,UACI,MAAM;AAAA,UACN,QAAQA,KAAI;AAAA,UACZ;AAAA,QACJ;AAAA,MACJ,CAAC;AAAA,IACL;AAEA,aAAS,UAAU,OAAe;AAC9B,aAAO,MAAM,QAAQ,YAAY,EAAE;AAAA,IACvC;AAEA,aAAS,UAAU,SAAiB;AAChC,aAAO;AAAA,QACH,MAAM;AAAA,QACN,SAAS,CAAC,OAAO;AAAA,MACrB;AAAA,IACJ;AAEA,aAAS,cAAcC,QAAyB;AAC5C,aAAOA,OAAM;AAAA,QACT,CAAC,MAAM,YAAY;AACf,cAAI,OAAO,YAAY,YAAY,QAAQ,KAAK,MAAM,IAAI;AACtD,mBAAO;AAAA,UACX;AACA,cAAI,KAAK,WAAW,GAAG;AACnB,mBAAO,CAAC,OAAO;AAAA,UACnB,OAAO;AACH,mBAAO;AAAA,cACH,GAAG;AAAA,cACH;AAAA,gBACI,WAAW;AAAA,cACf;AAAA,cACA;AAAA,YACJ;AAAA,UACJ;AAAA,QACJ;AAAA,QACA,CAAC;AAAA,MACL;AAAA,IACJ;AAEA,QAAI,oBAAqC;AACzC,QAAI,QAA0B,CAAC;AAE/B,aAAS,QAAQ,KAAK,OAAO;AACzB,UAAI,KAAK,eAAe,UAAU,KAAK,OAAO;AAC1C,YAAI,CAAC,KAAK,SAAU;AAEpB,cAAM,iBAAiB,eAAe,UAAU,KAAK,QAAQ;AAE7D,YAAI,CAAC,eAAe,QAAS;AAC7B,cAAM,WAAW,eAAe;AAChC,YAAI,SAAS,SAAS,QAAQ;AAC1B,cAAI,YAAY,oBAAoB,SAAS,EAAE;AAC/C,cAAI,CAAC,WAAW;AACZ,gBAAI,SAAS,YAAY,SAAS,SAAS;AACvC,oBAAM,SAAS,UAAU,SAAS,QAAQ;AAC1C,kBAAI,CAAC,QAAQ;AACT,sBAAM,IAAI;AAAA,kBACN,sCACI,SAAS;AAAA,gBACjB;AAAA,cACJ;AAEA,kBAAI,SAAS,OAAO;AAChB,4BAAY;AAAA,kBACR,MAAM;AAAA,kBACN,SAAS,SAAS;AAAA,kBAClB,OAAO,SAAS;AAAA,gBACpB;AAAA,cACJ,WAAW,mBAAmB,OAAO;AACjC,oBACI,kBAAkB,YACd,SAAS,WACb,kBAAkB,SAAS,QAC7B;AACE,wBAAM,IAAI;AAAA,oBACN;AAAA,kBACJ;AAAA,gBACJ;AAGA,4BAAY;AAAA,kBACR,MAAM;AAAA,kBACN,SAAS,SAAS;AAAA,kBAClB,OAAO,kBAAkB;AAAA,gBAC7B;AAAA,cACJ,OAAO;AACH,sBAAM,IAAI;AAAA,kBACN;AAAA,gBACJ;AAAA,cACJ;AAAA,YACJ,WAAW,SAAS,MAAM,kBAAkB,QAAQ;AAChD,0BAAY;AAAA,gBACR,SAAS,KAAK,iBAAiB,CAAC;AAAA,cACpC;AACA,kBAAI,CAAC,WAAW;AACZ,sBAAM,IAAI;AAAA,kBACN;AAAA,gBACJ;AAAA,cACJ;AAAA,YACJ,OAAO;AACH,oBAAM,IAAI;AAAA,gBACN;AAAA,cACJ;AAAA,YACJ;AAAA,UACJ;AAEA,cACI,sBACC,UAAU,SAAS,kBAAkB,QAClC,UAAU,YAAY,kBAAkB,WACxC,UAAU,UAAU,kBAAkB,QAC5C;AACE,yBAAa,mBAAmB,cAAc,KAAK,CAAC;AACpD,oBAAQ,CAAC;AAAA,UACb;AAEA,8BAAoB;AAEpB,cAAI,CAAC,KAAK,GAAI,MAAK,KAAK,UAAU;AAElC,gBAAM,MAAM,KAAK,QAAQ;AAAA,YACrB,KAAK;AAAA,YACL;AAAA,UACJ;AAEA,mBAAS,WAAW,IAAI,UAAU;AAC9B,kBAAM,eAAe,QAAQ;AAAA,cACzB;AAAA,YACJ;AACA,uBAAW,YAAY,cAAc;AACjC,oBAAM,eACF,SAAS,aAAa,eAAe;AACzC,kBAAI,cAAc;AACd,sBAAM,cACF,KAAK,QAAQ;AAAA,kBACT;AAAA,kBACA;AAAA,gBACJ;AACJ,sBAAM,eACF,mBAAmB,WAAW;AAClC,oBAAI,cAAc;AACd,qCAAmB,UAAU,SAAS;AAAA,oBAClC,QAAQ,KAAK;AAAA,oBACb,MAAM;AAAA,oBACN,QAAQ;AAAA,kBACZ,CAAC;AAAA,gBACL;AAAA,cACJ;AACA,uBAAS,OAAO;AAAA,YACpB;AAAA,UACJ;AAEA,gBAAM,UAAU,mBAAmB,GAAG;AAEtC,cAAI,SAAS;AAET,kBAAM,WAAW,QAAQ,MAAM,IAAI;AAEnC,kBAAM,KAAK,GAAG,QAAQ;AAAA,UAC1B;AAAA,QACJ,WAAW,SAAS,SAAS,YAAY;AACrC,gBAAM,YAAY,oBAAoB,GAAG,SAAS,EAAE,IAAI;AACxD,cAAI,WAAW;AACX,kBAAM,MAAM,KAAK,QAAQ;AAAA,cACrB,KAAK;AAAA,cACL;AAAA,YACJ;AACA,kBAAM,UAAU,mBAAmB,GAAG;AAEtC,gBAAI,CAAC,SAAS;AACV;AAAA,YACJ;AAGA,kBAAMA,SAAQ,QACT,MAAM,IAAI,EACV,OAAyB,CAAC,MAAM,YAAY;AACzC,kBAAI,KAAK,WAAW;AAChB,uBAAO,CAAC,UAAU,OAAO,CAAC;AAAA;AAE1B,uBAAO;AAAA,kBACH,GAAG;AAAA,kBACH;AAAA,oBACI,MAAM;AAAA,kBACV;AAAA,kBACA,UAAU,OAAO;AAAA,gBACrB;AAAA,YACR,GAAG,CAAC,CAAC;AAET,YAAAA,OAAM;AAAA,cAAQ,CAAC,SACX,kBAAkB,UAAU,SAAS,CAAC,IAAI,CAAC;AAAA,YAC/C;AAAA,UACJ,OAAO;AACH,kBAAM,MAAM,KAAK,QAAQ;AAAA,cACrB,KAAK;AAAA,cACL;AAAA,YACJ;AACA,kBAAM,UAAU,mBAAmB,GAAG;AAGtC,gBAAI,WAAW,CAAC,KAAK,UAAU;AAC3B,kBAAI,aAAa;AACb,gBAAC,YAAwB,UAAU,KAAK;AAAA,kBACpC,QAAQ,KAAK;AAAA,kBACb,MAAM;AAAA,kBACN,QAAQ;AAAA,gBACZ,CAAC;AAAA,cACL;AAAA,YACJ;AAAA,UACJ;AAAA,QACJ;AAAA,MACJ;AAAA,IACJ;AAEA,QAAI,mBAAmB;AACnB,mBAAa,mBAAmB,cAAc,KAAK,CAAC;AAAA,IACxD;AAEA,aAAS,WAAW,SAAS,OAAO,GAAG;AACnC,WAAK,QAAQ,KAAK,OAAO;AAAA,IAC7B;AAEA,WAAO;AAAA,EACX;AACJ;",
   "names": ["ref", "lines"]
 }

package/dist/types/parser/codex-parser.d.ts

@@ -26,17 +26,30 @@ export declare const verseSchema: z.ZodObject<{
     bookCode: z.ZodOptional<z.ZodString>;
     chapter: z.ZodOptional<z.ZodNumber>;
     verse: z.ZodOptional<z.ZodNumber>;
+    data: z.ZodOptional<z.ZodObject<{
+        globalReferences: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        globalReferences?: string[] | undefined;
+    }, {
+        globalReferences?: string[] | undefined;
+    }>>;
 }, "strip", z.ZodTypeAny, {
     id: string;
     type: "text";
     chapter?: number | undefined;
     verse?: number | undefined;
+    data?: {
+        globalReferences?: string[] | undefined;
+    } | undefined;
     bookCode?: string | undefined;
 }, {
     id: string;
     type: "text";
     chapter?: number | undefined;
     verse?: number | undefined;
+    data?: {
+        globalReferences?: string[] | undefined;
+    } | undefined;
     bookCode?: string | undefined;
 }>;
 export declare const paratextSchema: z.ZodObject<{
@@ -74,17 +87,30 @@ export declare const metadataSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObjec
     bookCode: z.ZodOptional<z.ZodString>;
     chapter: z.ZodOptional<z.ZodNumber>;
     verse: z.ZodOptional<z.ZodNumber>;
+    data: z.ZodOptional<z.ZodObject<{
+        globalReferences: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        globalReferences?: string[] | undefined;
+    }, {
+        globalReferences?: string[] | undefined;
+    }>>;
 }, "strip", z.ZodTypeAny, {
     id: string;
     type: "text";
     chapter?: number | undefined;
     verse?: number | undefined;
+    data?: {
+        globalReferences?: string[] | undefined;
+    } | undefined;
     bookCode?: string | undefined;
 }, {
     id: string;
     type: "text";
     chapter?: number | undefined;
     verse?: number | undefined;
+    data?: {
+        globalReferences?: string[] | undefined;
+    } | undefined;
     bookCode?: string | undefined;
 }>, z.ZodObject<{
     type: z.ZodLiteral<"paratext">;
@@ -147,6 +173,8 @@ export declare const codexSchema: z.ZodObject<{
  */
 export declare class CodexParser {
     private _noteCounter;
+    private _parser;
+    constructor(parser: DOMParser);
     /**
      * Parses the specified codex content.
      *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helloao/tools",
-  "version": "0.0.19",
+  "version": "0.1.0",
   "description": "A set of tools for managing HelloAO's Free Bible API",
   "main": "./dist/cjs/index.cjs",
   "module": "./dist/esm/index.js",