@helloao/tools 0.1.0 → 0.3.0

package/dist/esm/parser/usfm-parser.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../parser/usfm-parser.ts"],
-  "sourcesContent": ["import { findLast } from 'lodash';\nimport {\n    ParseTree,\n    Chapter,\n    Verse,\n    HebrewSubtitle,\n    FootnoteReference,\n    Footnote,\n    Text,\n} from './types.js';\nimport { getLogger } from '../log.js';\n\n/**\n * Defines a class that can tokenize a stream of characters into tokens.\n */\nexport class UsfmTokenizer {\n    private _input: string = '';\n    private _index: number = 0;\n    private _start: number = 0;\n\n    private get _tokenLength() {\n        return this._index - this._start;\n    }\n\n    /**\n     * Converts the given input into a list of tokens.\n     * @param input The input that should be tokenized.\n     */\n    tokenize(input: string): SimpleToken[] {\n        this._input = input;\n        this._index = 0;\n\n        return this._parseTokens();\n    }\n\n    private _parseTokens() {\n        let tokens: SimpleToken[] = [];\n        let token = this._parseToken();\n        while (token) {\n            tokens.push(token);\n            token = this._parseToken();\n        }\n\n        return tokens;\n    }\n\n    private _parseToken(): SimpleToken | null {\n        let state:\n            | 'none'\n            | 'marker_start'\n            | 'marker_number'\n            | 'whitespace'\n            | 'word' = 'none';\n        let kind: T | null = null;\n        this._start = this._index;\n\n        while (this._index < this._input.length) {\n            const codePointNumber = this._input.codePointAt(this._index);\n\n            if (typeof codePointNumber === 'undefined') {\n                throw new Error('Unable to get code point!');\n            }\n\n            const codePoint = String.fromCodePoint(codePointNumber);\n\n            if (state === 'none') {\n                if (codePoint === '\\\\') {\n                    state = 'marker_start';\n                } else if (isWhitespace(codePoint)) {\n                    state = 'whitespace';\n                } else {\n                    state = 'word';\n                }\n            } else if (state === 'marker_start') {\n                if (isDigit(codePoint)) {\n                    if (this._tokenLength === 0) {\n                        throw new Error(\n                            'Invalid Marker: Markers must not contain only digits.'\n                        );\n                    }\n                    state = 'marker_number';\n                } else if (codePoint === '*') {\n                    this._index += codePoint.length;\n                    kind = 'marker';\n                    break;\n                } else if (isWhitespace(codePoint)) {\n                    kind = 'marker';\n                    break;\n                }\n            } else if (state === 'marker_number') {\n                if (codePoint === '*') {\n                    this._index += codePoint.length;\n                    kind = 'marker';\n                    break;\n                } else if (!isDigit(codePoint)) {\n                    kind = 'marker';\n                    break;\n                }\n            } else if (state === 'whitespace') {\n                if (!isWhitespace(codePoint)) {\n                    kind = 'whitespace';\n                    break;\n                }\n            } else if (state === 'word') {\n                if (isWhitespace(codePoint) || codePoint === '\\\\') {\n                    kind = 'word';\n                    break;\n                }\n            }\n\n            this._index += codePoint.length;\n        }\n\n        if (!kind) {\n            if (this._index >= this._input.length) {\n                if (state == 'marker_start' || state === 'marker_number') {\n                    kind = 'marker';\n                } else if (state === 'word') {\n                    kind = 'word';\n                } else if (state === 'whitespace') {\n                    kind = 'whitespace';\n                }\n            }\n        }\n\n        if (kind) {\n            return t(loc(this._start, this._index), kind);\n        }\n\n        return null;\n    }\n}\n\nexport interface UsfmParseOptions {\n    paragraphs: Set<string>;\n}\n\n/**\n * Defines a USFM Parser.\n */\nexport class UsfmParser {\n    private _poem: number | null = null;\n    private _wordsOfJesus: boolean = false;\n\n    tokenize(input: string): Token[] {\n        const simpleTokens = new UsfmTokenizer().tokenize(input);\n        let tokens: Token[] = [];\n\n        for (let t of simpleTokens) {\n            if (t.kind === 'marker') {\n                let source = input.substring(t.loc.start, t.loc.end);\n                const isEnd = source.endsWith('*');\n\n                if (isEnd) {\n                    source = source.substring(0, source.length - 1);\n                }\n\n                let numberIndex = -1;\n                for (let i = 0; i < source.length; i++) {\n                    if (isDigit(source[i])) {\n                        numberIndex = i;\n                        break;\n                    }\n                }\n\n                let number: number | null = null;\n                if (numberIndex === 1) {\n                    throw new Error(\n                        'Markers must not be made only of numbers!'\n                    );\n                }\n                if (numberIndex > 0) {\n                    number = parseInt(source.substring(numberIndex));\n                    source = source.substring(0, numberIndex);\n                }\n\n                if (source.length === 1) {\n                    if (isEnd) {\n                        // Ending marker does not have a command.\n                        // We should look for a matching start marker.\n                        const startMarker = findLast(\n                            tokens,\n                            (t) => t.kind === 'marker' && t.type === 'start'\n                        ) as MarkerToken;\n                        if (startMarker) {\n                            source = startMarker.command;\n                        }\n                    }\n\n                    if (source.length === 1) {\n                        throw new Error(\n                            `Markers must have a command! Token: ${t.loc.start}-${t.loc.end}`\n                        );\n                    }\n                }\n\n                tokens.push(\n                    marker(t.loc, source, number, isEnd ? 'end' : 'start')\n                );\n            } else if (t.kind === 'whitespace') {\n                let source = input.substring(t.loc.start, t.loc.end);\n                tokens.push(whitespace(t.loc, source));\n            } else if (t.kind === 'word') {\n                let source = input.substring(t.loc.start, t.loc.end);\n                tokens.push(word(t.loc, source));\n            }\n        }\n\n        return tokens;\n    }\n\n    parse(input: string): ParseTree {\n        let root: ParseTree = {\n            type: 'root',\n            content: [],\n        };\n\n        const tokens = this.tokenize(input);\n\n        let expectingId = 0;\n        let expectingName = 0;\n        let expectingTitle = 0;\n        let expectingSectionHeading = 0;\n        let expectingFootnote = 0;\n        let expectingFootnoteReference = 0;\n        let expectingFootnoteText = 0;\n        let expectingReferenceText = 0;\n        let expectingWordAttribute = 0;\n        let expectingNestedWordAttribute = 0;\n        let expectingWordsOfJesus = 0;\n        let expectingIntroParagraph = 0;\n        let expectingCrossReference = 0;\n        let expectingUnknownCommand = 0;\n\n        let canParseFootnotes = true;\n        let chapter: Chapter | null = null;\n        let lastVerse: Verse | null = null;\n        let verse: Verse | null = null;\n        let subtitle: HebrewSubtitle | null = null;\n        let words: string[] = [];\n        let verseContent: (Text | FootnoteReference | string)[] = [];\n        let sectionContent: string = '';\n        let currentFootnoteId = 0;\n        let footnote: Footnote | null = null;\n\n        this._poem = null;\n\n        const addWordsToVerseOrSubtitle = () => {\n            if (words.length > 0) {\n                const text = this._text(words.join('').trimEnd());\n                if (verse) {\n                    verse.content.push(text);\n                } else if (subtitle) {\n                    subtitle.content.push(text);\n                } else {\n                    verseContent.push(text);\n                }\n                words = [];\n            }\n        };\n\n        const addVerseContentToChapter = (token: Token | null) => {\n            if (!chapter) {\n                return;\n            }\n            if (verseContent.length > 0) {\n                if (chapter.content.some((c) => c.type === 'verse')) {\n                    this._throwError(\n                        input,\n                        token,\n                        'Cannot infer first verse after other verses have been added to the chapter!'\n                    );\n                }\n                // Implicit first verse\n                verse = {\n                    type: 'verse',\n                    number: 1,\n                    content: verseContent,\n                };\n                chapter.content.push(verse);\n                verseContent = [];\n            }\n        };\n\n        const cleanupVerse = () => {\n            if (!verse || !chapter) {\n                return;\n            }\n            let chapterContent: Chapter['content'] = [];\n            for (let i = verse.content.length - 1; i >= 0; i--) {\n                let content = verse.content[i];\n                if (typeof content === 'object' && 'heading' in content) {\n                    // move headings that occur at the end of a verse to the chapter\n                    chapterContent.unshift({\n                        type: 'heading',\n                        content: [content.heading],\n                    });\n                    verse.content.splice(i, 1);\n                } else if (\n                    typeof content === 'object' &&\n                    'lineBreak' in content &&\n                    content.lineBreak\n                ) {\n                    // move line breaks that occur at the end of a verse to the chapter\n                    chapterContent.unshift({\n                        type: 'line_break',\n                    });\n                    verse.content.splice(i, 1);\n                } else {\n                    break;\n                }\n            }\n\n            for (let content of chapterContent) {\n                chapter.content.push(content);\n            }\n        };\n\n        const completeVerseOrSubtitle = (token: Token | null) => {\n            if (verse && isNaN(verse.number)) {\n                // Verse is invalid for some reason.\n                const index = chapter!.content.indexOf(verse);\n                if (index >= 0) {\n                    chapter!.content.splice(index, 1);\n                }\n                verse = null;\n            }\n            if (verse || subtitle) {\n                addWordsToVerseOrSubtitle();\n            }\n\n            addVerseContentToChapter(token);\n            cleanupVerse();\n        };\n\n        const completeSection = () => {\n            if (expectingSectionHeading > 0) {\n                if (verse) {\n                    addWordsToVerseOrSubtitle();\n                    verse.content.push({\n                        heading: sectionContent,\n                    });\n                } else if (chapter) {\n                    chapter.content.push({\n                        type: 'heading',\n                        content: [sectionContent],\n                    });\n                } else {\n                    root.content.push({\n                        type: 'heading',\n                        content: [sectionContent],\n                    });\n                }\n                sectionContent = '';\n                expectingSectionHeading = 0;\n            }\n        };\n\n        const addWordsToFootnote = () => {\n            if (footnote && words.length > 0) {\n                footnote.text += words.join(' ');\n                words = [];\n            }\n        };\n\n        for (let token of tokens) {\n            if (token.kind === 'marker') {\n                if (token.command === '\\\\c') {\n                    addWordsToVerseOrSubtitle();\n                    cleanupVerse();\n\n                    chapter = {\n                        type: 'chapter',\n                        number: NaN,\n                        content: [],\n                        footnotes: [],\n                    };\n                    verse = null;\n                    verseContent = [];\n\n                    root.content.push(chapter);\n                } else if (token.command === '\\\\v') {\n                    if (!chapter) {\n                        this._throwError(\n                            input,\n                            token,\n                            'Cannot parse a verse without chapter information!'\n                        );\n                    } else {\n                        completeSection();\n                        completeVerseOrSubtitle(token);\n\n                        lastVerse = verse;\n                        verse = {\n                            type: 'verse',\n                            number: NaN,\n                            content: [],\n                        };\n\n                        chapter.content.push(verse);\n                    }\n                } else if (token.command === '\\\\d') {\n                    if (!chapter) {\n                        this._throwError(\n                            input,\n                            token,\n                            'Cannot parse a hebrew subtitle without chapter information!'\n                        );\n                    } else {\n                        completeVerseOrSubtitle(token);\n\n                        subtitle = {\n                            type: 'hebrew_subtitle',\n                            content: [],\n                        };\n\n                        chapter.content.push(subtitle);\n                    }\n                } else if (token.command === '\\\\b' || token.command === '\\\\p') {\n                    if (!chapter) {\n                        this._throwError(\n                            input,\n                            token,\n                            'Cannot parse a line break without chapter information!'\n                        );\n                    } else {\n                        if (verse) {\n                            addWordsToVerseOrSubtitle();\n                            verse.content.push({\n                                lineBreak: true,\n                            });\n                        } else {\n                            completeVerseOrSubtitle(token);\n                            chapter.content.push({\n                                type: 'line_break',\n                            });\n                        }\n                    }\n                } else if (token.command === '\\\\q') {\n                    addWordsToVerseOrSubtitle();\n                    this._poem = token.number;\n                } else if (token.command === '\\\\p') {\n                    addWordsToVerseOrSubtitle();\n                    this._poem = null;\n                } else if (token.command === '\\\\id') {\n                    expectingId = 1;\n                } else if (token.command === '\\\\h') {\n                    expectingName = 1;\n                    root.header = undefined;\n                } else if (\n                    token.command === '\\\\mt' ||\n                    token.command === '\\\\+mt'\n                ) {\n                    expectingTitle = 1;\n                } else if (token.command === '\\\\s') {\n                    expectingSectionHeading = 1;\n                } else if (token.command === '\\\\r') {\n                    expectingReferenceText = 1;\n                } else if (token.command === '\\\\f' && canParseFootnotes) {\n                    if (token.type === 'start') {\n                        if (!chapter) {\n                            this._throwError(\n                                input,\n                                token,\n                                'Cannot start a footnote outside of a chapter!',\n                                true\n                            );\n                        } else {\n                            addWordsToVerseOrSubtitle();\n                            footnote = {\n                                noteId: currentFootnoteId,\n                                text: '',\n                                caller: null,\n                            };\n                            const ref: FootnoteReference = {\n                                noteId: footnote.noteId,\n                            };\n                            expectingFootnote = 1;\n\n                            chapter.footnotes.push(footnote);\n\n                            if (verse) {\n                                verse.content.push(ref);\n                            } else if (subtitle) {\n                                subtitle.content.push(ref);\n                            } else {\n                                verseContent.push(ref);\n                            }\n\n                            currentFootnoteId += 1;\n                        }\n                    } else {\n                        addWordsToFootnote();\n                        expectingFootnote = 0;\n                        expectingFootnoteText = 0;\n                        expectingFootnoteReference = 0;\n                        footnote = null;\n                    }\n                } else if (token.command === '\\\\fr' && canParseFootnotes) {\n                    if (!footnote) {\n                        this._throwError(\n                            input,\n                            token,\n                            'Cannot start a footnote reference outside of a footnote!',\n                            true\n                        );\n                    } else {\n                        expectingFootnoteReference = 1;\n                    }\n                } else if (token.command === '\\\\ft' && canParseFootnotes) {\n                    if (!footnote) {\n                        this._throwError(\n                            input,\n                            token,\n                            'Cannot start footnote text outside of a footnote!',\n                            true\n                        );\n                    } else {\n                        expectingFootnoteText = 1;\n                    }\n                } else if (token.command === '\\\\w') {\n                    if (token.type === 'start') {\n                        expectingWordAttribute = 1;\n                    } else {\n                        expectingWordAttribute = 0;\n                    }\n                } else if (token.command === '\\\\+w') {\n                    if (token.type === 'start') {\n                        expectingNestedWordAttribute = 1;\n                    } else {\n                        expectingNestedWordAttribute = 0;\n                    }\n                } else if (\n                    token.command === '\\\\wj' ||\n                    token.command === '\\\\+wj'\n                ) {\n                    if (token.type === 'start') {\n                        addWordsToVerseOrSubtitle();\n                        this._wordsOfJesus = true;\n                    } else {\n                        addWordsToVerseOrSubtitle();\n                        this._wordsOfJesus = false;\n                    }\n                } else if (token.command === '\\\\ip') {\n                    expectingIntroParagraph = 1;\n                    canParseFootnotes = false;\n                } else if (token.command === '\\\\x') {\n                    if (token.type === 'start') {\n                        expectingCrossReference = 1;\n                    } else {\n                        expectingCrossReference = 0;\n                    }\n                } else if (token.command.indexOf('-') >= 0) {\n                    if (token.type === 'start') {\n                        expectingUnknownCommand = 1;\n                    }\n                } else if (token.type === 'end') {\n                    expectingUnknownCommand = 0;\n                }\n            } else if (token.kind === 'word') {\n                if (expectingId > 0) {\n                    if (expectingId === 1) {\n                        root.id = token.word;\n                        expectingId = 2;\n                    }\n                } else if (expectingName > 0) {\n                    if (root.header) {\n                        root.header += ' ' + token.word;\n                    } else {\n                        root.header = token.word;\n                    }\n                } else if (expectingTitle > 0) {\n                    if (root.title) {\n                        root.title += ' ' + token.word;\n                    } else {\n                        root.title = token.word;\n                    }\n                } else if (expectingSectionHeading > 0) {\n                    if (sectionContent) {\n                        sectionContent += ' ' + token.word;\n                    } else {\n                        sectionContent = token.word;\n                    }\n                } else if (expectingFootnoteReference > 0) {\n                    if ((expectingFootnoteReference = 1)) {\n                        const [chapter, verse] = token.word.split(/[\\.\\:]/);\n\n                        if (footnote) {\n                            footnote.reference = {\n                                chapter: parseInt(chapter),\n                                verse: parseInt(verse),\n                            };\n                        }\n\n                        expectingFootnoteReference = 0;\n                    }\n                } else if (expectingFootnoteText > 0) {\n                    words.push(token.word);\n                } else if (expectingFootnote > 0) {\n                    if (expectingFootnote === 1) {\n                        if (token.word) {\n                            if (footnote) {\n                                if (token.word === '+' || token.word !== '-') {\n                                    footnote.caller = token.word;\n                                } else {\n                                    footnote.caller = null;\n                                }\n                            }\n                            // this._throwError(input, token, 'Footnotes must use the \"+\" caller.');\n                        }\n                        expectingFootnote = 2;\n                    } else {\n                        words.push(token.word);\n                    }\n                } else if (expectingReferenceText > 0) {\n                    // Skip processing words for references\n                    // because references aren't included in the JSON format\n                    // (for now)\n                } else if (expectingCrossReference > 0) {\n                    // Skip processing words for cross references\n                } else if (chapter && isNaN(chapter.number)) {\n                    chapter.number = parseInt(token.word);\n                    if (isNaN(chapter.number)) {\n                        this._throwError(\n                            input,\n                            token,\n                            'The first word token after a chapter marker must be parsable to an integer!'\n                        );\n                    }\n                } else if (verse && isNaN(verse.number)) {\n                    verse.number = parseInt(token.word);\n                    if (isNaN(verse.number)) {\n                        this._throwError(\n                            input,\n                            token,\n                            'The first word token after a verse marker must be parsable to an integer!'\n                        );\n                    }\n                } else if (expectingWordAttribute > 0) {\n                    if (expectingWordAttribute === 1) {\n                        const firstVerticalBarIndex = token.word.indexOf('|');\n\n                        if (firstVerticalBarIndex >= 0) {\n                            const name = token.word.slice(\n                                0,\n                                firstVerticalBarIndex\n                            );\n                            // const rest = token.word.slice(firstVerticalBarIndex + '|'.length);\n                            words.push(name);\n                            expectingWordAttribute = 2;\n                        } else {\n                            words.push(token.word);\n                        }\n                    }\n                } else if (expectingNestedWordAttribute > 0) {\n                    if (expectingNestedWordAttribute === 1) {\n                        const firstVerticalBarIndex = token.word.indexOf('|');\n\n                        if (firstVerticalBarIndex >= 0) {\n                            const name = token.word.slice(\n                                0,\n                                firstVerticalBarIndex\n                            );\n                            // const rest = token.word.slice(firstVerticalBarIndex + '|'.length);\n                            words.push(name);\n                            expectingNestedWordAttribute = 2;\n                        } else {\n                            words.push(token.word);\n                        }\n                    }\n                } else if (expectingUnknownCommand > 0) {\n                } else if (expectingIntroParagraph > 0) {\n                    // Skip processing words for intro paragraphs\n                } else {\n                    words.push(token.word);\n                }\n            } else if (token.kind === 'whitespace') {\n                if (expectingId > 0) {\n                    if (token.whitespace.includes('\\n')) {\n                        expectingId = 0;\n                    }\n                } else if (expectingName > 0) {\n                    if (token.whitespace.includes('\\n')) {\n                        expectingName = 0;\n                    }\n                } else if (expectingTitle > 0) {\n                    if (token.whitespace.includes('\\n')) {\n                        expectingTitle = 0;\n                    }\n                } else if (expectingSectionHeading > 0) {\n                    if (token.whitespace.includes('\\n')) {\n                        completeSection();\n                    }\n                } else if (expectingReferenceText > 0) {\n                    if (token.whitespace.includes('\\n')) {\n                        expectingReferenceText = 0;\n                    }\n                } else if (expectingIntroParagraph > 0) {\n                    if (token.whitespace.includes('\\n')) {\n                        expectingIntroParagraph = 0;\n                        canParseFootnotes = true;\n                    }\n                } else if (\n                    expectingId > 0 ||\n                    expectingName > 0 ||\n                    expectingTitle > 0 ||\n                    expectingSectionHeading > 0 ||\n                    expectingFootnote > 0 ||\n                    expectingFootnoteReference > 0 ||\n                    expectingFootnoteText > 0 ||\n                    expectingReferenceText > 0 ||\n                    expectingCrossReference > 0 ||\n                    expectingWordAttribute > 0 ||\n                    expectingNestedWordAttribute > 0\n                ) {\n                    // Skip\n                } else if (expectingUnknownCommand > 0) {\n                    if (token.whitespace.includes('\\n')) {\n                        expectingUnknownCommand = 0;\n                    }\n                } else if (words.length > 0) {\n                    let lastWord = words[words.length - 1];\n                    if (lastWord !== ' ') {\n                        words.push(' ');\n                    }\n                }\n            }\n        }\n\n        completeVerseOrSubtitle(null);\n\n        return root;\n    }\n\n    renderMarkdown(tree: ParseTree): string {\n        let md = '';\n\n        if (tree.header) {\n            md += `# ${tree.header}\\n`;\n        }\n\n        for (let c of tree.content) {\n            if (c.type === 'heading') {\n                md += `## ${c.content.join(' ')}\\n`;\n            } else if (c.type === 'chapter') {\n                md += `### ${c.number}\\n`;\n\n                for (let content of c.content) {\n                    if (content.type === 'heading') {\n                        md += `#### ${content.content.join(' ')}\\n`;\n                    } else if (content.type === 'line_break') {\n                        md += '\\n\\n';\n                    } else if (content.type === 'verse') {\n                        md += `<em>${content.number}</em>`;\n                        for (let v of content.content) {\n                            if (typeof v === 'string') {\n                                md += v + ' ';\n                            } else if ('text' in v) {\n                                md += v.text + ' ';\n                            }\n                        }\n                        md += '\\n';\n                    }\n                }\n            }\n        }\n\n        return md;\n    }\n\n    private _hasAttribute() {\n        return this._poem !== null || this._wordsOfJesus;\n    }\n\n    private _text(text: string): Text | string {\n        if (!this._hasAttribute()) {\n            return text;\n        }\n        const t: Text = {\n            text,\n        };\n\n        if (this._poem !== null) {\n            t.poem = this._poem;\n        }\n\n        if (this._wordsOfJesus) {\n            t.wordsOfJesus = true;\n        }\n\n        return t;\n    }\n\n    private _throwError(\n        source: string,\n        token: Token | null,\n        message: string,\n        warn: boolean = false\n    ): void {\n        const logger = getLogger();\n        if (token) {\n            let line = 1;\n            let column = 1;\n\n            let start = token.loc.start;\n\n            for (let i = 0; i < start; i++) {\n                let char = source[i];\n                if (char === '\\n') {\n                    line += 1;\n                    column = 1;\n                } else {\n                    column += 1;\n                }\n            }\n\n            let tokenDebug = '';\n            if (token.kind === 'word') {\n                tokenDebug = ', word';\n            } else if (token.kind === 'marker') {\n                tokenDebug = ', ' + token.command;\n            } else {\n                tokenDebug = '';\n            }\n\n            if (warn) {\n                logger.warn(`(${line}, ${column}${tokenDebug}) ${message}`);\n            } else {\n                throw new Error(`(${line}, ${column}${tokenDebug}) ${message}`);\n            }\n        } else {\n            if (warn) {\n                logger.warn(message);\n            } else {\n                throw new Error(message);\n            }\n        }\n    }\n}\n\n/**\n * Determines if the given character is a digit.\n * @param char The character.\n */\nexport function isDigit(char: string): boolean {\n    return char.length === 1 && char >= '0' && char <= '9';\n}\n\n/**\n * Determines if the given character is considered whitespace.\n * @param char The character.\n */\nexport function isWhitespace(char: string): boolean {\n    return char === ' ' || char === '\\t' || char === '\\n' || char === '\\r';\n}\n\nexport function t(loc: SourceLocation, kind: T): SimpleToken {\n    return {\n        loc,\n        kind,\n    };\n}\n\n/**\n * Creates a new source location.\n * @param start The start of the location.\n * @param end The end of the location.\n */\nexport function loc(start: number, end: number): SourceLocation {\n    return {\n        start,\n        end,\n    };\n}\n\n/**\n * Creates a new marker token.\n * @param loc The location for the token.\n * @param command The command that the token contains.\n * @param number The number that the marker contains.\n * @param type The type of the marker.\n */\nexport function marker(\n    loc: SourceLocation,\n    command: string,\n    number: number | null = null,\n    type: MarkerToken['type'] = 'start'\n): MarkerToken {\n    return {\n        kind: 'marker',\n        loc,\n        command,\n        number,\n        type,\n    };\n}\n\n/**\n * Creates a new word token.\n * @param loc The location for the token.\n * @param word The word contained by the token.\n */\nexport function word(loc: SourceLocation, word: string): WordToken {\n    return {\n        kind: 'word',\n        loc,\n        word,\n    };\n}\n\nexport function whitespace(\n    loc: SourceLocation,\n    whitespace: string\n): WhitespaceToken {\n    return {\n        kind: 'whitespace',\n        loc,\n        whitespace,\n    };\n}\n\nexport type T = Token['kind'];\n\n/**\n * Defines a simple token.\n */\nexport interface SimpleToken {\n    kind: T;\n    loc: SourceLocation;\n}\n\nexport type Token = MarkerToken | WordToken | WhitespaceToken;\n\n/**\n * Defines an interface for a USFM marker node.\n * That is, the syntax for a marker.\n */\nexport interface MarkerToken {\n    kind: 'marker';\n\n    /**\n     * The command that the marker represents.\n     * This is generally the name of the marker (like \"p\" or \"v\" for paragraph and verse markers)\n     */\n    command: string;\n\n    /**\n     * The number that the marker contains.\n     * Null if no number was specified.\n     */\n    number: number | null;\n\n    /**\n     * Whether the marker represents a start or an end.\n     */\n    type: 'start' | 'end';\n\n    /**\n     * The location of the node.\n     */\n    loc: SourceLocation;\n}\n\n/**\n * Defines an interface for a Whitespace node.\n */\nexport interface WhitespaceToken {\n    kind: 'whitespace';\n\n    /**\n     * The whitespace contained by the node.\n     */\n    whitespace: string;\n\n    /**\n     * The location of the node.\n     */\n    loc: SourceLocation;\n}\n\n/**\n * Defines an interface for a word token.\n */\nexport interface WordToken {\n    kind: 'word';\n\n    /**\n     * The word contained by the token.\n     */\n    word: string;\n\n    /**\n     * The location of the word.\n     */\n    loc: SourceLocation;\n}\n\n/**\n * The source location of the node.\n */\nexport interface SourceLocation {\n    start: number;\n    end: number;\n}\n"],
+  "sourcesContent": ["import { findLast } from 'es-toolkit/compat';\r\nimport {\r\n    ParseTree,\r\n    Chapter,\r\n    Verse,\r\n    HebrewSubtitle,\r\n    FootnoteReference,\r\n    Footnote,\r\n    Text,\r\n} from './types.js';\r\nimport { getLogger } from '../log.js';\r\n\r\n/**\r\n * Defines a class that can tokenize a stream of characters into tokens.\r\n */\r\nexport class UsfmTokenizer {\r\n    private _input: string = '';\r\n    private _index: number = 0;\r\n    private _start: number = 0;\r\n\r\n    private get _tokenLength() {\r\n        return this._index - this._start;\r\n    }\r\n\r\n    /**\r\n     * Converts the given input into a list of tokens.\r\n     * @param input The input that should be tokenized.\r\n     */\r\n    tokenize(input: string): SimpleToken[] {\r\n        this._input = input;\r\n        this._index = 0;\r\n\r\n        return this._parseTokens();\r\n    }\r\n\r\n    private _parseTokens() {\r\n        let tokens: SimpleToken[] = [];\r\n        let token = this._parseToken();\r\n        while (token) {\r\n            tokens.push(token);\r\n            token = this._parseToken();\r\n        }\r\n\r\n        return tokens;\r\n    }\r\n\r\n    private _parseToken(): SimpleToken | null {\r\n        let state:\r\n            | 'none'\r\n            | 'marker_start'\r\n            | 'marker_number'\r\n            | 'whitespace'\r\n            | 'word' = 'none';\r\n        let kind: T | null = null;\r\n        this._start = this._index;\r\n\r\n        while (this._index < this._input.length) {\r\n            const codePointNumber = this._input.codePointAt(this._index);\r\n\r\n            if (typeof codePointNumber === 'undefined') {\r\n                throw new Error('Unable to get code point!');\r\n            }\r\n\r\n            const codePoint = String.fromCodePoint(codePointNumber);\r\n\r\n            if (state === 'none') {\r\n                if (codePoint === '\\\\') {\r\n                    state = 'marker_start';\r\n                } else if (isWhitespace(codePoint)) {\r\n                    state = 'whitespace';\r\n                } else {\r\n                    state = 'word';\r\n                }\r\n            } else if (state === 'marker_start') {\r\n                if (isDigit(codePoint)) {\r\n                    if (this._tokenLength === 0) {\r\n                        throw new Error(\r\n                            'Invalid Marker: Markers must not contain only digits.'\r\n                        );\r\n                    }\r\n                    state = 'marker_number';\r\n                } else if (codePoint === '*') {\r\n                    this._index += codePoint.length;\r\n                    kind = 'marker';\r\n                    break;\r\n                } else if (isWhitespace(codePoint)) {\r\n                    kind = 'marker';\r\n                    break;\r\n                }\r\n            } else if (state === 'marker_number') {\r\n                if (codePoint === '*') {\r\n                    this._index += codePoint.length;\r\n                    kind = 'marker';\r\n                    break;\r\n                } else if (!isDigit(codePoint)) {\r\n                    kind = 'marker';\r\n                    break;\r\n                }\r\n            } else if (state === 'whitespace') {\r\n                if (!isWhitespace(codePoint)) {\r\n                    kind = 'whitespace';\r\n                    break;\r\n                }\r\n            } else if (state === 'word') {\r\n                if (isWhitespace(codePoint) || codePoint === '\\\\') {\r\n                    kind = 'word';\r\n                    break;\r\n                }\r\n            }\r\n\r\n            this._index += codePoint.length;\r\n        }\r\n\r\n        if (!kind) {\r\n            if (this._index >= this._input.length) {\r\n                if (state == 'marker_start' || state === 'marker_number') {\r\n                    kind = 'marker';\r\n                } else if (state === 'word') {\r\n                    kind = 'word';\r\n                } else if (state === 'whitespace') {\r\n                    kind = 'whitespace';\r\n                }\r\n            }\r\n        }\r\n\r\n        if (kind) {\r\n            return t(loc(this._start, this._index), kind);\r\n        }\r\n\r\n        return null;\r\n    }\r\n}\r\n\r\nexport interface UsfmParseOptions {\r\n    paragraphs: Set<string>;\r\n}\r\n\r\n/**\r\n * Defines a USFM Parser.\r\n */\r\nexport class UsfmParser {\r\n    private _poem: number | null = null;\r\n    private _wordsOfJesus: boolean = false;\r\n\r\n    tokenize(input: string): Token[] {\r\n        const simpleTokens = new UsfmTokenizer().tokenize(input);\r\n        let tokens: Token[] = [];\r\n\r\n        for (let t of simpleTokens) {\r\n            if (t.kind === 'marker') {\r\n                let source = input.substring(t.loc.start, t.loc.end);\r\n                const isEnd = source.endsWith('*');\r\n\r\n                if (isEnd) {\r\n                    source = source.substring(0, source.length - 1);\r\n                }\r\n\r\n                let numberIndex = -1;\r\n                for (let i = 0; i < source.length; i++) {\r\n                    if (isDigit(source[i])) {\r\n                        numberIndex = i;\r\n                        break;\r\n                    }\r\n                }\r\n\r\n                let number: number | null = null;\r\n                if (numberIndex === 1) {\r\n                    throw new Error(\r\n                        'Markers must not be made only of numbers!'\r\n                    );\r\n                }\r\n                if (numberIndex > 0) {\r\n                    number = parseInt(source.substring(numberIndex));\r\n                    source = source.substring(0, numberIndex);\r\n                }\r\n\r\n                if (source.length === 1) {\r\n                    if (isEnd) {\r\n                        // Ending marker does not have a command.\r\n                        // We should look for a matching start marker.\r\n                        const startMarker = findLast(\r\n                            tokens,\r\n                            (t) => t.kind === 'marker' && t.type === 'start'\r\n                        ) as MarkerToken;\r\n                        if (startMarker) {\r\n                            source = startMarker.command;\r\n                        }\r\n                    }\r\n\r\n                    if (source.length === 1) {\r\n                        throw new Error(\r\n                            `Markers must have a command! Token: ${t.loc.start}-${t.loc.end}`\r\n                        );\r\n                    }\r\n                }\r\n\r\n                tokens.push(\r\n                    marker(t.loc, source, number, isEnd ? 'end' : 'start')\r\n                );\r\n            } else if (t.kind === 'whitespace') {\r\n                let source = input.substring(t.loc.start, t.loc.end);\r\n                tokens.push(whitespace(t.loc, source));\r\n            } else if (t.kind === 'word') {\r\n                let source = input.substring(t.loc.start, t.loc.end);\r\n                tokens.push(word(t.loc, source));\r\n            }\r\n        }\r\n\r\n        return tokens;\r\n    }\r\n\r\n    parse(input: string): ParseTree {\r\n        let root: ParseTree = {\r\n            type: 'root',\r\n            content: [],\r\n        };\r\n\r\n        const tokens = this.tokenize(input);\r\n\r\n        let expectingId = 0;\r\n        let expectingName = 0;\r\n        let expectingTitle = 0;\r\n        let expectingSectionHeading = 0;\r\n        let expectingFootnote = 0;\r\n        let expectingFootnoteReference = 0;\r\n        let expectingFootnoteText = 0;\r\n        let expectingReferenceText = 0;\r\n        let expectingWordAttribute = 0;\r\n        let expectingNestedWordAttribute = 0;\r\n        let expectingWordsOfJesus = 0;\r\n        let expectingIntroParagraph = 0;\r\n        let expectingCrossReference = 0;\r\n        let expectingUnknownCommand = 0;\r\n\r\n        let canParseFootnotes = true;\r\n        let chapter: Chapter | null = null;\r\n        let lastVerse: Verse | null = null;\r\n        let verse: Verse | null = null;\r\n        let subtitle: HebrewSubtitle | null = null;\r\n        let words: string[] = [];\r\n        let verseContent: (Text | FootnoteReference | string)[] = [];\r\n        let sectionContent: string = '';\r\n        let currentFootnoteId = 0;\r\n        let footnote: Footnote | null = null;\r\n\r\n        this._poem = null;\r\n\r\n        const addWordsToVerseOrSubtitle = () => {\r\n            if (words.length > 0) {\r\n                const text = this._text(words.join('').trimEnd());\r\n                if (verse) {\r\n                    verse.content.push(text);\r\n                } else if (subtitle) {\r\n                    subtitle.content.push(text);\r\n                } else {\r\n                    verseContent.push(text);\r\n                }\r\n                words = [];\r\n            }\r\n        };\r\n\r\n        const addVerseContentToChapter = (token: Token | null) => {\r\n            if (!chapter) {\r\n                return;\r\n            }\r\n            if (verseContent.length > 0) {\r\n                if (chapter.content.some((c) => c.type === 'verse')) {\r\n                    this._throwError(\r\n                        input,\r\n                        token,\r\n                        'Cannot infer first verse after other verses have been added to the chapter!'\r\n                    );\r\n                }\r\n                // Implicit first verse\r\n                verse = {\r\n                    type: 'verse',\r\n                    number: 1,\r\n                    content: verseContent,\r\n                };\r\n                chapter.content.push(verse);\r\n                verseContent = [];\r\n            }\r\n        };\r\n\r\n        const cleanupVerse = () => {\r\n            if (!verse || !chapter) {\r\n                return;\r\n            }\r\n            let chapterContent: Chapter['content'] = [];\r\n            for (let i = verse.content.length - 1; i >= 0; i--) {\r\n                let content = verse.content[i];\r\n                if (typeof content === 'object' && 'heading' in content) {\r\n                    // move headings that occur at the end of a verse to the chapter\r\n                    chapterContent.unshift({\r\n                        type: 'heading',\r\n                        content: [content.heading],\r\n                    });\r\n                    verse.content.splice(i, 1);\r\n                } else if (\r\n                    typeof content === 'object' &&\r\n                    'lineBreak' in content &&\r\n                    content.lineBreak\r\n                ) {\r\n                    // move line breaks that occur at the end of a verse to the chapter\r\n                    chapterContent.unshift({\r\n                        type: 'line_break',\r\n                    });\r\n                    verse.content.splice(i, 1);\r\n                } else {\r\n                    break;\r\n                }\r\n            }\r\n\r\n            for (let content of chapterContent) {\r\n                chapter.content.push(content);\r\n            }\r\n        };\r\n\r\n        const completeVerseOrSubtitle = (token: Token | null) => {\r\n            if (verse && isNaN(verse.number)) {\r\n                // Verse is invalid for some reason.\r\n                const index = chapter!.content.indexOf(verse);\r\n                if (index >= 0) {\r\n                    chapter!.content.splice(index, 1);\r\n                }\r\n                verse = null;\r\n            }\r\n            if (verse || subtitle) {\r\n                addWordsToVerseOrSubtitle();\r\n            }\r\n\r\n            addVerseContentToChapter(token);\r\n            cleanupVerse();\r\n        };\r\n\r\n        const completeSection = () => {\r\n            if (expectingSectionHeading > 0) {\r\n                if (verse) {\r\n                    addWordsToVerseOrSubtitle();\r\n                    verse.content.push({\r\n                        heading: sectionContent,\r\n                    });\r\n                } else if (chapter) {\r\n                    chapter.content.push({\r\n                        type: 'heading',\r\n                        content: [sectionContent],\r\n                    });\r\n                } else {\r\n                    root.content.push({\r\n                        type: 'heading',\r\n                        content: [sectionContent],\r\n                    });\r\n                }\r\n                sectionContent = '';\r\n                expectingSectionHeading = 0;\r\n            }\r\n        };\r\n\r\n        const addWordsToFootnote = () => {\r\n            if (footnote && words.length > 0) {\r\n                footnote.text += words.join(' ');\r\n                words = [];\r\n            }\r\n        };\r\n\r\n        for (let token of tokens) {\r\n            if (token.kind === 'marker') {\r\n                if (token.command === '\\\\c') {\r\n                    addWordsToVerseOrSubtitle();\r\n                    cleanupVerse();\r\n\r\n                    chapter = {\r\n                        type: 'chapter',\r\n                        number: NaN,\r\n                        content: [],\r\n                        footnotes: [],\r\n                    };\r\n                    verse = null;\r\n                    verseContent = [];\r\n\r\n                    root.content.push(chapter);\r\n                } else if (token.command === '\\\\v') {\r\n                    if (!chapter) {\r\n                        this._throwError(\r\n                            input,\r\n                            token,\r\n                            'Cannot parse a verse without chapter information!'\r\n                        );\r\n                    } else {\r\n                        completeSection();\r\n                        completeVerseOrSubtitle(token);\r\n\r\n                        lastVerse = verse;\r\n                        verse = {\r\n                            type: 'verse',\r\n                            number: NaN,\r\n                            content: [],\r\n                        };\r\n\r\n                        chapter.content.push(verse);\r\n                    }\r\n                } else if (token.command === '\\\\d') {\r\n                    if (!chapter) {\r\n                        this._throwError(\r\n                            input,\r\n                            token,\r\n                            'Cannot parse a hebrew subtitle without chapter information!'\r\n                        );\r\n                    } else {\r\n                        completeVerseOrSubtitle(token);\r\n\r\n                        subtitle = {\r\n                            type: 'hebrew_subtitle',\r\n                            content: [],\r\n                        };\r\n\r\n                        chapter.content.push(subtitle);\r\n                    }\r\n                } else if (token.command === '\\\\b' || token.command === '\\\\p') {\r\n                    if (!chapter) {\r\n                        this._throwError(\r\n                            input,\r\n                            token,\r\n                            'Cannot parse a line break without chapter information!'\r\n                        );\r\n                    } else {\r\n                        if (verse) {\r\n                            addWordsToVerseOrSubtitle();\r\n                            verse.content.push({\r\n                                lineBreak: true,\r\n                            });\r\n                        } else {\r\n                            completeVerseOrSubtitle(token);\r\n                            chapter.content.push({\r\n                                type: 'line_break',\r\n                            });\r\n                        }\r\n                    }\r\n                } else if (token.command === '\\\\q') {\r\n                    addWordsToVerseOrSubtitle();\r\n                    this._poem = token.number;\r\n                } else if (token.command === '\\\\p') {\r\n                    addWordsToVerseOrSubtitle();\r\n                    this._poem = null;\r\n                } else if (token.command === '\\\\id') {\r\n                    expectingId = 1;\r\n                } else if (token.command === '\\\\h') {\r\n                    expectingName = 1;\r\n                    root.header = undefined;\r\n                } else if (\r\n                    token.command === '\\\\mt' ||\r\n                    token.command === '\\\\+mt'\r\n                ) {\r\n                    expectingTitle = 1;\r\n                } else if (token.command === '\\\\s') {\r\n                    expectingSectionHeading = 1;\r\n                } else if (token.command === '\\\\r') {\r\n                    expectingReferenceText = 1;\r\n                } else if (token.command === '\\\\f' && canParseFootnotes) {\r\n                    if (token.type === 'start') {\r\n                        if (!chapter) {\r\n                            this._throwError(\r\n                                input,\r\n                                token,\r\n                                'Cannot start a footnote outside of a chapter!',\r\n                                true\r\n                            );\r\n                        } else {\r\n                            addWordsToVerseOrSubtitle();\r\n                            footnote = {\r\n                                noteId: currentFootnoteId,\r\n                                text: '',\r\n                                caller: null,\r\n                            };\r\n                            const ref: FootnoteReference = {\r\n                                noteId: footnote.noteId,\r\n                            };\r\n                            expectingFootnote = 1;\r\n\r\n                            chapter.footnotes.push(footnote);\r\n\r\n                            if (verse) {\r\n                                verse.content.push(ref);\r\n                            } else if (subtitle) {\r\n                                subtitle.content.push(ref);\r\n                            } else {\r\n                                verseContent.push(ref);\r\n                            }\r\n\r\n                            currentFootnoteId += 1;\r\n                        }\r\n                    } else {\r\n                        addWordsToFootnote();\r\n                        expectingFootnote = 0;\r\n                        expectingFootnoteText = 0;\r\n                        expectingFootnoteReference = 0;\r\n                        footnote = null;\r\n                    }\r\n                } else if (token.command === '\\\\fr' && canParseFootnotes) {\r\n                    if (!footnote) {\r\n                        this._throwError(\r\n                            input,\r\n                            token,\r\n                            'Cannot start a footnote reference outside of a footnote!',\r\n                            true\r\n                        );\r\n                    } else {\r\n                        expectingFootnoteReference = 1;\r\n                    }\r\n                } else if (token.command === '\\\\ft' && canParseFootnotes) {\r\n                    if (!footnote) {\r\n                        this._throwError(\r\n                            input,\r\n                            token,\r\n                            'Cannot start footnote text outside of a footnote!',\r\n                            true\r\n                        );\r\n                    } else {\r\n                        expectingFootnoteText = 1;\r\n                    }\r\n                } else if (token.command === '\\\\w') {\r\n                    if (token.type === 'start') {\r\n                        expectingWordAttribute = 1;\r\n                    } else {\r\n                        expectingWordAttribute = 0;\r\n                    }\r\n                } else if (token.command === '\\\\+w') {\r\n                    if (token.type === 'start') {\r\n                        expectingNestedWordAttribute = 1;\r\n                    } else {\r\n                        expectingNestedWordAttribute = 0;\r\n                    }\r\n                } else if (\r\n                    token.command === '\\\\wj' ||\r\n                    token.command === '\\\\+wj'\r\n                ) {\r\n                    if (token.type === 'start') {\r\n                        addWordsToVerseOrSubtitle();\r\n                        this._wordsOfJesus = true;\r\n                    } else {\r\n                        addWordsToVerseOrSubtitle();\r\n                        this._wordsOfJesus = false;\r\n                    }\r\n                } else if (token.command === '\\\\ip') {\r\n                    expectingIntroParagraph = 1;\r\n                    canParseFootnotes = false;\r\n                } else if (token.command === '\\\\x') {\r\n                    if (token.type === 'start') {\r\n                        expectingCrossReference = 1;\r\n                    } else {\r\n                        expectingCrossReference = 0;\r\n                    }\r\n                } else if (token.command.indexOf('-') >= 0) {\r\n                    if (token.type === 'start') {\r\n                        expectingUnknownCommand = 1;\r\n                    }\r\n                } else if (token.type === 'end') {\r\n                    expectingUnknownCommand = 0;\r\n                }\r\n            } else if (token.kind === 'word') {\r\n                if (expectingId > 0) {\r\n                    if (expectingId === 1) {\r\n                        root.id = token.word;\r\n                        expectingId = 2;\r\n                    }\r\n                } else if (expectingName > 0) {\r\n                    if (root.header) {\r\n                        root.header += ' ' + token.word;\r\n                    } else {\r\n                        root.header = token.word;\r\n                    }\r\n                } else if (expectingTitle > 0) {\r\n                    if (root.title) {\r\n                        root.title += ' ' + token.word;\r\n                    } else {\r\n                        root.title = token.word;\r\n                    }\r\n                } else if (expectingSectionHeading > 0) {\r\n                    if (sectionContent) {\r\n                        sectionContent += ' ' + token.word;\r\n                    } else {\r\n                        sectionContent = token.word;\r\n                    }\r\n                } else if (expectingFootnoteReference > 0) {\r\n                    if ((expectingFootnoteReference = 1)) {\r\n                        const [chapter, verse] = token.word.split(/[\\.\\:]/);\r\n\r\n                        if (footnote) {\r\n                            footnote.reference = {\r\n                                chapter: parseInt(chapter),\r\n                                verse: parseInt(verse),\r\n                            };\r\n                        }\r\n\r\n                        expectingFootnoteReference = 0;\r\n                    }\r\n                } else if (expectingFootnoteText > 0) {\r\n                    words.push(token.word);\r\n                } else if (expectingFootnote > 0) {\r\n                    if (expectingFootnote === 1) {\r\n                        if (token.word) {\r\n                            if (footnote) {\r\n                                if (token.word === '+' || token.word !== '-') {\r\n                                    footnote.caller = token.word;\r\n                                } else {\r\n                                    footnote.caller = null;\r\n                                }\r\n                            }\r\n                            // this._throwError(input, token, 'Footnotes must use the \"+\" caller.');\r\n                        }\r\n                        expectingFootnote = 2;\r\n                    } else {\r\n                        words.push(token.word);\r\n                    }\r\n                } else if (expectingReferenceText > 0) {\r\n                    // Skip processing words for references\r\n                    // because references aren't included in the JSON format\r\n                    // (for now)\r\n                } else if (expectingCrossReference > 0) {\r\n                    // Skip processing words for cross references\r\n                } else if (chapter && isNaN(chapter.number)) {\r\n                    chapter.number = parseInt(token.word);\r\n                    if (isNaN(chapter.number)) {\r\n                        this._throwError(\r\n                            input,\r\n                            token,\r\n                            'The first word token after a chapter marker must be parsable to an integer!'\r\n                        );\r\n                    }\r\n                } else if (verse && isNaN(verse.number)) {\r\n                    verse.number = parseInt(token.word);\r\n                    if (isNaN(verse.number)) {\r\n                        this._throwError(\r\n                            input,\r\n                            token,\r\n                            'The first word token after a verse marker must be parsable to an integer!'\r\n                        );\r\n                    }\r\n                } else if (expectingWordAttribute > 0) {\r\n                    if (expectingWordAttribute === 1) {\r\n                        const firstVerticalBarIndex = token.word.indexOf('|');\r\n\r\n                        if (firstVerticalBarIndex >= 0) {\r\n                            const name = token.word.slice(\r\n                                0,\r\n                                firstVerticalBarIndex\r\n                            );\r\n                            // const rest = token.word.slice(firstVerticalBarIndex + '|'.length);\r\n                            words.push(name);\r\n                            expectingWordAttribute = 2;\r\n                        } else {\r\n                            words.push(token.word);\r\n                        }\r\n                    }\r\n                } else if (expectingNestedWordAttribute > 0) {\r\n                    if (expectingNestedWordAttribute === 1) {\r\n                        const firstVerticalBarIndex = token.word.indexOf('|');\r\n\r\n                        if (firstVerticalBarIndex >= 0) {\r\n                            const name = token.word.slice(\r\n                                0,\r\n                                firstVerticalBarIndex\r\n                            );\r\n                            // const rest = token.word.slice(firstVerticalBarIndex + '|'.length);\r\n                            words.push(name);\r\n                            expectingNestedWordAttribute = 2;\r\n                        } else {\r\n                            words.push(token.word);\r\n                        }\r\n                    }\r\n                } else if (expectingUnknownCommand > 0) {\r\n                } else if (expectingIntroParagraph > 0) {\r\n                    // Skip processing words for intro paragraphs\r\n                } else {\r\n                    words.push(token.word);\r\n                }\r\n            } else if (token.kind === 'whitespace') {\r\n                if (expectingId > 0) {\r\n                    if (token.whitespace.includes('\\n')) {\r\n                        expectingId = 0;\r\n                    }\r\n                } else if (expectingName > 0) {\r\n                    if (token.whitespace.includes('\\n')) {\r\n                        expectingName = 0;\r\n                    }\r\n                } else if (expectingTitle > 0) {\r\n                    if (token.whitespace.includes('\\n')) {\r\n                        expectingTitle = 0;\r\n                    }\r\n                } else if (expectingSectionHeading > 0) {\r\n                    if (token.whitespace.includes('\\n')) {\r\n                        completeSection();\r\n                    }\r\n                } else if (expectingReferenceText > 0) {\r\n                    if (token.whitespace.includes('\\n')) {\r\n                        expectingReferenceText = 0;\r\n                    }\r\n                } else if (expectingIntroParagraph > 0) {\r\n                    if (token.whitespace.includes('\\n')) {\r\n                        expectingIntroParagraph = 0;\r\n                        canParseFootnotes = true;\r\n                    }\r\n                } else if (\r\n                    expectingId > 0 ||\r\n                    expectingName > 0 ||\r\n                    expectingTitle > 0 ||\r\n                    expectingSectionHeading > 0 ||\r\n                    expectingFootnote > 0 ||\r\n                    expectingFootnoteReference > 0 ||\r\n                    expectingFootnoteText > 0 ||\r\n                    expectingReferenceText > 0 ||\r\n                    expectingCrossReference > 0 ||\r\n                    expectingWordAttribute > 0 ||\r\n                    expectingNestedWordAttribute > 0\r\n                ) {\r\n                    // Skip\r\n                } else if (expectingUnknownCommand > 0) {\r\n                    if (token.whitespace.includes('\\n')) {\r\n                        expectingUnknownCommand = 0;\r\n                    }\r\n                } else if (words.length > 0) {\r\n                    let lastWord = words[words.length - 1];\r\n                    if (lastWord !== ' ') {\r\n                        words.push(' ');\r\n                    }\r\n                }\r\n            }\r\n        }\r\n\r\n        completeVerseOrSubtitle(null);\r\n\r\n        return root;\r\n    }\r\n\r\n    renderMarkdown(tree: ParseTree): string {\r\n        let md = '';\r\n\r\n        if (tree.header) {\r\n            md += `# ${tree.header}\\n`;\r\n        }\r\n\r\n        for (let c of tree.content) {\r\n            if (c.type === 'heading') {\r\n                md += `## ${c.content.join(' ')}\\n`;\r\n            } else if (c.type === 'chapter') {\r\n                md += `### ${c.number}\\n`;\r\n\r\n                for (let content of c.content) {\r\n                    if (content.type === 'heading') {\r\n                        md += `#### ${content.content.join(' ')}\\n`;\r\n                    } else if (content.type === 'line_break') {\r\n                        md += '\\n\\n';\r\n                    } else if (content.type === 'verse') {\r\n                        md += `<em>${content.number}</em>`;\r\n                        for (let v of content.content) {\r\n                            if (typeof v === 'string') {\r\n                                md += v + ' ';\r\n                            } else if ('text' in v) {\r\n                                md += v.text + ' ';\r\n                            }\r\n                        }\r\n                        md += '\\n';\r\n                    }\r\n                }\r\n            }\r\n        }\r\n\r\n        return md;\r\n    }\r\n\r\n    private _hasAttribute() {\r\n        return this._poem !== null || this._wordsOfJesus;\r\n    }\r\n\r\n    private _text(text: string): Text | string {\r\n        if (!this._hasAttribute()) {\r\n            return text;\r\n        }\r\n        const t: Text = {\r\n            text,\r\n        };\r\n\r\n        if (this._poem !== null) {\r\n            t.poem = this._poem;\r\n        }\r\n\r\n        if (this._wordsOfJesus) {\r\n            t.wordsOfJesus = true;\r\n        }\r\n\r\n        return t;\r\n    }\r\n\r\n    private _throwError(\r\n        source: string,\r\n        token: Token | null,\r\n        message: string,\r\n        warn: boolean = false\r\n    ): void {\r\n        const logger = getLogger();\r\n        if (token) {\r\n            let line = 1;\r\n            let column = 1;\r\n\r\n            let start = token.loc.start;\r\n\r\n            for (let i = 0; i < start; i++) {\r\n                let char = source[i];\r\n                if (char === '\\n') {\r\n                    line += 1;\r\n                    column = 1;\r\n                } else {\r\n                    column += 1;\r\n                }\r\n            }\r\n\r\n            let tokenDebug = '';\r\n            if (token.kind === 'word') {\r\n                tokenDebug = ', word';\r\n            } else if (token.kind === 'marker') {\r\n                tokenDebug = ', ' + token.command;\r\n            } else {\r\n                tokenDebug = '';\r\n            }\r\n\r\n            if (warn) {\r\n                logger.warn(`(${line}, ${column}${tokenDebug}) ${message}`);\r\n            } else {\r\n                throw new Error(`(${line}, ${column}${tokenDebug}) ${message}`);\r\n            }\r\n        } else {\r\n            if (warn) {\r\n                logger.warn(message);\r\n            } else {\r\n                throw new Error(message);\r\n            }\r\n        }\r\n    }\r\n}\r\n\r\n/**\r\n * Determines if the given character is a digit.\r\n * @param char The character.\r\n */\r\nexport function isDigit(char: string): boolean {\r\n    return char.length === 1 && char >= '0' && char <= '9';\r\n}\r\n\r\n/**\r\n * Determines if the given character is considered whitespace.\r\n * @param char The character.\r\n */\r\nexport function isWhitespace(char: string): boolean {\r\n    return char === ' ' || char === '\\t' || char === '\\n' || char === '\\r';\r\n}\r\n\r\nexport function t(loc: SourceLocation, kind: T): SimpleToken {\r\n    return {\r\n        loc,\r\n        kind,\r\n    };\r\n}\r\n\r\n/**\r\n * Creates a new source location.\r\n * @param start The start of the location.\r\n * @param end The end of the location.\r\n */\r\nexport function loc(start: number, end: number): SourceLocation {\r\n    return {\r\n        start,\r\n        end,\r\n    };\r\n}\r\n\r\n/**\r\n * Creates a new marker token.\r\n * @param loc The location for the token.\r\n * @param command The command that the token contains.\r\n * @param number The number that the marker contains.\r\n * @param type The type of the marker.\r\n */\r\nexport function marker(\r\n    loc: SourceLocation,\r\n    command: string,\r\n    number: number | null = null,\r\n    type: MarkerToken['type'] = 'start'\r\n): MarkerToken {\r\n    return {\r\n        kind: 'marker',\r\n        loc,\r\n        command,\r\n        number,\r\n        type,\r\n    };\r\n}\r\n\r\n/**\r\n * Creates a new word token.\r\n * @param loc The location for the token.\r\n * @param word The word contained by the token.\r\n */\r\nexport function word(loc: SourceLocation, word: string): WordToken {\r\n    return {\r\n        kind: 'word',\r\n        loc,\r\n        word,\r\n    };\r\n}\r\n\r\nexport function whitespace(\r\n    loc: SourceLocation,\r\n    whitespace: string\r\n): WhitespaceToken {\r\n    return {\r\n        kind: 'whitespace',\r\n        loc,\r\n        whitespace,\r\n    };\r\n}\r\n\r\nexport type T = Token['kind'];\r\n\r\n/**\r\n * Defines a simple token.\r\n */\r\nexport interface SimpleToken {\r\n    kind: T;\r\n    loc: SourceLocation;\r\n}\r\n\r\nexport type Token = MarkerToken | WordToken | WhitespaceToken;\r\n\r\n/**\r\n * Defines an interface for a USFM marker node.\r\n * That is, the syntax for a marker.\r\n */\r\nexport interface MarkerToken {\r\n    kind: 'marker';\r\n\r\n    /**\r\n     * The command that the marker represents.\r\n     * This is generally the name of the marker (like \"p\" or \"v\" for paragraph and verse markers)\r\n     */\r\n    command: string;\r\n\r\n    /**\r\n     * The number that the marker contains.\r\n     * Null if no number was specified.\r\n     */\r\n    number: number | null;\r\n\r\n    /**\r\n     * Whether the marker represents a start or an end.\r\n     */\r\n    type: 'start' | 'end';\r\n\r\n    /**\r\n     * The location of the node.\r\n     */\r\n    loc: SourceLocation;\r\n}\r\n\r\n/**\r\n * Defines an interface for a Whitespace node.\r\n */\r\nexport interface WhitespaceToken {\r\n    kind: 'whitespace';\r\n\r\n    /**\r\n     * The whitespace contained by the node.\r\n     */\r\n    whitespace: string;\r\n\r\n    /**\r\n     * The location of the node.\r\n     */\r\n    loc: SourceLocation;\r\n}\r\n\r\n/**\r\n * Defines an interface for a word token.\r\n */\r\nexport interface WordToken {\r\n    kind: 'word';\r\n\r\n    /**\r\n     * The word contained by the token.\r\n     */\r\n    word: string;\r\n\r\n    /**\r\n     * The location of the word.\r\n     */\r\n    loc: SourceLocation;\r\n}\r\n\r\n/**\r\n * The source location of the node.\r\n */\r\nexport interface SourceLocation {\r\n    start: number;\r\n    end: number;\r\n}\r\n"],
   "mappings": ";AAAA,SAAS,gBAAgB;AAUzB,SAAS,iBAAiB;AAKnB,aAAM,cAAc;AAAA,EACf,SAAiB;AAAA,EACjB,SAAiB;AAAA,EACjB,SAAiB;AAAA,EAEzB,IAAY,eAAe;AACvB,WAAO,KAAK,SAAS,KAAK;AAAA,EAC9B;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,SAAS,OAA8B;AACnC,SAAK,SAAS;AACd,SAAK,SAAS;AAEd,WAAO,KAAK,aAAa;AAAA,EAC7B;AAAA,EAEQ,eAAe;AACnB,QAAI,SAAwB,CAAC;AAC7B,QAAI,QAAQ,KAAK,YAAY;AAC7B,WAAO,OAAO;AACV,aAAO,KAAK,KAAK;AACjB,cAAQ,KAAK,YAAY;AAAA,IAC7B;AAEA,WAAO;AAAA,EACX;AAAA,EAEQ,cAAkC;AACtC,QAAI,QAKW;AACf,QAAI,OAAiB;AACrB,SAAK,SAAS,KAAK;AAEnB,WAAO,KAAK,SAAS,KAAK,OAAO,QAAQ;AACrC,YAAM,kBAAkB,KAAK,OAAO,YAAY,KAAK,MAAM;AAE3D,UAAI,OAAO,oBAAoB,aAAa;AACxC,cAAM,IAAI,MAAM,2BAA2B;AAAA,MAC/C;AAEA,YAAM,YAAY,OAAO,cAAc,eAAe;AAEtD,UAAI,UAAU,QAAQ;AAClB,YAAI,cAAc,MAAM;AACpB,kBAAQ;AAAA,QACZ,WAAW,aAAa,SAAS,GAAG;AAChC,kBAAQ;AAAA,QACZ,OAAO;AACH,kBAAQ;AAAA,QACZ;AAAA,MACJ,WAAW,UAAU,gBAAgB;AACjC,YAAI,QAAQ,SAAS,GAAG;AACpB,cAAI,KAAK,iBAAiB,GAAG;AACzB,kBAAM,IAAI;AAAA,cACN;AAAA,YACJ;AAAA,UACJ;AACA,kBAAQ;AAAA,QACZ,WAAW,cAAc,KAAK;AAC1B,eAAK,UAAU,UAAU;AACzB,iBAAO;AACP;AAAA,QACJ,WAAW,aAAa,SAAS,GAAG;AAChC,iBAAO;AACP;AAAA,QACJ;AAAA,MACJ,WAAW,UAAU,iBAAiB;AAClC,YAAI,cAAc,KAAK;AACnB,eAAK,UAAU,UAAU;AACzB,iBAAO;AACP;AAAA,QACJ,WAAW,CAAC,QAAQ,SAAS,GAAG;AAC5B,iBAAO;AACP;AAAA,QACJ;AAAA,MACJ,WAAW,UAAU,cAAc;AAC/B,YAAI,CAAC,aAAa,SAAS,GAAG;AAC1B,iBAAO;AACP;AAAA,QACJ;AAAA,MACJ,WAAW,UAAU,QAAQ;AACzB,YAAI,aAAa,SAAS,KAAK,cAAc,MAAM;AAC/C,iBAAO;AACP;AAAA,QACJ;AAAA,MACJ;AAEA,WAAK,UAAU,UAAU;AAAA,IAC7B;AAEA,QAAI,CAAC,MAAM;AACP,UAAI,KAAK,UAAU,KAAK,OAAO,QAAQ;AACnC,YAAI,SAAS,kBAAkB,UAAU,iBAAiB;AACtD,iBAAO;AAAA,QACX,WAAW,UAAU,QAAQ;AACzB,iBAAO;AAAA,QACX,WAAW,UAAU,cAAc;AAC/B,iBAAO;AAAA,QACX;AAAA,MACJ;AAAA,IACJ;AAEA,QAAI,MAAM;AACN,aAAO,EAAE,IAAI,KAAK,QAAQ,KAAK,MAAM,GAAG,IAAI;AAAA,IAChD;AAEA,WAAO;AAAA,EACX;AACJ;AASO,aAAM,WAAW;AAAA,EACZ,QAAuB;AAAA,EACvB,gBAAyB;AAAA,EAEjC,SAAS,OAAwB;AAC7B,UAAM,eAAe,IAAI,cAAc,EAAE,SAAS,KAAK;AACvD,QAAI,SAAkB,CAAC;AAEvB,aAASA,MAAK,cAAc;AACxB,UAAIA,GAAE,SAAS,UAAU;AACrB,YAAI,SAAS,MAAM,UAAUA,GAAE,IAAI,OAAOA,GAAE,IAAI,GAAG;AACnD,cAAM,QAAQ,OAAO,SAAS,GAAG;AAEjC,YAAI,OAAO;AACP,mBAAS,OAAO,UAAU,GAAG,OAAO,SAAS,CAAC;AAAA,QAClD;AAEA,YAAI,cAAc;AAClB,iBAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACpC,cAAI,QAAQ,OAAO,CAAC,CAAC,GAAG;AACpB,0BAAc;AACd;AAAA,UACJ;AAAA,QACJ;AAEA,YAAI,SAAwB;AAC5B,YAAI,gBAAgB,GAAG;AACnB,gBAAM,IAAI;AAAA,YACN;AAAA,UACJ;AAAA,QACJ;AACA,YAAI,cAAc,GAAG;AACjB,mBAAS,SAAS,OAAO,UAAU,WAAW,CAAC;AAC/C,mBAAS,OAAO,UAAU,GAAG,WAAW;AAAA,QAC5C;AAEA,YAAI,OAAO,WAAW,GAAG;AACrB,cAAI,OAAO;AAGP,kBAAM,cAAc;AAAA,cAChB;AAAA,cACA,CAACA,OAAMA,GAAE,SAAS,YAAYA,GAAE,SAAS;AAAA,YAC7C;AACA,gBAAI,aAAa;AACb,uBAAS,YAAY;AAAA,YACzB;AAAA,UACJ;AAEA,cAAI,OAAO,WAAW,GAAG;AACrB,kBAAM,IAAI;AAAA,cACN,uCAAuCA,GAAE,IAAI,KAAK,IAAIA,GAAE,IAAI,GAAG;AAAA,YACnE;AAAA,UACJ;AAAA,QACJ;AAEA,eAAO;AAAA,UACH,OAAOA,GAAE,KAAK,QAAQ,QAAQ,QAAQ,QAAQ,OAAO;AAAA,QACzD;AAAA,MACJ,WAAWA,GAAE,SAAS,cAAc;AAChC,YAAI,SAAS,MAAM,UAAUA,GAAE,IAAI,OAAOA,GAAE,IAAI,GAAG;AACnD,eAAO,KAAK,WAAWA,GAAE,KAAK,MAAM,CAAC;AAAA,MACzC,WAAWA,GAAE,SAAS,QAAQ;AAC1B,YAAI,SAAS,MAAM,UAAUA,GAAE,IAAI,OAAOA,GAAE,IAAI,GAAG;AACnD,eAAO,KAAK,KAAKA,GAAE,KAAK,MAAM,CAAC;AAAA,MACnC;AAAA,IACJ;AAEA,WAAO;AAAA,EACX;AAAA,EAEA,MAAM,OAA0B;AAC5B,QAAI,OAAkB;AAAA,MAClB,MAAM;AAAA,MACN,SAAS,CAAC;AAAA,IACd;AAEA,UAAM,SAAS,KAAK,SAAS,KAAK;AAElC,QAAI,cAAc;AAClB,QAAI,gBAAgB;AACpB,QAAI,iBAAiB;AACrB,QAAI,0BAA0B;AAC9B,QAAI,oBAAoB;AACxB,QAAI,6BAA6B;AACjC,QAAI,wBAAwB;AAC5B,QAAI,yBAAyB;AAC7B,QAAI,yBAAyB;AAC7B,QAAI,+BAA+B;AACnC,QAAI,wBAAwB;AAC5B,QAAI,0BAA0B;AAC9B,QAAI,0BAA0B;AAC9B,QAAI,0BAA0B;AAE9B,QAAI,oBAAoB;AACxB,QAAI,UAA0B;AAC9B,QAAI,YAA0B;AAC9B,QAAI,QAAsB;AAC1B,QAAI,WAAkC;AACtC,QAAI,QAAkB,CAAC;AACvB,QAAI,eAAsD,CAAC;AAC3D,QAAI,iBAAyB;AAC7B,QAAI,oBAAoB;AACxB,QAAI,WAA4B;AAEhC,SAAK,QAAQ;AAEb,UAAM,4BAA4B,MAAM;AACpC,UAAI,MAAM,SAAS,GAAG;AAClB,cAAM,OAAO,KAAK,MAAM,MAAM,KAAK,EAAE,EAAE,QAAQ,CAAC;AAChD,YAAI,OAAO;AACP,gBAAM,QAAQ,KAAK,IAAI;AAAA,QAC3B,WAAW,UAAU;AACjB,mBAAS,QAAQ,KAAK,IAAI;AAAA,QAC9B,OAAO;AACH,uBAAa,KAAK,IAAI;AAAA,QAC1B;AACA,gBAAQ,CAAC;AAAA,MACb;AAAA,IACJ;AAEA,UAAM,2BAA2B,CAAC,UAAwB;AACtD,UAAI,CAAC,SAAS;AACV;AAAA,MACJ;AACA,UAAI,aAAa,SAAS,GAAG;AACzB,YAAI,QAAQ,QAAQ,KAAK,CAAC,MAAM,EAAE,SAAS,OAAO,GAAG;AACjD,eAAK;AAAA,YACD;AAAA,YACA;AAAA,YACA;AAAA,UACJ;AAAA,QACJ;AAEA,gBAAQ;AAAA,UACJ,MAAM;AAAA,UACN,QAAQ;AAAA,UACR,SAAS;AAAA,QACb;AACA,gBAAQ,QAAQ,KAAK,KAAK;AAC1B,uBAAe,CAAC;AAAA,MACpB;AAAA,IACJ;AAEA,UAAM,eAAe,MAAM;AACvB,UAAI,CAAC,SAAS,CAAC,SAAS;AACpB;AAAA,MACJ;AACA,UAAI,iBAAqC,CAAC;AAC1C,eAAS,IAAI,MAAM,QAAQ,SAAS,GAAG,KAAK,GAAG,KAAK;AAChD,YAAI,UAAU,MAAM,QAAQ,CAAC;AAC7B,YAAI,OAAO,YAAY,YAAY,aAAa,SAAS;AAErD,yBAAe,QAAQ;AAAA,YACnB,MAAM;AAAA,YACN,SAAS,CAAC,QAAQ,OAAO;AAAA,UAC7B,CAAC;AACD,gBAAM,QAAQ,OAAO,GAAG,CAAC;AAAA,QAC7B,WACI,OAAO,YAAY,YACnB,eAAe,WACf,QAAQ,WACV;AAEE,yBAAe,QAAQ;AAAA,YACnB,MAAM;AAAA,UACV,CAAC;AACD,gBAAM,QAAQ,OAAO,GAAG,CAAC;AAAA,QAC7B,OAAO;AACH;AAAA,QACJ;AAAA,MACJ;AAEA,eAAS,WAAW,gBAAgB;AAChC,gBAAQ,QAAQ,KAAK,OAAO;AAAA,MAChC;AAAA,IACJ;AAEA,UAAM,0BAA0B,CAAC,UAAwB;AACrD,UAAI,SAAS,MAAM,MAAM,MAAM,GAAG;AAE9B,cAAM,QAAQ,QAAS,QAAQ,QAAQ,KAAK;AAC5C,YAAI,SAAS,GAAG;AACZ,kBAAS,QAAQ,OAAO,OAAO,CAAC;AAAA,QACpC;AACA,gBAAQ;AAAA,MACZ;AACA,UAAI,SAAS,UAAU;AACnB,kCAA0B;AAAA,MAC9B;AAEA,+BAAyB,KAAK;AAC9B,mBAAa;AAAA,IACjB;AAEA,UAAM,kBAAkB,MAAM;AAC1B,UAAI,0BAA0B,GAAG;AAC7B,YAAI,OAAO;AACP,oCAA0B;AAC1B,gBAAM,QAAQ,KAAK;AAAA,YACf,SAAS;AAAA,UACb,CAAC;AAAA,QACL,WAAW,SAAS;AAChB,kBAAQ,QAAQ,KAAK;AAAA,YACjB,MAAM;AAAA,YACN,SAAS,CAAC,cAAc;AAAA,UAC5B,CAAC;AAAA,QACL,OAAO;AACH,eAAK,QAAQ,KAAK;AAAA,YACd,MAAM;AAAA,YACN,SAAS,CAAC,cAAc;AAAA,UAC5B,CAAC;AAAA,QACL;AACA,yBAAiB;AACjB,kCAA0B;AAAA,MAC9B;AAAA,IACJ;AAEA,UAAM,qBAAqB,MAAM;AAC7B,UAAI,YAAY,MAAM,SAAS,GAAG;AAC9B,iBAAS,QAAQ,MAAM,KAAK,GAAG;AAC/B,gBAAQ,CAAC;AAAA,MACb;AAAA,IACJ;AAEA,aAAS,SAAS,QAAQ;AACtB,UAAI,MAAM,SAAS,UAAU;AACzB,YAAI,MAAM,YAAY,OAAO;AACzB,oCAA0B;AAC1B,uBAAa;AAEb,oBAAU;AAAA,YACN,MAAM;AAAA,YACN,QAAQ;AAAA,YACR,SAAS,CAAC;AAAA,YACV,WAAW,CAAC;AAAA,UAChB;AACA,kBAAQ;AACR,yBAAe,CAAC;AAEhB,eAAK,QAAQ,KAAK,OAAO;AAAA,QAC7B,WAAW,MAAM,YAAY,OAAO;AAChC,cAAI,CAAC,SAAS;AACV,iBAAK;AAAA,cACD;AAAA,cACA;AAAA,cACA;AAAA,YACJ;AAAA,UACJ,OAAO;AACH,4BAAgB;AAChB,oCAAwB,KAAK;AAE7B,wBAAY;AACZ,oBAAQ;AAAA,cACJ,MAAM;AAAA,cACN,QAAQ;AAAA,cACR,SAAS,CAAC;AAAA,YACd;AAEA,oBAAQ,QAAQ,KAAK,KAAK;AAAA,UAC9B;AAAA,QACJ,WAAW,MAAM,YAAY,OAAO;AAChC,cAAI,CAAC,SAAS;AACV,iBAAK;AAAA,cACD;AAAA,cACA;AAAA,cACA;AAAA,YACJ;AAAA,UACJ,OAAO;AACH,oCAAwB,KAAK;AAE7B,uBAAW;AAAA,cACP,MAAM;AAAA,cACN,SAAS,CAAC;AAAA,YACd;AAEA,oBAAQ,QAAQ,KAAK,QAAQ;AAAA,UACjC;AAAA,QACJ,WAAW,MAAM,YAAY,SAAS,MAAM,YAAY,OAAO;AAC3D,cAAI,CAAC,SAAS;AACV,iBAAK;AAAA,cACD;AAAA,cACA;AAAA,cACA;AAAA,YACJ;AAAA,UACJ,OAAO;AACH,gBAAI,OAAO;AACP,wCAA0B;AAC1B,oBAAM,QAAQ,KAAK;AAAA,gBACf,WAAW;AAAA,cACf,CAAC;AAAA,YACL,OAAO;AACH,sCAAwB,KAAK;AAC7B,sBAAQ,QAAQ,KAAK;AAAA,gBACjB,MAAM;AAAA,cACV,CAAC;AAAA,YACL;AAAA,UACJ;AAAA,QACJ,WAAW,MAAM,YAAY,OAAO;AAChC,oCAA0B;AAC1B,eAAK,QAAQ,MAAM;AAAA,QACvB,WAAW,MAAM,YAAY,OAAO;AAChC,oCAA0B;AAC1B,eAAK,QAAQ;AAAA,QACjB,WAAW,MAAM,YAAY,QAAQ;AACjC,wBAAc;AAAA,QAClB,WAAW,MAAM,YAAY,OAAO;AAChC,0BAAgB;AAChB,eAAK,SAAS;AAAA,QAClB,WACI,MAAM,YAAY,UAClB,MAAM,YAAY,SACpB;AACE,2BAAiB;AAAA,QACrB,WAAW,MAAM,YAAY,OAAO;AAChC,oCAA0B;AAAA,QAC9B,WAAW,MAAM,YAAY,OAAO;AAChC,mCAAyB;AAAA,QAC7B,WAAW,MAAM,YAAY,SAAS,mBAAmB;AACrD,cAAI,MAAM,SAAS,SAAS;AACxB,gBAAI,CAAC,SAAS;AACV,mBAAK;AAAA,gBACD;AAAA,gBACA;AAAA,gBACA;AAAA,gBACA;AAAA,cACJ;AAAA,YACJ,OAAO;AACH,wCAA0B;AAC1B,yBAAW;AAAA,gBACP,QAAQ;AAAA,gBACR,MAAM;AAAA,gBACN,QAAQ;AAAA,cACZ;AACA,oBAAM,MAAyB;AAAA,gBAC3B,QAAQ,SAAS;AAAA,cACrB;AACA,kCAAoB;AAEpB,sBAAQ,UAAU,KAAK,QAAQ;AAE/B,kBAAI,OAAO;AACP,sBAAM,QAAQ,KAAK,GAAG;AAAA,cAC1B,WAAW,UAAU;AACjB,yBAAS,QAAQ,KAAK,GAAG;AAAA,cAC7B,OAAO;AACH,6BAAa,KAAK,GAAG;AAAA,cACzB;AAEA,mCAAqB;AAAA,YACzB;AAAA,UACJ,OAAO;AACH,+BAAmB;AACnB,gCAAoB;AACpB,oCAAwB;AACxB,yCAA6B;AAC7B,uBAAW;AAAA,UACf;AAAA,QACJ,WAAW,MAAM,YAAY,UAAU,mBAAmB;AACtD,cAAI,CAAC,UAAU;AACX,iBAAK;AAAA,cACD;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,YACJ;AAAA,UACJ,OAAO;AACH,yCAA6B;AAAA,UACjC;AAAA,QACJ,WAAW,MAAM,YAAY,UAAU,mBAAmB;AACtD,cAAI,CAAC,UAAU;AACX,iBAAK;AAAA,cACD;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,YACJ;AAAA,UACJ,OAAO;AACH,oCAAwB;AAAA,UAC5B;AAAA,QACJ,WAAW,MAAM,YAAY,OAAO;AAChC,cAAI,MAAM,SAAS,SAAS;AACxB,qCAAyB;AAAA,UAC7B,OAAO;AACH,qCAAyB;AAAA,UAC7B;AAAA,QACJ,WAAW,MAAM,YAAY,QAAQ;AACjC,cAAI,MAAM,SAAS,SAAS;AACxB,2CAA+B;AAAA,UACnC,OAAO;AACH,2CAA+B;AAAA,UACnC;AAAA,QACJ,WACI,MAAM,YAAY,UAClB,MAAM,YAAY,SACpB;AACE,cAAI,MAAM,SAAS,SAAS;AACxB,sCAA0B;AAC1B,iBAAK,gBAAgB;AAAA,UACzB,OAAO;AACH,sCAA0B;AAC1B,iBAAK,gBAAgB;AAAA,UACzB;AAAA,QACJ,WAAW,MAAM,YAAY,QAAQ;AACjC,oCAA0B;AAC1B,8BAAoB;AAAA,QACxB,WAAW,MAAM,YAAY,OAAO;AAChC,cAAI,MAAM,SAAS,SAAS;AACxB,sCAA0B;AAAA,UAC9B,OAAO;AACH,sCAA0B;AAAA,UAC9B;AAAA,QACJ,WAAW,MAAM,QAAQ,QAAQ,GAAG,KAAK,GAAG;AACxC,cAAI,MAAM,SAAS,SAAS;AACxB,sCAA0B;AAAA,UAC9B;AAAA,QACJ,WAAW,MAAM,SAAS,OAAO;AAC7B,oCAA0B;AAAA,QAC9B;AAAA,MACJ,WAAW,MAAM,SAAS,QAAQ;AAC9B,YAAI,cAAc,GAAG;AACjB,cAAI,gBAAgB,GAAG;AACnB,iBAAK,KAAK,MAAM;AAChB,0BAAc;AAAA,UAClB;AAAA,QACJ,WAAW,gBAAgB,GAAG;AAC1B,cAAI,KAAK,QAAQ;AACb,iBAAK,UAAU,MAAM,MAAM;AAAA,UAC/B,OAAO;AACH,iBAAK,SAAS,MAAM;AAAA,UACxB;AAAA,QACJ,WAAW,iBAAiB,GAAG;AAC3B,cAAI,KAAK,OAAO;AACZ,iBAAK,SAAS,MAAM,MAAM;AAAA,UAC9B,OAAO;AACH,iBAAK,QAAQ,MAAM;AAAA,UACvB;AAAA,QACJ,WAAW,0BAA0B,GAAG;AACpC,cAAI,gBAAgB;AAChB,8BAAkB,MAAM,MAAM;AAAA,UAClC,OAAO;AACH,6BAAiB,MAAM;AAAA,UAC3B;AAAA,QACJ,WAAW,6BAA6B,GAAG;AACvC,cAAK,6BAA6B,GAAI;AAClC,kBAAM,CAACC,UAASC,MAAK,IAAI,MAAM,KAAK,MAAM,QAAQ;AAElD,gBAAI,UAAU;AACV,uBAAS,YAAY;AAAA,gBACjB,SAAS,SAASD,QAAO;AAAA,gBACzB,OAAO,SAASC,MAAK;AAAA,cACzB;AAAA,YACJ;AAEA,yCAA6B;AAAA,UACjC;AAAA,QACJ,WAAW,wBAAwB,GAAG;AAClC,gBAAM,KAAK,MAAM,IAAI;AAAA,QACzB,WAAW,oBAAoB,GAAG;AAC9B,cAAI,sBAAsB,GAAG;AACzB,gBAAI,MAAM,MAAM;AACZ,kBAAI,UAAU;AACV,oBAAI,MAAM,SAAS,OAAO,MAAM,SAAS,KAAK;AAC1C,2BAAS,SAAS,MAAM;AAAA,gBAC5B,OAAO;AACH,2BAAS,SAAS;AAAA,gBACtB;AAAA,cACJ;AAAA,YAEJ;AACA,gCAAoB;AAAA,UACxB,OAAO;AACH,kBAAM,KAAK,MAAM,IAAI;AAAA,UACzB;AAAA,QACJ,WAAW,yBAAyB,GAAG;AAAA,QAIvC,WAAW,0BAA0B,GAAG;AAAA,QAExC,WAAW,WAAW,MAAM,QAAQ,MAAM,GAAG;AACzC,kBAAQ,SAAS,SAAS,MAAM,IAAI;AACpC,cAAI,MAAM,QAAQ,MAAM,GAAG;AACvB,iBAAK;AAAA,cACD;AAAA,cACA;AAAA,cACA;AAAA,YACJ;AAAA,UACJ;AAAA,QACJ,WAAW,SAAS,MAAM,MAAM,MAAM,GAAG;AACrC,gBAAM,SAAS,SAAS,MAAM,IAAI;AAClC,cAAI,MAAM,MAAM,MAAM,GAAG;AACrB,iBAAK;AAAA,cACD;AAAA,cACA;AAAA,cACA;AAAA,YACJ;AAAA,UACJ;AAAA,QACJ,WAAW,yBAAyB,GAAG;AACnC,cAAI,2BAA2B,GAAG;AAC9B,kBAAM,wBAAwB,MAAM,KAAK,QAAQ,GAAG;AAEpD,gBAAI,yBAAyB,GAAG;AAC5B,oBAAM,OAAO,MAAM,KAAK;AAAA,gBACpB;AAAA,gBACA;AAAA,cACJ;AAEA,oBAAM,KAAK,IAAI;AACf,uCAAyB;AAAA,YAC7B,OAAO;AACH,oBAAM,KAAK,MAAM,IAAI;AAAA,YACzB;AAAA,UACJ;AAAA,QACJ,WAAW,+BAA+B,GAAG;AACzC,cAAI,iCAAiC,GAAG;AACpC,kBAAM,wBAAwB,MAAM,KAAK,QAAQ,GAAG;AAEpD,gBAAI,yBAAyB,GAAG;AAC5B,oBAAM,OAAO,MAAM,KAAK;AAAA,gBACpB;AAAA,gBACA;AAAA,cACJ;AAEA,oBAAM,KAAK,IAAI;AACf,6CAA+B;AAAA,YACnC,OAAO;AACH,oBAAM,KAAK,MAAM,IAAI;AAAA,YACzB;AAAA,UACJ;AAAA,QACJ,WAAW,0BAA0B,GAAG;AAAA,QACxC,WAAW,0BAA0B,GAAG;AAAA,QAExC,OAAO;AACH,gBAAM,KAAK,MAAM,IAAI;AAAA,QACzB;AAAA,MACJ,WAAW,MAAM,SAAS,cAAc;AACpC,YAAI,cAAc,GAAG;AACjB,cAAI,MAAM,WAAW,SAAS,IAAI,GAAG;AACjC,0BAAc;AAAA,UAClB;AAAA,QACJ,WAAW,gBAAgB,GAAG;AAC1B,cAAI,MAAM,WAAW,SAAS,IAAI,GAAG;AACjC,4BAAgB;AAAA,UACpB;AAAA,QACJ,WAAW,iBAAiB,GAAG;AAC3B,cAAI,MAAM,WAAW,SAAS,IAAI,GAAG;AACjC,6BAAiB;AAAA,UACrB;AAAA,QACJ,WAAW,0BAA0B,GAAG;AACpC,cAAI,MAAM,WAAW,SAAS,IAAI,GAAG;AACjC,4BAAgB;AAAA,UACpB;AAAA,QACJ,WAAW,yBAAyB,GAAG;AACnC,cAAI,MAAM,WAAW,SAAS,IAAI,GAAG;AACjC,qCAAyB;AAAA,UAC7B;AAAA,QACJ,WAAW,0BAA0B,GAAG;AACpC,cAAI,MAAM,WAAW,SAAS,IAAI,GAAG;AACjC,sCAA0B;AAC1B,gCAAoB;AAAA,UACxB;AAAA,QACJ,WACI,cAAc,KACd,gBAAgB,KAChB,iBAAiB,KACjB,0BAA0B,KAC1B,oBAAoB,KACpB,6BAA6B,KAC7B,wBAAwB,KACxB,yBAAyB,KACzB,0BAA0B,KAC1B,yBAAyB,KACzB,+BAA+B,GACjC;AAAA,QAEF,WAAW,0BAA0B,GAAG;AACpC,cAAI,MAAM,WAAW,SAAS,IAAI,GAAG;AACjC,sCAA0B;AAAA,UAC9B;AAAA,QACJ,WAAW,MAAM,SAAS,GAAG;AACzB,cAAI,WAAW,MAAM,MAAM,SAAS,CAAC;AACrC,cAAI,aAAa,KAAK;AAClB,kBAAM,KAAK,GAAG;AAAA,UAClB;AAAA,QACJ;AAAA,MACJ;AAAA,IACJ;AAEA,4BAAwB,IAAI;AAE5B,WAAO;AAAA,EACX;AAAA,EAEA,eAAe,MAAyB;AACpC,QAAI,KAAK;AAET,QAAI,KAAK,QAAQ;AACb,YAAM,KAAK,KAAK,MAAM;AAAA;AAAA,IAC1B;AAEA,aAAS,KAAK,KAAK,SAAS;AACxB,UAAI,EAAE,SAAS,WAAW;AACtB,cAAM,MAAM,EAAE,QAAQ,KAAK,GAAG,CAAC;AAAA;AAAA,MACnC,WAAW,EAAE,SAAS,WAAW;AAC7B,cAAM,OAAO,EAAE,MAAM;AAAA;AAErB,iBAAS,WAAW,EAAE,SAAS;AAC3B,cAAI,QAAQ,SAAS,WAAW;AAC5B,kBAAM,QAAQ,QAAQ,QAAQ,KAAK,GAAG,CAAC;AAAA;AAAA,UAC3C,WAAW,QAAQ,SAAS,cAAc;AACtC,kBAAM;AAAA,UACV,WAAW,QAAQ,SAAS,SAAS;AACjC,kBAAM,OAAO,QAAQ,MAAM;AAC3B,qBAAS,KAAK,QAAQ,SAAS;AAC3B,kBAAI,OAAO,MAAM,UAAU;AACvB,sBAAM,IAAI;AAAA,cACd,WAAW,UAAU,GAAG;AACpB,sBAAM,EAAE,OAAO;AAAA,cACnB;AAAA,YACJ;AACA,kBAAM;AAAA,UACV;AAAA,QACJ;AAAA,MACJ;AAAA,IACJ;AAEA,WAAO;AAAA,EACX;AAAA,EAEQ,gBAAgB;AACpB,WAAO,KAAK,UAAU,QAAQ,KAAK;AAAA,EACvC;AAAA,EAEQ,MAAM,MAA6B;AACvC,QAAI,CAAC,KAAK,cAAc,GAAG;AACvB,aAAO;AAAA,IACX;AACA,UAAMF,KAAU;AAAA,MACZ;AAAA,IACJ;AAEA,QAAI,KAAK,UAAU,MAAM;AACrB,MAAAA,GAAE,OAAO,KAAK;AAAA,IAClB;AAEA,QAAI,KAAK,eAAe;AACpB,MAAAA,GAAE,eAAe;AAAA,IACrB;AAEA,WAAOA;AAAA,EACX;AAAA,EAEQ,YACJ,QACA,OACA,SACA,OAAgB,OACZ;AACJ,UAAM,SAAS,UAAU;AACzB,QAAI,OAAO;AACP,UAAI,OAAO;AACX,UAAI,SAAS;AAEb,UAAI,QAAQ,MAAM,IAAI;AAEtB,eAAS,IAAI,GAAG,IAAI,OAAO,KAAK;AAC5B,YAAI,OAAO,OAAO,CAAC;AACnB,YAAI,SAAS,MAAM;AACf,kBAAQ;AACR,mBAAS;AAAA,QACb,OAAO;AACH,oBAAU;AAAA,QACd;AAAA,MACJ;AAEA,UAAI,aAAa;AACjB,UAAI,MAAM,SAAS,QAAQ;AACvB,qBAAa;AAAA,MACjB,WAAW,MAAM,SAAS,UAAU;AAChC,qBAAa,OAAO,MAAM;AAAA,MAC9B,OAAO;AACH,qBAAa;AAAA,MACjB;AAEA,UAAI,MAAM;AACN,eAAO,KAAK,IAAI,IAAI,KAAK,MAAM,GAAG,UAAU,KAAK,OAAO,EAAE;AAAA,MAC9D,OAAO;AACH,cAAM,IAAI,MAAM,IAAI,IAAI,KAAK,MAAM,GAAG,UAAU,KAAK,OAAO,EAAE;AAAA,MAClE;AAAA,IACJ,OAAO;AACH,UAAI,MAAM;AACN,eAAO,KAAK,OAAO;AAAA,MACvB,OAAO;AACH,cAAM,IAAI,MAAM,OAAO;AAAA,MAC3B;AAAA,IACJ;AAAA,EACJ;AACJ;AAMO,gBAAS,QAAQ,MAAuB;AAC3C,SAAO,KAAK,WAAW,KAAK,QAAQ,OAAO,QAAQ;AACvD;AAMO,gBAAS,aAAa,MAAuB;AAChD,SAAO,SAAS,OAAO,SAAS,OAAQ,SAAS,QAAQ,SAAS;AACtE;AAEO,gBAAS,EAAEG,MAAqB,MAAsB;AACzD,SAAO;AAAA,IACH,KAAAA;AAAA,IACA;AAAA,EACJ;AACJ;AAOO,gBAAS,IAAI,OAAe,KAA6B;AAC5D,SAAO;AAAA,IACH;AAAA,IACA;AAAA,EACJ;AACJ;AASO,gBAAS,OACZA,MACA,SACA,SAAwB,MACxB,OAA4B,SACjB;AACX,SAAO;AAAA,IACH,MAAM;AAAA,IACN,KAAAA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACJ;AACJ;AAOO,gBAAS,KAAKA,MAAqBC,OAAyB;AAC/D,SAAO;AAAA,IACH,MAAM;AAAA,IACN,KAAAD;AAAA,IACA,MAAAC;AAAA,EACJ;AACJ;AAEO,gBAAS,WACZD,MACAE,aACe;AACf,SAAO;AAAA,IACH,MAAM;AAAA,IACN,KAAAF;AAAA,IACA,YAAAE;AAAA,EACJ;AACJ;",
   "names": ["t", "chapter", "verse", "loc", "word", "whitespace"]
 }

package/dist/esm/parser/usx-parser.js

@@ -4,12 +4,21 @@ import {
   children,
   isParent
 } from "./iterators.js";
+import {
+  addChapterWords,
+  collapseWhitespaceMap,
+  isAnnotatedContent,
+  readWordAnnotations,
+  remapChapterWords,
+  removeChapterWordsForContent,
+  trimWordRange
+} from "./words.js";
 import { KNOWN_SKIPPED_VERSES } from "../utils.js";
 var NodeType = /* @__PURE__ */ ((NodeType2) => {
   NodeType2[NodeType2["Text"] = 3] = "Text";
   return NodeType2;
 })(NodeType || {});
-export const PARSER_VERSION = "2";
+export const PARSER_VERSION = 4;
 export class USXParser {
   _domParser;
   _noteCounter = 0;
@@ -140,7 +149,7 @@ export class USXParser {
       }
     }
   }
-  *iterateVerseContent(chapter, verse, nodes) {
+  *iterateVerseContent(chapter, verse, nodes, previousVerse) {
     let lastParent = null;
     while (true) {
       const { done, value: node } = nodes.next();
@@ -166,21 +175,35 @@ export class USXParser {
             );
             if (previousStyle === style && lastParent !== parent) {
               lastParent = parent;
-              yield {
-                lineBreak: true
-              };
+              const firstChild = parent.firstElementChild;
+              const startsWithVerse = firstChild?.nodeName === "verse" && !firstChild.hasAttribute("eid") && parseInt(
+                firstChild.getAttribute("number") || "0",
+                10
+              ) === verse.number;
+              if (startsWithVerse && previousVerse && previousVerse.number !== verse.number) {
+                addOrJoin(previousVerse.content, {
+                  lineBreak: true
+                });
+              } else {
+                yield {
+                  lineBreak: true
+                };
+              }
             }
           }
         } else if (style === "d") {
           descriptive = true;
         }
       }
-      for (let content of this.iterateNodeTextContent(
+      for (let value of this.iterateNodeTextContent(
         nodes,
         node,
         chapter,
         verse
       )) {
+        const words = isAnnotatedContent(value) ? value.words : null;
+        const content = isAnnotatedContent(value) ? value.annotatedContent : value;
+        let result = content;
         if (poem !== null || descriptive !== null) {
           if (typeof content === "string") {
             let text = {
@@ -192,7 +215,7 @@ export class USXParser {
             if (descriptive !== null) {
               text.descriptive = true;
             }
-            yield text;
+            result = text;
           } else {
             let text = {
               ...content
@@ -205,10 +228,16 @@ export class USXParser {
                 text.descriptive = true;
               }
             }
-            yield text;
+            result = text;
           }
+        }
+        if (words) {
+          yield {
+            annotatedContent: result,
+            words
+          };
         } else {
-          yield content;
+          yield result;
         }
       }
     }
@@ -241,11 +270,28 @@ export class USXParser {
         }
       }
     }
-    this._lastVerse = verse;
-    for (let content of this.iterateVerseContent(chapter, verse, nodes)) {
-      addOrJoin(verse.content, content);
+    const previousVerse = this._currentChapter?.number === chapter.number ? this._lastVerse : null;
+    const words = [];
+    for (let content of this.iterateVerseContent(
+      chapter,
+      verse,
+      nodes,
+      previousVerse
+    )) {
+      if (isAnnotatedContent(content)) {
+        addOrJoin(
+          verse.content,
+          content.annotatedContent,
+          content.words,
+          words
+        );
+      } else {
+        addOrJoin(verse.content, content);
+      }
     }
-    trimContent(verse.content);
+    trimContent(verse.content, words);
+    addChapterWords(chapter, verse.number, words);
+    this._lastVerse = verse;
     return verse;
   }
   *parseHebrewSubtitle(para, chapter, nodes) {
@@ -262,7 +308,10 @@ export class USXParser {
         yield content;
         continue;
       }
-      addOrJoin(subtitle.content, content);
+      addOrJoin(
+        subtitle.content,
+        isAnnotatedContent(content) ? content.annotatedContent : content
+      );
     }
     trimContent(subtitle.content);
     if (subtitle.content.length > 0) {
@@ -348,18 +397,48 @@ export class USXParser {
   *iterateChar(nodes, node) {
     const style = node.getAttribute("style");
     let text = "";
+    const words = [];
+    let currentWord = null;
+    let currentAnnotations = null;
+    let currentStart = 0;
+    const endCurrentWord = () => {
+      if (currentAnnotations) {
+        const range = trimWordRange(text, currentStart, text.length);
+        if (range) {
+          words.push({
+            ...range,
+            annotations: currentAnnotations
+          });
+        }
+      }
+      currentWord = null;
+      currentAnnotations = null;
+    };
     for (let char of children(nodes, node)) {
-      if (char.nodeType === 3 /* Text */) {
-        text += char.textContent || "";
+      if (char.nodeType !== 3 /* Text */) {
+        continue;
+      }
+      const word = wordElementFor(char, node);
+      if (word !== currentWord) {
+        endCurrentWord();
+        currentWord = word;
+        currentStart = text.length;
+        currentAnnotations = word ? readWordAnnotations(word) : null;
       }
+      text += char.textContent || "";
     }
-    if (style === "wj") {
+    endCurrentWord();
+    const content = style === "wj" ? {
+      text,
+      wordsOfJesus: true
+    } : text;
+    if (words.length > 0) {
       yield {
-        text,
-        wordsOfJesus: true
+        annotatedContent: content,
+        words
       };
     } else {
-      yield text;
+      yield content;
     }
   }
 }
@@ -455,40 +534,82 @@ function* iterateCharContent(char) {
 function trimText(text) {
   return text.replace(/\s+/g, " ");
 }
-function trimContent(content) {
+function wordElementFor(node, root) {
+  let element = node.parentElement;
+  while (element) {
+    if (element.nodeName === "char" && element.getAttribute("style") === "w") {
+      return element;
+    }
+    if (element === root) {
+      return null;
+    }
+    element = element.parentElement;
+  }
+  return null;
+}
+function trimContent(content, words) {
   for (let i = 0; i < content.length; i++) {
     const value = content[i];
+    let text = null;
     if (typeof value === "string") {
-      content[i] = trimText(value).trim();
-      if (content[i] === "") {
-        content.splice(i, 1);
-        i--;
-        continue;
+      const collapsed = collapseWhitespaceMap(value);
+      content[i] = collapsed.text;
+      text = collapsed.text;
+      if (words) {
+        remapChapterWords(words, i, collapsed.map);
       }
     } else if (isVerseText(value)) {
-      value.text = trimText(value.text).trim();
-      if (value.text === "") {
-        content.splice(i, 1);
-        i--;
-        continue;
+      const collapsed = collapseWhitespaceMap(value.text);
+      value.text = collapsed.text;
+      text = collapsed.text;
+      if (words) {
+        remapChapterWords(words, i, collapsed.map);
       }
     }
+    if (text === "") {
+      content.splice(i, 1);
+      if (words) {
+        removeChapterWordsForContent(words, i);
+      }
+      i--;
+      continue;
+    }
   }
   return content;
 }
-function addOrJoin(array, value) {
+function addOrJoin(array, value, ranges, words) {
+  let contentIndex;
+  let offset;
   if (array.length === 0) {
+    contentIndex = 0;
+    offset = 0;
     array.push(value);
   } else {
     const last = array[array.length - 1];
     if (typeof last === "string" && typeof value === "string") {
+      contentIndex = array.length - 1;
+      offset = last.length;
       array[array.length - 1] = last + value;
     } else if (isVerseText(last) && isVerseText(value) && hasSameFormatting(last, value)) {
+      contentIndex = array.length - 1;
+      offset = last.text.length;
       last.text += value.text;
     } else {
+      contentIndex = array.length;
+      offset = 0;
       array.push(value);
     }
   }
+  if (ranges && words) {
+    for (let range of ranges) {
+      words.push({
+        contentIndex,
+        start: offset + range.start,
+        end: offset + range.end,
+        ...range.annotations
+      });
+    }
+  }
 }
 function isVerseText(value) {
   return typeof value === "object" && value !== null && "text" in value;