@helloao/tools 0.0.5 → 0.0.6-alpha

package/dist/esm/parser/usfm-parser.js

@@ -0,0 +1,749 @@
+"use strict";
+import { findLast } from "lodash";
+export class UsfmTokenizer {
+  _input = "";
+  _index = 0;
+  _start = 0;
+  get _tokenLength() {
+    return this._index - this._start;
+  }
+  /**
+   * Converts the given input into a list of tokens.
+   * @param input The input that should be tokenized.
+   */
+  tokenize(input) {
+    this._input = input;
+    this._index = 0;
+    return this._parseTokens();
+  }
+  _parseTokens() {
+    let tokens = [];
+    let token = this._parseToken();
+    while (token) {
+      tokens.push(token);
+      token = this._parseToken();
+    }
+    return tokens;
+  }
+  _parseToken() {
+    let state = "none";
+    let kind = null;
+    this._start = this._index;
+    while (this._index < this._input.length) {
+      const codePointNumber = this._input.codePointAt(this._index);
+      if (typeof codePointNumber === "undefined") {
+        throw new Error("Unable to get code point!");
+      }
+      const codePoint = String.fromCodePoint(codePointNumber);
+      if (state === "none") {
+        if (codePoint === "\\") {
+          state = "marker_start";
+        } else if (isWhitespace(codePoint)) {
+          state = "whitespace";
+        } else {
+          state = "word";
+        }
+      } else if (state === "marker_start") {
+        if (isDigit(codePoint)) {
+          if (this._tokenLength === 0) {
+            throw new Error(
+              "Invalid Marker: Markers must not contain only digits."
+            );
+          }
+          state = "marker_number";
+        } else if (codePoint === "*") {
+          this._index += codePoint.length;
+          kind = "marker";
+          break;
+        } else if (isWhitespace(codePoint)) {
+          kind = "marker";
+          break;
+        }
+      } else if (state === "marker_number") {
+        if (codePoint === "*") {
+          this._index += codePoint.length;
+          kind = "marker";
+          break;
+        } else if (!isDigit(codePoint)) {
+          kind = "marker";
+          break;
+        }
+      } else if (state === "whitespace") {
+        if (!isWhitespace(codePoint)) {
+          kind = "whitespace";
+          break;
+        }
+      } else if (state === "word") {
+        if (isWhitespace(codePoint) || codePoint === "\\") {
+          kind = "word";
+          break;
+        }
+      }
+      this._index += codePoint.length;
+    }
+    if (!kind) {
+      if (this._index >= this._input.length) {
+        if (state == "marker_start" || state === "marker_number") {
+          kind = "marker";
+        } else if (state === "word") {
+          kind = "word";
+        } else if (state === "whitespace") {
+          kind = "whitespace";
+        }
+      }
+    }
+    if (kind) {
+      return t(loc(this._start, this._index), kind);
+    }
+    return null;
+  }
+}
+export class UsfmParser {
+  _poem = null;
+  _wordsOfJesus = false;
+  tokenize(input) {
+    const simpleTokens = new UsfmTokenizer().tokenize(input);
+    let tokens = [];
+    for (let t2 of simpleTokens) {
+      if (t2.kind === "marker") {
+        let source = input.substring(t2.loc.start, t2.loc.end);
+        const isEnd = source.endsWith("*");
+        if (isEnd) {
+          source = source.substring(0, source.length - 1);
+        }
+        let numberIndex = -1;
+        for (let i = 0; i < source.length; i++) {
+          if (isDigit(source[i])) {
+            numberIndex = i;
+            break;
+          }
+        }
+        let number = null;
+        if (numberIndex === 1) {
+          throw new Error(
+            "Markers must not be made only of numbers!"
+          );
+        }
+        if (numberIndex > 0) {
+          number = parseInt(source.substring(numberIndex));
+          source = source.substring(0, numberIndex);
+        }
+        if (source.length === 1) {
+          if (isEnd) {
+            const startMarker = findLast(
+              tokens,
+              (t3) => t3.kind === "marker" && t3.type === "start"
+            );
+            if (startMarker) {
+              source = startMarker.command;
+            }
+          }
+          if (source.length === 1) {
+            throw new Error(
+              `Markers must have a command! Token: ${t2.loc.start}-${t2.loc.end}`
+            );
+          }
+        }
+        tokens.push(
+          marker(t2.loc, source, number, isEnd ? "end" : "start")
+        );
+      } else if (t2.kind === "whitespace") {
+        let source = input.substring(t2.loc.start, t2.loc.end);
+        tokens.push(whitespace(t2.loc, source));
+      } else if (t2.kind === "word") {
+        let source = input.substring(t2.loc.start, t2.loc.end);
+        tokens.push(word(t2.loc, source));
+      }
+    }
+    return tokens;
+  }
+  parse(input) {
+    let root = {
+      type: "root",
+      content: []
+    };
+    const tokens = this.tokenize(input);
+    let expectingId = 0;
+    let expectingName = 0;
+    let expectingTitle = 0;
+    let expectingSectionHeading = 0;
+    let expectingFootnote = 0;
+    let expectingFootnoteReference = 0;
+    let expectingFootnoteText = 0;
+    let expectingReferenceText = 0;
+    let expectingWordAttribute = 0;
+    let expectingNestedWordAttribute = 0;
+    let expectingWordsOfJesus = 0;
+    let expectingIntroParagraph = 0;
+    let expectingCrossReference = 0;
+    let expectingUnknownCommand = 0;
+    let canParseFootnotes = true;
+    let chapter = null;
+    let lastVerse = null;
+    let verse = null;
+    let subtitle = null;
+    let words = [];
+    let verseContent = [];
+    let sectionContent = "";
+    let currentFootnoteId = 0;
+    let footnote = null;
+    this._poem = null;
+    const addWordsToVerseOrSubtitle = () => {
+      if (words.length > 0) {
+        const text = this._text(words.join("").trimEnd());
+        if (verse) {
+          verse.content.push(text);
+        } else if (subtitle) {
+          subtitle.content.push(text);
+        } else {
+          verseContent.push(text);
+        }
+        words = [];
+      }
+    };
+    const addVerseContentToChapter = (token) => {
+      if (!chapter) {
+        return;
+      }
+      if (verseContent.length > 0) {
+        if (chapter.content.some((c) => c.type === "verse")) {
+          this._throwError(
+            input,
+            token,
+            "Cannot infer first verse after other verses have been added to the chapter!"
+          );
+        }
+        verse = {
+          type: "verse",
+          number: 1,
+          content: verseContent
+        };
+        chapter.content.push(verse);
+        verseContent = [];
+      }
+    };
+    const cleanupVerse = () => {
+      if (!verse || !chapter) {
+        return;
+      }
+      let chapterContent = [];
+      for (let i = verse.content.length - 1; i >= 0; i--) {
+        let content = verse.content[i];
+        if (typeof content === "object" && "heading" in content) {
+          chapterContent.unshift({
+            type: "heading",
+            content: [content.heading]
+          });
+          verse.content.splice(i, 1);
+        } else if (typeof content === "object" && "lineBreak" in content && content.lineBreak) {
+          chapterContent.unshift({
+            type: "line_break"
+          });
+          verse.content.splice(i, 1);
+        } else {
+          break;
+        }
+      }
+      for (let content of chapterContent) {
+        chapter.content.push(content);
+      }
+    };
+    const completeVerseOrSubtitle = (token) => {
+      if (verse && isNaN(verse.number)) {
+        const index = chapter.content.indexOf(verse);
+        if (index >= 0) {
+          chapter.content.splice(index, 1);
+        }
+        verse = null;
+      }
+      if (verse || subtitle) {
+        addWordsToVerseOrSubtitle();
+      }
+      addVerseContentToChapter(token);
+      cleanupVerse();
+    };
+    const completeSection = () => {
+      if (expectingSectionHeading > 0) {
+        if (verse) {
+          addWordsToVerseOrSubtitle();
+          verse.content.push({
+            heading: sectionContent
+          });
+        } else if (chapter) {
+          chapter.content.push({
+            type: "heading",
+            content: [sectionContent]
+          });
+        } else {
+          root.content.push({
+            type: "heading",
+            content: [sectionContent]
+          });
+        }
+        sectionContent = "";
+        expectingSectionHeading = 0;
+      }
+    };
+    const addWordsToFootnote = () => {
+      if (footnote && words.length > 0) {
+        footnote.text += words.join(" ");
+        words = [];
+      }
+    };
+    for (let token of tokens) {
+      if (token.kind === "marker") {
+        if (token.command === "\\c") {
+          addWordsToVerseOrSubtitle();
+          cleanupVerse();
+          chapter = {
+            type: "chapter",
+            number: NaN,
+            content: [],
+            footnotes: []
+          };
+          verse = null;
+          verseContent = [];
+          root.content.push(chapter);
+        } else if (token.command === "\\v") {
+          if (!chapter) {
+            this._throwError(
+              input,
+              token,
+              "Cannot parse a verse without chapter information!"
+            );
+          } else {
+            completeSection();
+            completeVerseOrSubtitle(token);
+            lastVerse = verse;
+            verse = {
+              type: "verse",
+              number: NaN,
+              content: []
+            };
+            chapter.content.push(verse);
+          }
+        } else if (token.command === "\\d") {
+          if (!chapter) {
+            this._throwError(
+              input,
+              token,
+              "Cannot parse a hebrew subtitle without chapter information!"
+            );
+          } else {
+            completeVerseOrSubtitle(token);
+            subtitle = {
+              type: "hebrew_subtitle",
+              content: []
+            };
+            chapter.content.push(subtitle);
+          }
+        } else if (token.command === "\\b" || token.command === "\\p") {
+          if (!chapter) {
+            this._throwError(
+              input,
+              token,
+              "Cannot parse a line break without chapter information!"
+            );
+          } else {
+            if (verse) {
+              addWordsToVerseOrSubtitle();
+              verse.content.push({
+                lineBreak: true
+              });
+            } else {
+              completeVerseOrSubtitle(token);
+              chapter.content.push({
+                type: "line_break"
+              });
+            }
+          }
+        } else if (token.command === "\\q") {
+          addWordsToVerseOrSubtitle();
+          this._poem = token.number;
+        } else if (token.command === "\\p") {
+          addWordsToVerseOrSubtitle();
+          this._poem = null;
+        } else if (token.command === "\\id") {
+          expectingId = 1;
+        } else if (token.command === "\\h") {
+          expectingName = 1;
+          root.header = void 0;
+        } else if (token.command === "\\mt" || token.command === "\\+mt") {
+          expectingTitle = 1;
+        } else if (token.command === "\\s") {
+          expectingSectionHeading = 1;
+        } else if (token.command === "\\r") {
+          expectingReferenceText = 1;
+        } else if (token.command === "\\f" && canParseFootnotes) {
+          if (token.type === "start") {
+            if (!chapter) {
+              this._throwError(
+                input,
+                token,
+                "Cannot start a footnote outside of a chapter!",
+                true
+              );
+            } else {
+              addWordsToVerseOrSubtitle();
+              footnote = {
+                noteId: currentFootnoteId,
+                text: "",
+                caller: null
+              };
+              const ref = {
+                noteId: footnote.noteId
+              };
+              expectingFootnote = 1;
+              chapter.footnotes.push(footnote);
+              if (verse) {
+                verse.content.push(ref);
+              } else if (subtitle) {
+                subtitle.content.push(ref);
+              } else {
+                verseContent.push(ref);
+              }
+              currentFootnoteId += 1;
+            }
+          } else {
+            addWordsToFootnote();
+            expectingFootnote = 0;
+            expectingFootnoteText = 0;
+            expectingFootnoteReference = 0;
+            footnote = null;
+          }
+        } else if (token.command === "\\fr" && canParseFootnotes) {
+          if (!footnote) {
+            this._throwError(
+              input,
+              token,
+              "Cannot start a footnote reference outside of a footnote!",
+              true
+            );
+          } else {
+            expectingFootnoteReference = 1;
+          }
+        } else if (token.command === "\\ft" && canParseFootnotes) {
+          if (!footnote) {
+            this._throwError(
+              input,
+              token,
+              "Cannot start footnote text outside of a footnote!",
+              true
+            );
+          } else {
+            expectingFootnoteText = 1;
+          }
+        } else if (token.command === "\\w") {
+          if (token.type === "start") {
+            expectingWordAttribute = 1;
+          } else {
+            expectingWordAttribute = 0;
+          }
+        } else if (token.command === "\\+w") {
+          if (token.type === "start") {
+            expectingNestedWordAttribute = 1;
+          } else {
+            expectingNestedWordAttribute = 0;
+          }
+        } else if (token.command === "\\wj" || token.command === "\\+wj") {
+          if (token.type === "start") {
+            addWordsToVerseOrSubtitle();
+            this._wordsOfJesus = true;
+          } else {
+            addWordsToVerseOrSubtitle();
+            this._wordsOfJesus = false;
+          }
+        } else if (token.command === "\\ip") {
+          expectingIntroParagraph = 1;
+          canParseFootnotes = false;
+        } else if (token.command === "\\x") {
+          if (token.type === "start") {
+            expectingCrossReference = 1;
+          } else {
+            expectingCrossReference = 0;
+          }
+        } else if (token.command.indexOf("-") >= 0) {
+          if (token.type === "start") {
+            expectingUnknownCommand = 1;
+          }
+        } else if (token.type === "end") {
+          expectingUnknownCommand = 0;
+        }
+      } else if (token.kind === "word") {
+        if (expectingId > 0) {
+          if (expectingId === 1) {
+            root.id = token.word;
+            expectingId = 2;
+          }
+        } else if (expectingName > 0) {
+          if (root.header) {
+            root.header += " " + token.word;
+          } else {
+            root.header = token.word;
+          }
+        } else if (expectingTitle > 0) {
+          if (root.title) {
+            root.title += " " + token.word;
+          } else {
+            root.title = token.word;
+          }
+        } else if (expectingSectionHeading > 0) {
+          if (sectionContent) {
+            sectionContent += " " + token.word;
+          } else {
+            sectionContent = token.word;
+          }
+        } else if (expectingFootnoteReference > 0) {
+          if (expectingFootnoteReference = 1) {
+            const [chapter2, verse2] = token.word.split(/[\.\:]/);
+            if (footnote) {
+              footnote.reference = {
+                chapter: parseInt(chapter2),
+                verse: parseInt(verse2)
+              };
+            }
+            expectingFootnoteReference = 0;
+          }
+        } else if (expectingFootnoteText > 0) {
+          words.push(token.word);
+        } else if (expectingFootnote > 0) {
+          if (expectingFootnote === 1) {
+            if (token.word) {
+              if (footnote) {
+                if (token.word === "+" || token.word !== "-") {
+                  footnote.caller = token.word;
+                } else {
+                  footnote.caller = null;
+                }
+              }
+            }
+            expectingFootnote = 2;
+          } else {
+            words.push(token.word);
+          }
+        } else if (expectingReferenceText > 0) {
+        } else if (expectingCrossReference > 0) {
+        } else if (chapter && isNaN(chapter.number)) {
+          chapter.number = parseInt(token.word);
+          if (isNaN(chapter.number)) {
+            this._throwError(
+              input,
+              token,
+              "The first word token after a chapter marker must be parsable to an integer!"
+            );
+          }
+        } else if (verse && isNaN(verse.number)) {
+          verse.number = parseInt(token.word);
+          if (isNaN(verse.number)) {
+            this._throwError(
+              input,
+              token,
+              "The first word token after a verse marker must be parsable to an integer!"
+            );
+          }
+        } else if (expectingWordAttribute > 0) {
+          if (expectingWordAttribute === 1) {
+            const firstVerticalBarIndex = token.word.indexOf("|");
+            if (firstVerticalBarIndex >= 0) {
+              const name = token.word.slice(
+                0,
+                firstVerticalBarIndex
+              );
+              words.push(name);
+              expectingWordAttribute = 2;
+            } else {
+              words.push(token.word);
+            }
+          }
+        } else if (expectingNestedWordAttribute > 0) {
+          if (expectingNestedWordAttribute === 1) {
+            const firstVerticalBarIndex = token.word.indexOf("|");
+            if (firstVerticalBarIndex >= 0) {
+              const name = token.word.slice(
+                0,
+                firstVerticalBarIndex
+              );
+              words.push(name);
+              expectingNestedWordAttribute = 2;
+            } else {
+              words.push(token.word);
+            }
+          }
+        } else if (expectingUnknownCommand > 0) {
+        } else if (expectingIntroParagraph > 0) {
+        } else {
+          words.push(token.word);
+        }
+      } else if (token.kind === "whitespace") {
+        if (expectingId > 0) {
+          if (token.whitespace.includes("\n")) {
+            expectingId = 0;
+          }
+        } else if (expectingName > 0) {
+          if (token.whitespace.includes("\n")) {
+            expectingName = 0;
+          }
+        } else if (expectingTitle > 0) {
+          if (token.whitespace.includes("\n")) {
+            expectingTitle = 0;
+          }
+        } else if (expectingSectionHeading > 0) {
+          if (token.whitespace.includes("\n")) {
+            completeSection();
+          }
+        } else if (expectingReferenceText > 0) {
+          if (token.whitespace.includes("\n")) {
+            expectingReferenceText = 0;
+          }
+        } else if (expectingIntroParagraph > 0) {
+          if (token.whitespace.includes("\n")) {
+            expectingIntroParagraph = 0;
+            canParseFootnotes = true;
+          }
+        } else if (expectingId > 0 || expectingName > 0 || expectingTitle > 0 || expectingSectionHeading > 0 || expectingFootnote > 0 || expectingFootnoteReference > 0 || expectingFootnoteText > 0 || expectingReferenceText > 0 || expectingCrossReference > 0 || expectingWordAttribute > 0 || expectingNestedWordAttribute > 0) {
+        } else if (expectingUnknownCommand > 0) {
+          if (token.whitespace.includes("\n")) {
+            expectingUnknownCommand = 0;
+          }
+        } else if (words.length > 0) {
+          let lastWord = words[words.length - 1];
+          if (lastWord !== " ") {
+            words.push(" ");
+          }
+        }
+      }
+    }
+    completeVerseOrSubtitle(null);
+    return root;
+  }
+  renderMarkdown(tree) {
+    let md = "";
+    if (tree.header) {
+      md += `# ${tree.header}
+`;
+    }
+    for (let c of tree.content) {
+      if (c.type === "heading") {
+        md += `## ${c.content.join(" ")}
+`;
+      } else if (c.type === "chapter") {
+        md += `### ${c.number}
+`;
+        for (let content of c.content) {
+          if (content.type === "heading") {
+            md += `#### ${content.content.join(" ")}
+`;
+          } else if (content.type === "line_break") {
+            md += "\n\n";
+          } else if (content.type === "verse") {
+            md += `<em>${content.number}</em>`;
+            for (let v of content.content) {
+              if (typeof v === "string") {
+                md += v + " ";
+              } else if ("text" in v) {
+                md += v.text + " ";
+              }
+            }
+            md += "\n";
+          }
+        }
+      }
+    }
+    return md;
+  }
+  _hasAttribute() {
+    return this._poem !== null || this._wordsOfJesus;
+  }
+  _text(text) {
+    if (!this._hasAttribute()) {
+      return text;
+    }
+    const t2 = {
+      text
+    };
+    if (this._poem !== null) {
+      t2.poem = this._poem;
+    }
+    if (this._wordsOfJesus) {
+      t2.wordsOfJesus = true;
+    }
+    return t2;
+  }
+  _throwError(source, token, message, warn = false) {
+    if (token) {
+      let line = 1;
+      let column = 1;
+      let start = token.loc.start;
+      for (let i = 0; i < start; i++) {
+        let char = source[i];
+        if (char === "\n") {
+          line += 1;
+          column = 1;
+        } else {
+          column += 1;
+        }
+      }
+      let tokenDebug = "";
+      if (token.kind === "word") {
+        tokenDebug = ", word";
+      } else if (token.kind === "marker") {
+        tokenDebug = ", " + token.command;
+      } else {
+        tokenDebug = "";
+      }
+      if (warn) {
+        console.warn(`(${line}, ${column}${tokenDebug}) ${message}`);
+      } else {
+        throw new Error(`(${line}, ${column}${tokenDebug}) ${message}`);
+      }
+    } else {
+      if (warn) {
+        console.warn(message);
+      } else {
+        throw new Error(message);
+      }
+    }
+  }
+}
+export function isDigit(char) {
+  return char.length === 1 && char >= "0" && char <= "9";
+}
+export function isWhitespace(char) {
+  return char === " " || char === "	" || char === "\n" || char === "\r";
+}
+export function t(loc2, kind) {
+  return {
+    loc: loc2,
+    kind
+  };
+}
+export function loc(start, end) {
+  return {
+    start,
+    end
+  };
+}
+export function marker(loc2, command, number = null, type = "start") {
+  return {
+    kind: "marker",
+    loc: loc2,
+    command,
+    number,
+    type
+  };
+}
+export function word(loc2, word2) {
+  return {
+    kind: "word",
+    loc: loc2,
+    word: word2
+  };
+}
+export function whitespace(loc2, whitespace2) {
+  return {
+    kind: "whitespace",
+    loc: loc2,
+    whitespace: whitespace2
+  };
+}
+//# sourceMappingURL=usfm-parser.js.map