axyseo 2.1.12 → 2.1.13

package/build/cjs/parsedPaper/build/linguisticParsing/SentenceTokenizer.js

@@ -0,0 +1,294 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _lodash = require("lodash");
+var _core = _interopRequireDefault(require("tokenizer2/core"));
+var _quotes = require("../../../languageProcessing/helpers/sanitize/quotes");
+var _Sentence = _interopRequireDefault(require("./Sentence"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+// All characters that indicate a sentence delimiter.
+const fullStop = ".";
+// The \u2026 character is an ellipsis
+const sentenceDelimiters = "?!;\u2026";
+const fullStopRegex = new RegExp("^[" + fullStop + "]$");
+const sentenceDelimiterRegex = new RegExp("^[" + sentenceDelimiters + "]$");
+const sentenceRegex = new RegExp("^[^" + fullStop + sentenceDelimiters + "\\(\\)\\[\\]]+$");
+const blockStartRegex = /^\s*[[({]\s*$/;
+const blockEndRegex = /^\s*[\])}]\s*$/;
+const whiteSpaceStartRegex = /^\s*/;
+const whiteSpaceEndRegex = /\s*$/;
+
+/**
+ * Class for tokenizing a (html) text into sentences.
+ */
+class SentenceTokenizer {
+  /**
+   * Returns whether or not a certain character is a number.
+   *
+   * @param {string} character The character to check.
+   * @returns {boolean} Whether or not the character is a capital letter.
+   */
+  isNumber(character) {
+    return !(0, _lodash.isNaN)(parseInt(character, 10));
+  }
+
+  /**
+   * Returns whether or not a given character is quotation mark.
+   *
+   * @param {string} character The character to check.
+   *
+   * @returns {boolean} Whether or not the given character is a quotation mark.
+   */
+  isQuotation(character) {
+    character = (0, _quotes.normalize)(character);
+    return "'" === character || "\"" === character;
+  }
+
+  /**
+   * Returns whether or not a given character is a punctuation mark that can be at the beginning
+   * of a sentence, like ¿ and ¡ used in Spanish.
+   *
+   * @param {string} character The character to check.
+   *
+   * @returns {boolean} Whether or not the given character is a punctuation mark.
+   */
+  isPunctuation(character) {
+    return "¿" === character || "¡" === character;
+  }
+
+  /**
+   * Removes duplicate whitespace from a given text.
+   *
+   * @param {string} text The text with duplicate whitespace.
+   * @returns {string} The text without duplicate whitespace.
+   */
+  removeDuplicateWhitespace(text) {
+    return text.replace(/\s+/, " ");
+  }
+
+  /**
+   * Returns whether or not a certain character is a capital letter.
+   *
+   * @param {string} character The character to check.
+   *
+   * @returns {boolean} Whether or not the character is a capital letter.
+   */
+  isCapitalLetter(character) {
+    return character !== character.toLocaleLowerCase();
+  }
+
+  /**
+   * Retrieves the next two characters from an array with the two next tokens.
+   *
+   * @param {Array} nextTokens The two next tokens. Might be undefined.
+   *
+   * @returns {string} The next two characters.
+   */
+  getNextTwoCharacters(nextTokens) {
+    let next = "";
+    if (!(0, _lodash.isUndefined)(nextTokens[0])) {
+      next += nextTokens[0].src;
+    }
+    if (!(0, _lodash.isUndefined)(nextTokens[1])) {
+      next += nextTokens[1].src;
+    }
+    next = this.removeDuplicateWhitespace(next);
+    return next;
+  }
+
+  /**
+   * Checks if the sentenceBeginning beginning is a valid beginning.
+   *
+   * @param {string} sentenceBeginning The beginning of the sentence to validate.
+   *
+   * @returns {boolean} Returns true if it is a valid beginning, false if it is not.
+   */
+  isValidSentenceBeginning(sentenceBeginning) {
+    return this.isCapitalLetter(sentenceBeginning) || this.isNumber(sentenceBeginning) || this.isQuotation(sentenceBeginning) || this.isPunctuation(sentenceBeginning);
+  }
+
+  /**
+   * Checks if the token is a valid sentence ending.
+   *
+   * @param {Object} token The token to validate.
+   *
+   * @returns {boolean} Returns true if the token is valid ending, false if it is not.
+   */
+  isSentenceStart(token) {
+    return !(0, _lodash.isUndefined)(token) && ("html-start" === token.type || "html-end" === token.type || "block-start" === token.type);
+  }
+
+  /**
+   * Creates a tokenizer.
+   *
+   * @returns {Object} The tokenizer and the tokens.
+   */
+  createTokenizer() {
+    const tokens = [];
+    const tokenizer = (0, _core.default)(function (token) {
+      tokens.push(token);
+    });
+    tokenizer.addRule(fullStopRegex, "full-stop");
+    tokenizer.addRule(blockStartRegex, "block-start");
+    tokenizer.addRule(blockEndRegex, "block-end");
+    tokenizer.addRule(sentenceDelimiterRegex, "sentence-delimiter");
+    tokenizer.addRule(sentenceRegex, "sentence");
+    return {
+      tokenizer,
+      tokens
+    };
+  }
+
+  /**
+   * Tokenizes the given text using the given tokenizer.
+   *
+   * @param {Tokenizer} tokenizer The tokenizer to use.
+   * @param {string}    text      The text to tokenize.
+   *
+   * @returns {void}
+   */
+  tokenize(tokenizer, text) {
+    tokenizer.onText(text);
+    try {
+      tokenizer.end();
+    } catch (e) {
+      console.error("Tokenizer end error:", e, e.tokenizer2);
+    }
+  }
+
+  /**
+   * Determines the start and end indices of a set of sentences form a text.
+   *
+   * @param {Sentence[]} sentences A set of sentences for which to determine indices.
+   *
+   * @returns {void}
+   */
+  determineIndices(sentences) {
+    let currentIndex = 0;
+    for (const sentence of sentences) {
+      const startIndex = currentIndex;
+      sentence.setStartIndex(currentIndex);
+      const endIndex = startIndex + sentence.text.length - 1;
+      sentence.setEndIndex(endIndex);
+      currentIndex = endIndex + 1;
+    }
+  }
+
+  /**
+   * Trims the white space from the beginning of a sentence and adjusts the sentence start index accordingly.
+   *
+   * @param {Sentence} sentence The sentence for which to trim the white space at the start.
+   *
+   * @returns {void}
+   */
+  trimWhiteSpaceAtStart(sentence) {
+    const whiteSpaceLength = sentence.text.match(whiteSpaceStartRegex)[0].length;
+    sentence.setText(sentence.getText().slice(whiteSpaceLength));
+    sentence.setStartIndex(sentence.getStartIndex() + whiteSpaceLength);
+  }
+
+  /**
+   * Trims the white space from the end of a sentence and adjusts the sentence end index accordingly.
+   *
+   * @param {Sentence} sentence The sentence for which to trim the white space at the end.
+   *
+   * @returns {void}
+   */
+  trimWhiteSpaceAtEnd(sentence) {
+    const whiteSpaceLength = sentence.text.match(whiteSpaceEndRegex)[0].length;
+    sentence.setText(sentence.getText().slice(0, sentence.getText().length - whiteSpaceLength));
+    sentence.setEndIndex(sentence.getEndIndex() - whiteSpaceLength);
+  }
+
+  /**
+   * Trims white space from the beginning and end of sentences and adjusts the indices
+   * of the sentence beginnings and ends accordingly.
+   *
+   * @param {Sentence[]} sentences The sentences for which to trim the whitespace.
+   *
+   * @returns {void}
+   */
+  trimWhiteSpaces(sentences) {
+    for (const sentence of sentences) {
+      this.trimWhiteSpaceAtStart(sentence);
+      this.trimWhiteSpaceAtEnd(sentence);
+    }
+  }
+
+  /**
+   * Returns an array of sentence objects for a given array of tokens; assumes that the text has already been split into blocks.
+   *
+   * @param {Object[]} tokenArray The tokens from the sentence tokenizer.
+   *
+   * @returns {Sentence[]} An array of sentence objects.
+   */
+  getSentencesFromTokens(tokenArray) {
+    const tokenSentences = [];
+    let currentSentence = new _Sentence.default("", 0, 0),
+      nextSentenceStart;
+    tokenArray.forEach((token, i) => {
+      let hasNextSentence, nextCharacters;
+      const nextToken = tokenArray[i + 1];
+      const secondToNextToken = tokenArray[i + 2];
+      switch (token.type) {
+        case "sentence":
+          currentSentence.appendText(token.src);
+          break;
+        case "sentence-delimiter":
+          currentSentence.appendText(token.src);
+          if (!(0, _lodash.isUndefined)(nextToken) && "block-end" !== nextToken.type && "sentence-delimiter" !== nextToken.type) {
+            tokenSentences.push(currentSentence);
+            currentSentence = new _Sentence.default("");
+          }
+          break;
+        case "full-stop":
+          currentSentence.appendText(token.src);
+          nextCharacters = this.getNextTwoCharacters([nextToken, secondToNextToken]);
+
+          // For a new sentence we need to check the next two characters.
+          hasNextSentence = nextCharacters.length >= 2;
+          nextSentenceStart = hasNextSentence ? nextCharacters[1] : "";
+          // If the next character is a number, never split. For example: IPv4-numbers.
+          if (hasNextSentence && this.isNumber(nextCharacters[0])) {
+            break;
+          }
+          // Only split on sentence delimiters when the next sentence looks like the start of a sentence.
+          if (hasNextSentence && this.isValidSentenceBeginning(nextSentenceStart) || this.isSentenceStart(nextToken)) {
+            tokenSentences.push(currentSentence);
+            currentSentence = new _Sentence.default("");
+          }
+          break;
+        case "block-start":
+          currentSentence.appendText(token.src);
+          break;
+        case "block-end":
+          currentSentence.appendText(token.src);
+          nextCharacters = this.getNextTwoCharacters([nextToken, secondToNextToken]);
+
+          // For a new sentence we need to check the next two characters.
+          hasNextSentence = nextCharacters.length >= 2;
+          nextSentenceStart = hasNextSentence ? nextCharacters[0] : "";
+          // If the next character is a number, never split. For example: IPv4-numbers.
+          if (hasNextSentence && this.isNumber(nextCharacters[0])) {
+            break;
+          }
+          if (hasNextSentence && this.isValidSentenceBeginning(nextSentenceStart) || this.isSentenceStart(nextToken)) {
+            tokenSentences.push(currentSentence);
+            currentSentence = new _Sentence.default("");
+          }
+          break;
+      }
+    });
+    if (currentSentence.getText() !== "") {
+      tokenSentences.push(currentSentence);
+    }
+    this.determineIndices(tokenSentences);
+    this.trimWhiteSpaces(tokenSentences);
+    return tokenSentences;
+  }
+}
+exports.default = SentenceTokenizer;
+//# sourceMappingURL=SentenceTokenizer.js.map

package/build/cjs/parsedPaper/build/linguisticParsing/SentenceTokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SentenceTokenizer.js","names":["_lodash","require","_core","_interopRequireDefault","_quotes","_Sentence","e","__esModule","default","fullStop","sentenceDelimiters","fullStopRegex","RegExp","sentenceDelimiterRegex","sentenceRegex","blockStartRegex","blockEndRegex","whiteSpaceStartRegex","whiteSpaceEndRegex","SentenceTokenizer","isNumber","character","isNaN","parseInt","isQuotation","normalizeQuotes","isPunctuation","removeDuplicateWhitespace","text","replace","isCapitalLetter","toLocaleLowerCase","getNextTwoCharacters","nextTokens","next","isUndefined","src","isValidSentenceBeginning","sentenceBeginning","isSentenceStart","token","type","createTokenizer","tokens","tokenizer","core","push","addRule","tokenize","onText","end","console","error","tokenizer2","determineIndices","sentences","currentIndex","sentence","startIndex","setStartIndex","endIndex","length","setEndIndex","trimWhiteSpaceAtStart","whiteSpaceLength","match","setText","getText","slice","getStartIndex","trimWhiteSpaceAtEnd","getEndIndex","trimWhiteSpaces","getSentencesFromTokens","tokenArray","tokenSentences","currentSentence","Sentence","nextSentenceStart","forEach","i","hasNextSentence","nextCharacters","nextToken","secondToNextToken","appendText","exports"],"sources":["../../../../../src/parsedPaper/build/linguisticParsing/SentenceTokenizer.js"],"sourcesContent":["import { isNaN, isUndefined } from \"lodash\";\n\nimport core from \"tokenizer2/core\";\nimport { normalize as normalizeQuotes } from \"@axyseo/languageProcessing/helpers/sanitize/quotes\";\n\nimport Sentence from \"./Sentence\";\n\n// All characters that indicate a sentence delimiter.\nconst fullStop = \".\";\n// The \\u2026 character is an ellipsis\nconst sentenceDelimiters = \"?!;\\u2026\";\n\nconst fullStopRegex = new RegExp( \"^[\" + fullStop + \"]$\" );\nconst sentenceDelimiterRegex = new RegExp( \"^[\" + sentenceDelimiters + \"]$\" );\nconst sentenceRegex = new RegExp( \"^[^\" + fullStop + sentenceDelimiters + \"\\\\(\\\\)\\\\[\\\\]]+$\" );\n\nconst blockStartRegex = /^\\s*[[({]\\s*$/;\nconst blockEndRegex = /^\\s*[\\])}]\\s*$/;\n\nconst whiteSpaceStartRegex = /^\\s*/;\nconst whiteSpaceEndRegex = /\\s*$/;\n\n/**\n * Class for tokenizing a (html) text into sentences.\n */\nexport default class SentenceTokenizer {\n\t/**\n\t * Returns whether or not a certain character is a number.\n\t *\n\t * @param {string} character The character to check.\n\t * @returns {boolean} Whether or not the character is a capital letter.\n\t */\n\tisNumber( character ) {\n\t\treturn ! isNaN( parseInt( character, 10 ) );\n\t}\n\n\t/**\n\t * Returns whether or not a given character is quotation mark.\n\t *\n\t * @param {string} character The character to check.\n\t *\n\t * @returns {boolean} Whether or not the given character is a quotation mark.\n\t */\n\tisQuotation( character ) {\n\t\tcharacter = normalizeQuotes( character );\n\n\t\treturn \"'\" === character ||\n\t\t\t\"\\\"\" === character;\n\t}\n\n\t/**\n\t * Returns whether or not a given character is a punctuation mark that can be at the beginning\n\t * of a sentence, like ¿ and ¡ used in Spanish.\n\t *\n\t * @param {string} character The character to check.\n\t *\n\t * @returns {boolean} Whether or not the given character is a punctuation mark.\n\t */\n\tisPunctuation( character ) {\n\t\treturn \"¿\" === character ||\n\t\t\t\"¡\" === character;\n\t}\n\n\t/**\n\t * Removes duplicate whitespace from a given text.\n\t *\n\t * @param {string} text The text with duplicate whitespace.\n\t * @returns {string} The text without duplicate whitespace.\n\t */\n\tremoveDuplicateWhitespace( text ) {\n\t\treturn text.replace( /\\s+/, \" \" );\n\t}\n\n\t/**\n\t * Returns whether or not a certain character is a capital letter.\n\t *\n\t * @param {string} character The character to check.\n\t *\n\t * @returns {boolean} Whether or not the character is a capital letter.\n\t */\n\tisCapitalLetter( character ) {\n\t\treturn character !== character.toLocaleLowerCase();\n\t}\n\n\t/**\n\t * Retrieves the next two characters from an array with the two next tokens.\n\t *\n\t * @param {Array} nextTokens The two next tokens. Might be undefined.\n\t *\n\t * @returns {string} The next two characters.\n\t */\n\tgetNextTwoCharacters( nextTokens ) {\n\t\tlet next = \"\";\n\n\t\tif ( ! isUndefined( nextTokens[ 0 ] ) ) {\n\t\t\tnext += nextTokens[ 0 ].src;\n\t\t}\n\n\t\tif ( ! isUndefined( nextTokens[ 1 ] ) ) {\n\t\t\tnext += nextTokens[ 1 ].src;\n\t\t}\n\n\t\tnext = this.removeDuplicateWhitespace( next );\n\n\t\treturn next;\n\t}\n\n\t/**\n\t * Checks if the sentenceBeginning beginning is a valid beginning.\n\t *\n\t * @param {string} sentenceBeginning The beginning of the sentence to validate.\n\t *\n\t * @returns {boolean} Returns true if it is a valid beginning, false if it is not.\n\t */\n\tisValidSentenceBeginning( sentenceBeginning ) {\n\t\treturn (\n\t\t\tthis.isCapitalLetter( sentenceBeginning ) ||\n\t\t\tthis.isNumber( sentenceBeginning ) ||\n\t\t\tthis.isQuotation( sentenceBeginning ) ||\n\t\t\tthis.isPunctuation( sentenceBeginning )\n\t\t);\n\t}\n\n\t/**\n\t * Checks if the token is a valid sentence ending.\n\t *\n\t * @param {Object} token The token to validate.\n\t *\n\t * @returns {boolean} Returns true if the token is valid ending, false if it is not.\n\t */\n\tisSentenceStart( token ) {\n\t\treturn ( ! isUndefined( token ) && (\n\t\t\t\"html-start\" === token.type ||\n\t\t\t\"html-end\" === token.type ||\n\t\t\t\"block-start\" === token.type\n\t\t) );\n\t}\n\n\t/**\n\t * Creates a tokenizer.\n\t *\n\t * @returns {Object} The tokenizer and the tokens.\n\t */\n\tcreateTokenizer() {\n\t\tconst tokens = [];\n\t\tconst tokenizer = core( function( token ) {\n\t\t\ttokens.push( token );\n\t\t} );\n\n\t\ttokenizer.addRule( fullStopRegex, \"full-stop\" );\n\t\ttokenizer.addRule( blockStartRegex, \"block-start\" );\n\t\ttokenizer.addRule( blockEndRegex, \"block-end\" );\n\t\ttokenizer.addRule( sentenceDelimiterRegex, \"sentence-delimiter\" );\n\t\ttokenizer.addRule( sentenceRegex, \"sentence\" );\n\n\t\treturn {\n\t\t\ttokenizer,\n\t\t\ttokens,\n\t\t};\n\t}\n\n\t/**\n\t * Tokenizes the given text using the given tokenizer.\n\t *\n\t * @param {Tokenizer} tokenizer The tokenizer to use.\n\t * @param {string}    text      The text to tokenize.\n\t *\n\t * @returns {void}\n\t */\n\ttokenize( tokenizer, text ) {\n\t\ttokenizer.onText( text );\n\n\t\ttry {\n\t\t\ttokenizer.end();\n\t\t} catch ( e ) {\n\t\t\tconsole.error( \"Tokenizer end error:\", e, e.tokenizer2 );\n\t\t}\n\t}\n\n\t/**\n\t * Determines the start and end indices of a set of sentences form a text.\n\t *\n\t * @param {Sentence[]} sentences A set of sentences for which to determine indices.\n\t *\n\t * @returns {void}\n\t */\n\tdetermineIndices( sentences ) {\n\t\tlet currentIndex = 0;\n\n\t\tfor ( const sentence of sentences ) {\n\t\t\tconst startIndex = currentIndex;\n\t\t\tsentence.setStartIndex( currentIndex );\n\t\t\tconst endIndex = startIndex + sentence.text.length - 1;\n\t\t\tsentence.setEndIndex( endIndex );\n\t\t\tcurrentIndex = endIndex + 1;\n\t\t}\n\t}\n\n\t/**\n\t * Trims the white space from the beginning of a sentence and adjusts the sentence start index accordingly.\n\t *\n\t * @param {Sentence} sentence The sentence for which to trim the white space at the start.\n\t *\n\t * @returns {void}\n\t */\n\ttrimWhiteSpaceAtStart( sentence ) {\n\t\tconst whiteSpaceLength = sentence.text.match( whiteSpaceStartRegex )[ 0 ].length;\n\t\tsentence.setText( sentence.getText().slice( whiteSpaceLength ) );\n\t\tsentence.setStartIndex( sentence.getStartIndex() + whiteSpaceLength  );\n\t}\n\n\t/**\n\t * Trims the white space from the end of a sentence and adjusts the sentence end index accordingly.\n\t *\n\t * @param {Sentence} sentence The sentence for which to trim the white space at the end.\n\t *\n\t * @returns {void}\n\t */\n\ttrimWhiteSpaceAtEnd( sentence ) {\n\t\tconst whiteSpaceLength = sentence.text.match( whiteSpaceEndRegex )[ 0 ].length;\n\t\tsentence.setText( sentence.getText().slice( 0, sentence.getText().length - whiteSpaceLength ) );\n\t\tsentence.setEndIndex( sentence.getEndIndex() - whiteSpaceLength );\n\t}\n\n\t/**\n\t * Trims white space from the beginning and end of sentences and adjusts the indices\n\t * of the sentence beginnings and ends accordingly.\n\t *\n\t * @param {Sentence[]} sentences The sentences for which to trim the whitespace.\n\t *\n\t * @returns {void}\n\t */\n\ttrimWhiteSpaces( sentences ) {\n\t\tfor ( const sentence of sentences ) {\n\t\t\tthis.trimWhiteSpaceAtStart( sentence );\n\t\t\tthis.trimWhiteSpaceAtEnd( sentence );\n\t\t}\n\t}\n\n\t/**\n\t * Returns an array of sentence objects for a given array of tokens; assumes that the text has already been split into blocks.\n\t *\n\t * @param {Object[]} tokenArray The tokens from the sentence tokenizer.\n\t *\n\t * @returns {Sentence[]} An array of sentence objects.\n\t */\n\tgetSentencesFromTokens( tokenArray ) {\n\t\tconst tokenSentences = [];\n\t\tlet currentSentence = new Sentence( \"\", 0, 0 ),\n\t\t\tnextSentenceStart;\n\n\t\ttokenArray.forEach( ( token, i ) => {\n\t\t\tlet hasNextSentence, nextCharacters;\n\t\t\tconst nextToken = tokenArray[ i + 1 ];\n\t\t\tconst secondToNextToken = tokenArray[ i + 2 ];\n\n\t\t\tswitch ( token.type ) {\n\t\t\t\tcase \"sentence\":\n\t\t\t\t\tcurrentSentence.appendText( token.src );\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase \"sentence-delimiter\":\n\t\t\t\t\tcurrentSentence.appendText( token.src );\n\t\t\t\t\tif ( ! isUndefined( nextToken ) && \"block-end\" !== nextToken.type && \"sentence-delimiter\" !== nextToken.type ) {\n\t\t\t\t\t\ttokenSentences.push( currentSentence );\n\t\t\t\t\t\tcurrentSentence = new Sentence( \"\" );\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase \"full-stop\":\n\t\t\t\t\tcurrentSentence.appendText( token.src );\n\n\t\t\t\t\tnextCharacters = this.getNextTwoCharacters( [ nextToken, secondToNextToken ] );\n\n\t\t\t\t\t// For a new sentence we need to check the next two characters.\n\t\t\t\t\thasNextSentence = nextCharacters.length >= 2;\n\t\t\t\t\tnextSentenceStart = hasNextSentence ? nextCharacters[ 1 ] : \"\";\n\t\t\t\t\t// If the next character is a number, never split. For example: IPv4-numbers.\n\t\t\t\t\tif ( hasNextSentence && this.isNumber( nextCharacters[ 0 ] ) ) {\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t\t// Only split on sentence delimiters when the next sentence looks like the start of a sentence.\n\t\t\t\t\tif ( ( hasNextSentence && this.isValidSentenceBeginning( nextSentenceStart ) ) || this.isSentenceStart( nextToken ) ) {\n\t\t\t\t\t\ttokenSentences.push( currentSentence );\n\t\t\t\t\t\tcurrentSentence = new Sentence( \"\" );\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase \"block-start\":\n\t\t\t\t\tcurrentSentence.appendText( token.src );\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase \"block-end\":\n\t\t\t\t\tcurrentSentence.appendText( token.src );\n\t\t\t\t\tnextCharacters = this.getNextTwoCharacters( [ nextToken, secondToNextToken ] );\n\n\t\t\t\t\t// For a new sentence we need to check the next two characters.\n\t\t\t\t\thasNextSentence = nextCharacters.length >= 2;\n\t\t\t\t\tnextSentenceStart = hasNextSentence ? nextCharacters[ 0 ] : \"\";\n\t\t\t\t\t// If the next character is a number, never split. For example: IPv4-numbers.\n\t\t\t\t\tif ( hasNextSentence && this.isNumber( nextCharacters[ 0 ] ) ) {\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\n\t\t\t\t\tif ( ( hasNextSentence && this.isValidSentenceBeginning( nextSentenceStart ) ) || this.isSentenceStart( nextToken ) ) {\n\t\t\t\t\t\ttokenSentences.push( currentSentence );\n\t\t\t\t\t\tcurrentSentence = new Sentence( \"\" );\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\t\t\t}\n\t\t} );\n\n\t\tif ( currentSentence.getText()  !== \"\"  ) {\n\t\t\ttokenSentences.push( currentSentence );\n\t\t}\n\n\t\tthis.determineIndices( tokenSentences );\n\n\t\tthis.trimWhiteSpaces( tokenSentences );\n\n\t\treturn tokenSentences;\n\t}\n}\n"],"mappings":";;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AAEA,IAAAC,KAAA,GAAAC,sBAAA,CAAAF,OAAA;AACA,IAAAG,OAAA,GAAAH,OAAA;AAEA,IAAAI,SAAA,GAAAF,sBAAA,CAAAF,OAAA;AAAkC,SAAAE,uBAAAG,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAElC;AACA,MAAMG,QAAQ,GAAG,GAAG;AACpB;AACA,MAAMC,kBAAkB,GAAG,WAAW;AAEtC,MAAMC,aAAa,GAAG,IAAIC,MAAM,CAAE,IAAI,GAAGH,QAAQ,GAAG,IAAK,CAAC;AAC1D,MAAMI,sBAAsB,GAAG,IAAID,MAAM,CAAE,IAAI,GAAGF,kBAAkB,GAAG,IAAK,CAAC;AAC7E,MAAMI,aAAa,GAAG,IAAIF,MAAM,CAAE,KAAK,GAAGH,QAAQ,GAAGC,kBAAkB,GAAG,iBAAkB,CAAC;AAE7F,MAAMK,eAAe,GAAG,eAAe;AACvC,MAAMC,aAAa,GAAG,gBAAgB;AAEtC,MAAMC,oBAAoB,GAAG,MAAM;AACnC,MAAMC,kBAAkB,GAAG,MAAM;;AAEjC;AACA;AACA;AACe,MAAMC,iBAAiB,CAAC;EACtC;AACD;AACA;AACA;AACA;AACA;EACCC,QAAQA,CAAEC,SAAS,EAAG;IACrB,OAAO,CAAE,IAAAC,aAAK,EAAEC,QAAQ,CAAEF,SAAS,EAAE,EAAG,CAAE,CAAC;EAC5C;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCG,WAAWA,CAAEH,SAAS,EAAG;IACxBA,SAAS,GAAG,IAAAI,iBAAe,EAAEJ,SAAU,CAAC;IAExC,OAAO,GAAG,KAAKA,SAAS,IACvB,IAAI,KAAKA,SAAS;EACpB;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;AACA;EACCK,aAAaA,CAAEL,SAAS,EAAG;IAC1B,OAAO,GAAG,KAAKA,SAAS,IACvB,GAAG,KAAKA,SAAS;EACnB;;EAEA;AACD;AACA;AACA;AACA;AACA;EACCM,yBAAyBA,CAAEC,IAAI,EAAG;IACjC,OAAOA,IAAI,CAACC,OAAO,CAAE,KAAK,EAAE,GAAI,CAAC;EAClC;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCC,eAAeA,CAAET,SAAS,EAAG;IAC5B,OAAOA,SAAS,KAAKA,SAAS,CAACU,iBAAiB,CAAC,CAAC;EACnD;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCC,oBAAoBA,CAAEC,UAAU,EAAG;IAClC,IAAIC,IAAI,GAAG,EAAE;IAEb,IAAK,CAAE,IAAAC,mBAAW,EAAEF,UAAU,CAAE,CAAC,CAAG,CAAC,EAAG;MACvCC,IAAI,IAAID,UAAU,CAAE,CAAC,CAAE,CAACG,GAAG;IAC5B;IAEA,IAAK,CAAE,IAAAD,mBAAW,EAAEF,UAAU,CAAE,CAAC,CAAG,CAAC,EAAG;MACvCC,IAAI,IAAID,UAAU,CAAE,CAAC,CAAE,CAACG,GAAG;IAC5B;IAEAF,IAAI,GAAG,IAAI,CAACP,yBAAyB,CAAEO,IAAK,CAAC;IAE7C,OAAOA,IAAI;EACZ;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCG,wBAAwBA,CAAEC,iBAAiB,EAAG;IAC7C,OACC,IAAI,CAACR,eAAe,CAAEQ,iBAAkB,CAAC,IACzC,IAAI,CAAClB,QAAQ,CAAEkB,iBAAkB,CAAC,IAClC,IAAI,CAACd,WAAW,CAAEc,iBAAkB,CAAC,IACrC,IAAI,CAACZ,aAAa,CAAEY,iBAAkB,CAAC;EAEzC;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCC,eAAeA,CAAEC,KAAK,EAAG;IACxB,OAAS,CAAE,IAAAL,mBAAW,EAAEK,KAAM,CAAC,KAC9B,YAAY,KAAKA,KAAK,CAACC,IAAI,IAC3B,UAAU,KAAKD,KAAK,CAACC,IAAI,IACzB,aAAa,KAAKD,KAAK,CAACC,IAAI,CAC5B;EACF;;EAEA;AACD;AACA;AACA;AACA;EACCC,eAAeA,CAAA,EAAG;IACjB,MAAMC,MAAM,GAAG,EAAE;IACjB,MAAMC,SAAS,GAAG,IAAAC,aAAI,EAAE,UAAUL,KAAK,EAAG;MACzCG,MAAM,CAACG,IAAI,CAAEN,KAAM,CAAC;IACrB,CAAE,CAAC;IAEHI,SAAS,CAACG,OAAO,CAAEpC,aAAa,EAAE,WAAY,CAAC;IAC/CiC,SAAS,CAACG,OAAO,CAAEhC,eAAe,EAAE,aAAc,CAAC;IACnD6B,SAAS,CAACG,OAAO,CAAE/B,aAAa,EAAE,WAAY,CAAC;IAC/C4B,SAAS,CAACG,OAAO,CAAElC,sBAAsB,EAAE,oBAAqB,CAAC;IACjE+B,SAAS,CAACG,OAAO,CAAEjC,aAAa,EAAE,UAAW,CAAC;IAE9C,OAAO;MACN8B,SAAS;MACTD;IACD,CAAC;EACF;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;AACA;EACCK,QAAQA,CAAEJ,SAAS,EAAEhB,IAAI,EAAG;IAC3BgB,SAAS,CAACK,MAAM,CAAErB,IAAK,CAAC;IAExB,IAAI;MACHgB,SAAS,CAACM,GAAG,CAAC,CAAC;IAChB,CAAC,CAAC,OAAQ5C,CAAC,EAAG;MACb6C,OAAO,CAACC,KAAK,CAAE,sBAAsB,EAAE9C,CAAC,EAAEA,CAAC,CAAC+C,UAAW,CAAC;IACzD;EACD;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCC,gBAAgBA,CAAEC,SAAS,EAAG;IAC7B,IAAIC,YAAY,GAAG,CAAC;IAEpB,KAAM,MAAMC,QAAQ,IAAIF,SAAS,EAAG;MACnC,MAAMG,UAAU,GAAGF,YAAY;MAC/BC,QAAQ,CAACE,aAAa,CAAEH,YAAa,CAAC;MACtC,MAAMI,QAAQ,GAAGF,UAAU,GAAGD,QAAQ,CAAC7B,IAAI,CAACiC,MAAM,GAAG,CAAC;MACtDJ,QAAQ,CAACK,WAAW,CAAEF,QAAS,CAAC;MAChCJ,YAAY,GAAGI,QAAQ,GAAG,CAAC;IAC5B;EACD;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCG,qBAAqBA,CAAEN,QAAQ,EAAG;IACjC,MAAMO,gBAAgB,GAAGP,QAAQ,CAAC7B,IAAI,CAACqC,KAAK,CAAEhD,oBAAqB,CAAC,CAAE,CAAC,CAAE,CAAC4C,MAAM;IAChFJ,QAAQ,CAACS,OAAO,CAAET,QAAQ,CAACU,OAAO,CAAC,CAAC,CAACC,KAAK,CAAEJ,gBAAiB,CAAE,CAAC;IAChEP,QAAQ,CAACE,aAAa,CAAEF,QAAQ,CAACY,aAAa,CAAC,CAAC,GAAGL,gBAAkB,CAAC;EACvE;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCM,mBAAmBA,CAAEb,QAAQ,EAAG;IAC/B,MAAMO,gBAAgB,GAAGP,QAAQ,CAAC7B,IAAI,CAACqC,KAAK,CAAE/C,kBAAmB,CAAC,CAAE,CAAC,CAAE,CAAC2C,MAAM;IAC9EJ,QAAQ,CAACS,OAAO,CAAET,QAAQ,CAACU,OAAO,CAAC,CAAC,CAACC,KAAK,CAAE,CAAC,EAAEX,QAAQ,CAACU,OAAO,CAAC,CAAC,CAACN,MAAM,GAAGG,gBAAiB,CAAE,CAAC;IAC/FP,QAAQ,CAACK,WAAW,CAAEL,QAAQ,CAACc,WAAW,CAAC,CAAC,GAAGP,gBAAiB,CAAC;EAClE;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;AACA;EACCQ,eAAeA,CAAEjB,SAAS,EAAG;IAC5B,KAAM,MAAME,QAAQ,IAAIF,SAAS,EAAG;MACnC,IAAI,CAACQ,qBAAqB,CAAEN,QAAS,CAAC;MACtC,IAAI,CAACa,mBAAmB,CAAEb,QAAS,CAAC;IACrC;EACD;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCgB,sBAAsBA,CAAEC,UAAU,EAAG;IACpC,MAAMC,cAAc,GAAG,EAAE;IACzB,IAAIC,eAAe,GAAG,IAAIC,iBAAQ,CAAE,EAAE,EAAE,CAAC,EAAE,CAAE,CAAC;MAC7CC,iBAAiB;IAElBJ,UAAU,CAACK,OAAO,CAAE,CAAEvC,KAAK,EAAEwC,CAAC,KAAM;MACnC,IAAIC,eAAe,EAAEC,cAAc;MACnC,MAAMC,SAAS,GAAGT,UAAU,CAAEM,CAAC,GAAG,CAAC,CAAE;MACrC,MAAMI,iBAAiB,GAAGV,UAAU,CAAEM,CAAC,GAAG,CAAC,CAAE;MAE7C,QAASxC,KAAK,CAACC,IAAI;QAClB,KAAK,UAAU;UACdmC,eAAe,CAACS,UAAU,CAAE7C,KAAK,CAACJ,GAAI,CAAC;UACvC;QAED,KAAK,oBAAoB;UACxBwC,eAAe,CAACS,UAAU,CAAE7C,KAAK,CAACJ,GAAI,CAAC;UACvC,IAAK,CAAE,IAAAD,mBAAW,EAAEgD,SAAU,CAAC,IAAI,WAAW,KAAKA,SAAS,CAAC1C,IAAI,IAAI,oBAAoB,KAAK0C,SAAS,CAAC1C,IAAI,EAAG;YAC9GkC,cAAc,CAAC7B,IAAI,CAAE8B,eAAgB,CAAC;YACtCA,eAAe,GAAG,IAAIC,iBAAQ,CAAE,EAAG,CAAC;UACrC;UACA;QAED,KAAK,WAAW;UACfD,eAAe,CAACS,UAAU,CAAE7C,KAAK,CAACJ,GAAI,CAAC;UAEvC8C,cAAc,GAAG,IAAI,CAAClD,oBAAoB,CAAE,CAAEmD,SAAS,EAAEC,iBAAiB,CAAG,CAAC;;UAE9E;UACAH,eAAe,GAAGC,cAAc,CAACrB,MAAM,IAAI,CAAC;UAC5CiB,iBAAiB,GAAGG,eAAe,GAAGC,cAAc,CAAE,CAAC,CAAE,GAAG,EAAE;UAC9D;UACA,IAAKD,eAAe,IAAI,IAAI,CAAC7D,QAAQ,CAAE8D,cAAc,CAAE,CAAC,CAAG,CAAC,EAAG;YAC9D;UACD;UACA;UACA,IAAOD,eAAe,IAAI,IAAI,CAAC5C,wBAAwB,CAAEyC,iBAAkB,CAAC,IAAM,IAAI,CAACvC,eAAe,CAAE4C,SAAU,CAAC,EAAG;YACrHR,cAAc,CAAC7B,IAAI,CAAE8B,eAAgB,CAAC;YACtCA,eAAe,GAAG,IAAIC,iBAAQ,CAAE,EAAG,CAAC;UACrC;UACA;QAED,KAAK,aAAa;UACjBD,eAAe,CAACS,UAAU,CAAE7C,KAAK,CAACJ,GAAI,CAAC;UACvC;QAED,KAAK,WAAW;UACfwC,eAAe,CAACS,UAAU,CAAE7C,KAAK,CAACJ,GAAI,CAAC;UACvC8C,cAAc,GAAG,IAAI,CAAClD,oBAAoB,CAAE,CAAEmD,SAAS,EAAEC,iBAAiB,CAAG,CAAC;;UAE9E;UACAH,eAAe,GAAGC,cAAc,CAACrB,MAAM,IAAI,CAAC;UAC5CiB,iBAAiB,GAAGG,eAAe,GAAGC,cAAc,CAAE,CAAC,CAAE,GAAG,EAAE;UAC9D;UACA,IAAKD,eAAe,IAAI,IAAI,CAAC7D,QAAQ,CAAE8D,cAAc,CAAE,CAAC,CAAG,CAAC,EAAG;YAC9D;UACD;UAEA,IAAOD,eAAe,IAAI,IAAI,CAAC5C,wBAAwB,CAAEyC,iBAAkB,CAAC,IAAM,IAAI,CAACvC,eAAe,CAAE4C,SAAU,CAAC,EAAG;YACrHR,cAAc,CAAC7B,IAAI,CAAE8B,eAAgB,CAAC;YACtCA,eAAe,GAAG,IAAIC,iBAAQ,CAAE,EAAG,CAAC;UACrC;UACA;MACF;IACD,CAAE,CAAC;IAEH,IAAKD,eAAe,CAACT,OAAO,CAAC,CAAC,KAAM,EAAE,EAAI;MACzCQ,cAAc,CAAC7B,IAAI,CAAE8B,eAAgB,CAAC;IACvC;IAEA,IAAI,CAACtB,gBAAgB,CAAEqB,cAAe,CAAC;IAEvC,IAAI,CAACH,eAAe,CAAEG,cAAe,CAAC;IAEtC,OAAOA,cAAc;EACtB;AACD;AAACW,OAAA,CAAA9E,OAAA,GAAAW,iBAAA","ignoreList":[]}

package/build/cjs/parsedPaper/build/linguisticParsing/parseText.js

@@ -0,0 +1,26 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.parseTextIntoSentences = void 0;
+var _SentenceTokenizer = _interopRequireDefault(require("./SentenceTokenizer"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Parses a text into sentences.
+ *
+ * @param {string} text The text to parse.
+ *
+ * @returns {string[]} An array of sentence objects.
+ */
+const parseTextIntoSentences = function (text) {
+  const sentenceTokenizer = new _SentenceTokenizer.default();
+  const {
+    tokenizer,
+    tokens
+  } = sentenceTokenizer.createTokenizer();
+  sentenceTokenizer.tokenize(tokenizer, text);
+  return sentenceTokenizer.getSentencesFromTokens(tokens);
+};
+exports.parseTextIntoSentences = parseTextIntoSentences;
+//# sourceMappingURL=parseText.js.map

package/build/cjs/parsedPaper/build/linguisticParsing/parseText.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parseText.js","names":["_SentenceTokenizer","_interopRequireDefault","require","e","__esModule","default","parseTextIntoSentences","text","sentenceTokenizer","SentenceTokenizer","tokenizer","tokens","createTokenizer","tokenize","getSentencesFromTokens","exports"],"sources":["../../../../../src/parsedPaper/build/linguisticParsing/parseText.js"],"sourcesContent":["import SentenceTokenizer from \"./SentenceTokenizer\";\n\n/**\n * Parses a text into sentences.\n *\n * @param {string} text The text to parse.\n *\n * @returns {string[]} An array of sentence objects.\n */\nconst parseTextIntoSentences = function( text ) {\n\tconst  sentenceTokenizer = new SentenceTokenizer();\n\tconst { tokenizer, tokens } = sentenceTokenizer.createTokenizer();\n\n\tsentenceTokenizer.tokenize( tokenizer, text );\n\treturn sentenceTokenizer.getSentencesFromTokens( tokens );\n};\n\nexport {\n\tparseTextIntoSentences,\n};\n"],"mappings":";;;;;;AAAA,IAAAA,kBAAA,GAAAC,sBAAA,CAAAC,OAAA;AAAoD,SAAAD,uBAAAE,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAEpD;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMG,sBAAsB,GAAG,SAAAA,CAAUC,IAAI,EAAG;EAC/C,MAAOC,iBAAiB,GAAG,IAAIC,0BAAiB,CAAC,CAAC;EAClD,MAAM;IAAEC,SAAS;IAAEC;EAAO,CAAC,GAAGH,iBAAiB,CAACI,eAAe,CAAC,CAAC;EAEjEJ,iBAAiB,CAACK,QAAQ,CAAEH,SAAS,EAAEH,IAAK,CAAC;EAC7C,OAAOC,iBAAiB,CAACM,sBAAsB,CAAEH,MAAO,CAAC;AAC1D,CAAC;AAACI,OAAA,CAAAT,sBAAA,GAAAA,sBAAA","ignoreList":[]}

package/build/cjs/parsedPaper/build/tree/TreeBuilder.js

@@ -0,0 +1,82 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _buildTree = _interopRequireDefault(require("./html/buildTree"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Builds a tree representation of a source text.
+ *
+ * Currently supported languages:
+ *  - HTML (`"html"`)
+ *
+ * @memberOf module:parsedPaper/builder
+ */
+class TreeBuilder {
+  /**
+   * Creates a new TreeBuilder.
+   */
+  constructor() {
+    /**
+     * A registry holding the parse function to be called
+     * for each supported formatting language.
+     *
+     * @type {Object<string,Function>}
+     * @private
+     */
+    this._buildFunctions = {
+      html: _buildTree.default
+    };
+    this.build = this.build.bind(this);
+    this.register = this.register.bind(this);
+  }
+
+  /**
+   * Parses the given source text to a tree representation.
+   *
+   * Chooses the right parser depending on the given formatting language in which the text has been written (defaults to HTML).
+   *
+   * @param {string} sourceText                The source text that needs to be parsed to a tree representation.
+   * @param {Object} [options]                 Parse options.
+   * @param {string} [options.language="html"] The formatting language in which the source text has been written, e.g. `"html"` for HTML.
+   *
+   * @returns {module:parsedPaper/structure.Node} The tree representation as parsed from the source text.
+   */
+  build(sourceText, options = {
+    language: "html"
+  }) {
+    return this._buildFunctions[options.language](sourceText);
+  }
+
+  /**
+   * Registers a build function for the given formatting language.
+   *
+   * Call this method if you want to add support for another formatting language.
+   *
+   * @example
+   *   // Create a new build function.
+   *   const myBuilder = sourceText => {
+   *       const node = new Paragraph();
+   *       node.textContainer.appendText( sourceText );
+   *       return node;
+   *   };
+   *
+   *   // Register the build function for the language "my-language".
+   *   treeBuilder.register( "my-language", myBuilder );
+   *
+   *   // Build the tree using the registered builder.
+   *   const tree = treeBuilder.build( "some input", { language: "my-language" } );
+   *
+   * @param {string}                                                            language      The language to register.
+   * @param {function( sourceText: string ): module:parsedPaper/structure.Node} buildFunction The build function for the given language.
+   *
+   * @returns {void}
+   */
+  register(language, buildFunction) {
+    this._buildFunctions[language] = buildFunction;
+  }
+}
+var _default = exports.default = TreeBuilder;
+//# sourceMappingURL=TreeBuilder.js.map

package/build/cjs/parsedPaper/build/tree/TreeBuilder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TreeBuilder.js","names":["_buildTree","_interopRequireDefault","require","e","__esModule","default","TreeBuilder","constructor","_buildFunctions","html","buildTree","build","bind","register","sourceText","options","language","buildFunction","_default","exports"],"sources":["../../../../../src/parsedPaper/build/tree/TreeBuilder.js"],"sourcesContent":["import buildTree from \"./html/buildTree\";\n\n/**\n * Builds a tree representation of a source text.\n *\n * Currently supported languages:\n *  - HTML (`\"html\"`)\n *\n * @memberOf module:parsedPaper/builder\n */\nclass TreeBuilder {\n\t/**\n\t * Creates a new TreeBuilder.\n\t */\n\tconstructor() {\n\t\t/**\n\t\t * A registry holding the parse function to be called\n\t\t * for each supported formatting language.\n\t\t *\n\t\t * @type {Object<string,Function>}\n\t\t * @private\n\t\t */\n\t\tthis._buildFunctions = {\n\t\t\thtml: buildTree,\n\t\t};\n\n\t\tthis.build = this.build.bind( this );\n\t\tthis.register = this.register.bind( this );\n\t}\n\n\t/**\n\t * Parses the given source text to a tree representation.\n\t *\n\t * Chooses the right parser depending on the given formatting language in which the text has been written (defaults to HTML).\n\t *\n\t * @param {string} sourceText                The source text that needs to be parsed to a tree representation.\n\t * @param {Object} [options]                 Parse options.\n\t * @param {string} [options.language=\"html\"] The formatting language in which the source text has been written, e.g. `\"html\"` for HTML.\n\t *\n\t * @returns {module:parsedPaper/structure.Node} The tree representation as parsed from the source text.\n\t */\n\tbuild( sourceText, options = { language: \"html\" } ) {\n\t\treturn this._buildFunctions[ options.language ]( sourceText );\n\t}\n\n\t/**\n\t * Registers a build function for the given formatting language.\n\t *\n\t * Call this method if you want to add support for another formatting language.\n\t *\n\t * @example\n\t *   // Create a new build function.\n\t *   const myBuilder = sourceText => {\n\t *       const node = new Paragraph();\n\t *       node.textContainer.appendText( sourceText );\n\t *       return node;\n\t *   };\n\t *\n\t *   // Register the build function for the language \"my-language\".\n\t *   treeBuilder.register( \"my-language\", myBuilder );\n\t *\n\t *   // Build the tree using the registered builder.\n\t *   const tree = treeBuilder.build( \"some input\", { language: \"my-language\" } );\n\t *\n\t * @param {string}                                                            language      The language to register.\n\t * @param {function( sourceText: string ): module:parsedPaper/structure.Node} buildFunction The build function for the given language.\n\t *\n\t * @returns {void}\n\t */\n\tregister( language, buildFunction ) {\n\t\tthis._buildFunctions[ language ] = buildFunction;\n\t}\n}\n\nexport default TreeBuilder;\n"],"mappings":";;;;;;AAAA,IAAAA,UAAA,GAAAC,sBAAA,CAAAC,OAAA;AAAyC,SAAAD,uBAAAE,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAEzC;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMG,WAAW,CAAC;EACjB;AACD;AACA;EACCC,WAAWA,CAAA,EAAG;IACb;AACF;AACA;AACA;AACA;AACA;AACA;IACE,IAAI,CAACC,eAAe,GAAG;MACtBC,IAAI,EAAEC;IACP,CAAC;IAED,IAAI,CAACC,KAAK,GAAG,IAAI,CAACA,KAAK,CAACC,IAAI,CAAE,IAAK,CAAC;IACpC,IAAI,CAACC,QAAQ,GAAG,IAAI,CAACA,QAAQ,CAACD,IAAI,CAAE,IAAK,CAAC;EAC3C;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACCD,KAAKA,CAAEG,UAAU,EAAEC,OAAO,GAAG;IAAEC,QAAQ,EAAE;EAAO,CAAC,EAAG;IACnD,OAAO,IAAI,CAACR,eAAe,CAAEO,OAAO,CAACC,QAAQ,CAAE,CAAEF,UAAW,CAAC;EAC9D;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACCD,QAAQA,CAAEG,QAAQ,EAAEC,aAAa,EAAG;IACnC,IAAI,CAACT,eAAe,CAAEQ,QAAQ,CAAE,GAAGC,aAAa;EACjD;AACD;AAAC,IAAAC,QAAA,GAAAC,OAAA,CAAAd,OAAA,GAEcC,WAAW","ignoreList":[]}

package/build/cjs/parsedPaper/build/tree/cleanup/calculateTextIndices.js

@@ -0,0 +1,183 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _lodash = require("lodash");
+var _htmlConstants = require("../html/htmlConstants");
+/**
+ * Gathers all elements that can be closed given the position of the current element in the source code.
+ *
+ * Elements that can be closed are all elements that are opened before this element, but in which this element is
+ * not nested.
+ * E.g.
+ * ```html
+ * <strong>Hello</strong><em>World<b>!!!</b></em>`
+ * ```
+ * with `<b>!!!</b>` as the current element,
+ * means that the `<strong>` needs to be closed, but `<em>` **not**.
+ *
+ * @param {module:parsedPaper/structure.FormattingElement} currentElement The current element.
+ * @param {module:parsedPaper/structure.FormattingElement[]} openElements The elements that are currently open.
+ *
+ * @returns {module:parsedPaper/structure.FormattingElement[]} The elements that can be closed.
+ */
+const elementsThatCanBeClosed = function (currentElement, openElements) {
+  return openElements.filter(el => {
+    const endTag = el.sourceCodeLocation.endTag;
+    return endTag.endOffset <= currentElement.sourceCodeLocation.startOffset;
+  });
+};
+
+/**
+ * Closes the elements that can be closed given the position of the current element within the source code.
+ *
+ * This does two things:
+ *  1. The closed element's text end index is calculated based on the current offset.
+ *  2. The closed element's end tag lengths are counted towards the current offset, to make sure that the computed position
+ * of the formatting elements are still correct.
+ *
+ * @param {module:parsedPaper/structure.FormattingElement[]} elementsToClose The list of open elements that need to be closed
+ * @param {number} currentOffset                                      The current offset when parsing the formatting elements
+ *
+ * @returns {number} The updated current offset
+ *
+ * @private
+ */
+const closeElements = function (elementsToClose, currentOffset) {
+  // Sort, so we close all elements in the right order.
+  elementsToClose.sort((a, b) => a.sourceCodeLocation.endTag.endOffset - b.sourceCodeLocation.endTag.endOffset);
+  elementsToClose.forEach(elementToClose => {
+    const endTag = elementToClose.sourceCodeLocation.endTag;
+    // Set the end position as seen in the text.
+    elementToClose.textEndIndex = endTag.startOffset - currentOffset;
+    /*
+      Add the end tag length of the to be closed element to the total offset.
+     */
+    const endTagLength = endTag.endOffset - endTag.startOffset;
+    currentOffset += endTagLength;
+  });
+  return currentOffset;
+};
+
+/**
+ * Adds the content length of the given element (the part between the tags) to the current offset
+ * and adds the content to the element as a parameter.
+ *
+ * @param {module:parsedPaper/structure.FormattingElement} element The element of which to add the content length.
+ * @param {number} currentOffset                            The current offset to which to add the length to.
+ *
+ * @returns {number} The updated current offset
+ */
+const handleIgnoredContent = function (element, currentOffset) {
+  // Has 0 length in text, so end = start.
+  element.textEndIndex = element.textStartIndex;
+
+  // Update current offset.
+  const end = element.sourceCodeLocation.endTag ? element.sourceCodeLocation.endTag.startOffset : element.sourceCodeLocation.endOffset;
+  const start = element.sourceCodeLocation.startTag ? element.sourceCodeLocation.startTag.endOffset : element.sourceCodeLocation.startOffset;
+  currentOffset += end - start;
+  return currentOffset;
+};
+
+/**
+ * Sets the start and end text positions of a comment.
+ *
+ * @param {module:parsedPaper/structure.FormattingElement}	element			The formatting element to assign start and end text positions to.
+ * @param {int}												currentOffset	A sum of all characters in the source code that don't get rendered
+ * 																			(e.g., tags, comments).
+ *
+ * @returns {number} The length of the comment.
+ *
+ * @private
+ */
+const computeCommentStartEndTextIndices = function (element, currentOffset) {
+  element.textStartIndex = element.sourceCodeLocation.startOffset - currentOffset;
+  element.textEndIndex = element.textStartIndex;
+  return element.sourceCodeLocation.endOffset - element.sourceCodeLocation.startOffset;
+};
+
+/**
+ * Sets the start and end text positions of one formatting element.
+ *
+ * @param {module:parsedPaper/structure.FormattingElement}	element			The formatting element to assign start and end text positions to.
+ * @param {int}												currentOffset	A sum of all characters in the source code that don't get rendered
+ * 																			(e.g., tags, comments).
+ *
+ * @returns {int} The updated currentOffset.
+ *
+ * @private
+ */
+const computeElementStartTextIndex = function (element, currentOffset) {
+  const startTag = element.sourceCodeLocation.startTag;
+
+  // For example: "<strong>".length
+  const startTagLength = startTag.endOffset - startTag.startOffset;
+  currentOffset += startTagLength;
+
+  // Set start position of element in heading's / paragraph's text.
+  element.textStartIndex = startTag.endOffset - currentOffset;
+
+  /*
+    Elements that have no end tags (e.g., void element like <img/> or self-closing elements) can be closed immediately.
+    The text length of those elements will be automatically 0.
+   */
+  if (!element.sourceCodeLocation.endTag) {
+    element.textEndIndex = element.textStartIndex;
+  }
+  return currentOffset;
+};
+
+/**
+ * Sets the start and end position of the text in formatting elements of the given node.
+ *
+ * @param {module:parsedPaper/structure.LeafNode} node The node containing a TextContainer
+ *
+ * @returns {void}
+ *
+ * @private
+ */
+const calculateTextIndices = function (node) {
+  if (!node.textContainer.formatting || node.textContainer.formatting.length === 0) {
+    return;
+  }
+  const openElements = [];
+
+  /*
+    Keeps track of the current total size of the start and end tags (and the ignored content)
+    These should not be counted towards the start and end position of the elements in the text.
+   */
+  let currentOffset = node.sourceCodeLocation.startTag ? node.sourceCodeLocation.startTag.endOffset : node.sourceCodeLocation.startOffset;
+  node.textContainer.formatting.forEach(element => {
+    // Close elements that can be closed and remove them from the list of open elements.
+    const elementsToClose = elementsThatCanBeClosed(element, openElements);
+    currentOffset = closeElements(elementsToClose, currentOffset);
+    (0, _lodash.pullAll)(openElements, elementsToClose);
+
+    // Comments are self-closing formatting elements that are completely ignored in rendering.
+    if (element.type === "#comment") {
+      currentOffset += computeCommentStartEndTextIndices(element, currentOffset);
+      return;
+    }
+    currentOffset = computeElementStartTextIndex(element, currentOffset);
+
+    // If there is an endTag, the element should be closed in one of the next iterations of the loop.
+    if (element.sourceCodeLocation.endTag) {
+      openElements.push(element);
+    }
+
+    /*
+      If this element is an ignored element its contents are not in the text,
+      so its content should be added to the respective formatting element instead,
+      and the current offset should be updated.
+     */
+    if (_htmlConstants.ignoredHtmlElements.includes(element.type)) {
+      currentOffset = handleIgnoredContent(element, currentOffset);
+    }
+  });
+  // Close all remaining elements.
+  closeElements(openElements, currentOffset);
+};
+var _default = exports.default = calculateTextIndices;
+//# sourceMappingURL=calculateTextIndices.js.map