axyseo 2.0.0-alpha.0.0.37 → 2.0.0-alpha.0.0.39

package/build/parsedPaper/build/PaperParser.js

@@ -0,0 +1,97 @@
+import ParsedPaper from "../ParsedPaper";
+import { forEach } from "lodash";
+
+/**
+ * A class responsible for pre-processing the Paper, returning a ParsedPaper.
+ *
+ * @module parsedPaper
+ *
+ * @see module:parsedPaper/builder
+ * @see module:parsedPaper/structure
+ */
+class PaperParser {
+  /**
+   * Constructs a PaperParser class.
+   *
+   * @param {Function} treeBuilder A function that receives text and returns a tree.
+   *
+   * @constructor
+   */
+  constructor(treeBuilder) {
+    this._parsedPaper = new ParsedPaper();
+    this._treeBuilder = treeBuilder;
+    this._metaDataModifiers = {};
+  }
+
+  /**
+   * Processes a Paper resulting in a ParsedPaper.
+   *
+   * @param {Paper} paper The Paper to parse.
+   *
+   * @returns {ParsedPaper} A parsedPaper instance.
+   */
+  parse(paper) {
+    // Build tree and set it to the ParsedPaper.
+    this._parsedPaper.setTree(this._treeBuilder(paper.getText()));
+
+    // Build metaData and set it to the ParsedPaper.
+    this._parsedPaper.setMetaData(this.constructMetaData(paper));
+    return this._parsedPaper;
+  }
+
+  /**
+   * Constructs the metaData from the Paper.
+   *
+   * @param {Paper} paper The paper to construct the metaData from.
+   *
+   * @returns {Object} The metaData.
+   */
+  constructMetaData(paper) {
+    let metaData = {};
+
+    // Map things to metaData.
+
+    // Run additional modifiers;
+    metaData = Object.assign({}, this.runMetaDataModifiers(metaData, paper));
+    return metaData;
+  }
+
+  /**
+   * Sets a metaData modifying function behind a function name on the internal metaDataModifiers object.
+   *
+   * @param {string}   modifierName       The name of the to be registered function.
+   * @param {Function} modifierFunction   The function that modifies the metaData. Should accept a metaData object
+   * 									    and optionally the paper.
+   * @returns {void}
+   */
+  registerMetaDataModifier(modifierName, modifierFunction) {
+    /*
+     * The metaDataModifier should accept the metaData and return the altered metaData.
+     */
+    this._metaDataModifiers[modifierName] = modifierFunction;
+  }
+
+  /**
+   * Runs all registered metaDataModifiers.
+   * If one of the functions errors, it is skipped and its metaData changes are discarded.
+   *
+   * @param {Object} metaData The initial state of the metaData, that should be modified.
+   * @param {Paper}  paper    The paper.
+   *
+   * @returns {Object} A modified metaData object.
+   */
+  runMetaDataModifiers(metaData, paper) {
+    let modifiedMetaData = metaData;
+    forEach(this._metaDataModifiers, (modifierFunction, modifierName) => {
+      try {
+        const previousMetaData = Object.assign({}, modifiedMetaData);
+        modifiedMetaData = modifierFunction(previousMetaData, paper);
+      } catch (modifierError) {
+        console.warn(`An error with message "${modifierError.message}" occurred in metaData modifier ` + `function called ${modifierName}. Skipping that function...`);
+      }
+    });
+    return modifiedMetaData;
+  }
+}
+export default PaperParser;
+//# sourceMappingURL=PaperParser.js.map

package/build/parsedPaper/build/PaperParser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PaperParser.js","names":["ParsedPaper","forEach","PaperParser","constructor","treeBuilder","_parsedPaper","_treeBuilder","_metaDataModifiers","parse","paper","setTree","getText","setMetaData","constructMetaData","metaData","Object","assign","runMetaDataModifiers","registerMetaDataModifier","modifierName","modifierFunction","modifiedMetaData","previousMetaData","modifierError","console","warn","message"],"sources":["../../../src/parsedPaper/build/PaperParser.js"],"sourcesContent":["import ParsedPaper from \"../ParsedPaper\";\nimport { forEach } from \"lodash\";\n\n/**\n * A class responsible for pre-processing the Paper, returning a ParsedPaper.\n *\n * @module parsedPaper\n *\n * @see module:parsedPaper/builder\n * @see module:parsedPaper/structure\n */\nclass PaperParser {\n\t/**\n\t * Constructs a PaperParser class.\n\t *\n\t * @param {Function} treeBuilder A function that receives text and returns a tree.\n\t *\n\t * @constructor\n\t */\n\tconstructor( treeBuilder ) {\n\t\tthis._parsedPaper = new ParsedPaper();\n\t\tthis._treeBuilder = treeBuilder;\n\n\t\tthis._metaDataModifiers = {};\n\t}\n\n\t/**\n\t * Processes a Paper resulting in a ParsedPaper.\n\t *\n\t * @param {Paper} paper The Paper to parse.\n\t *\n\t * @returns {ParsedPaper} A parsedPaper instance.\n\t */\n\tparse( paper ) {\n\t\t// Build tree and set it to the ParsedPaper.\n\t\tthis._parsedPaper.setTree(\n\t\t\tthis._treeBuilder( paper.getText() )\n\t\t);\n\n\t\t// Build metaData and set it to the ParsedPaper.\n\t\tthis._parsedPaper.setMetaData(\n\t\t\tthis.constructMetaData( paper )\n\t\t);\n\n\t\treturn this._parsedPaper;\n\t}\n\n\t/**\n\t * Constructs the metaData from the Paper.\n\t *\n\t * @param {Paper} paper The paper to construct the metaData from.\n\t *\n\t * @returns {Object} The metaData.\n\t */\n\tconstructMetaData( paper ) {\n\t\tlet metaData = {};\n\n\t\t// Map things to metaData.\n\n\t\t// Run additional modifiers;\n\t\tmetaData = Object.assign( {}, this.runMetaDataModifiers( metaData, paper ) );\n\n\t\treturn metaData;\n\t}\n\n\t/**\n\t * Sets a metaData modifying function behind a function name on the internal metaDataModifiers object.\n\t *\n\t * @param {string}   modifierName       The name of the to be registered function.\n\t * @param {Function} modifierFunction   The function that modifies the metaData. Should accept a metaData object\n\t * \t\t\t\t\t\t\t\t\t    and optionally the paper.\n\t * @returns {void}\n\t */\n\tregisterMetaDataModifier( modifierName, modifierFunction ) {\n\t\t/*\n\t\t * The metaDataModifier should accept the metaData and return the altered metaData.\n\t\t */\n\t\tthis._metaDataModifiers[ modifierName ] = modifierFunction;\n\t}\n\n\t/**\n\t * Runs all registered metaDataModifiers.\n\t * If one of the functions errors, it is skipped and its metaData changes are discarded.\n\t *\n\t * @param {Object} metaData The initial state of the metaData, that should be modified.\n\t * @param {Paper}  paper    The paper.\n\t *\n\t * @returns {Object} A modified metaData object.\n\t */\n\trunMetaDataModifiers( metaData, paper ) {\n\t\tlet modifiedMetaData = metaData;\n\t\tforEach( this._metaDataModifiers, ( modifierFunction, modifierName ) => {\n\t\t\ttry {\n\t\t\t\tconst previousMetaData = Object.assign( {}, modifiedMetaData );\n\t\t\t\tmodifiedMetaData = modifierFunction( previousMetaData, paper );\n\t\t\t} catch ( modifierError ) {\n\t\t\t\tconsole.warn( `An error with message \"${ modifierError.message}\" occurred in metaData modifier ` +\n\t\t\t\t`function called ${ modifierName }. Skipping that function...` );\n\t\t\t}\n\t\t} );\n\t\treturn modifiedMetaData;\n\t}\n}\n\nexport default PaperParser;\n"],"mappings":"AAAA,OAAOA,WAAW;AAClB,SAASC,OAAO,QAAQ,QAAQ;;AAEhC;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMC,WAAW,CAAC;EACjB;AACD;AACA;AACA;AACA;AACA;AACA;EACCC,WAAWA,CAAEC,WAAW,EAAG;IAC1B,IAAI,CAACC,YAAY,GAAG,IAAIL,WAAW,CAAC,CAAC;IACrC,IAAI,CAACM,YAAY,GAAGF,WAAW;IAE/B,IAAI,CAACG,kBAAkB,GAAG,CAAC,CAAC;EAC7B;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCC,KAAKA,CAAEC,KAAK,EAAG;IACd;IACA,IAAI,CAACJ,YAAY,CAACK,OAAO,CACxB,IAAI,CAACJ,YAAY,CAAEG,KAAK,CAACE,OAAO,CAAC,CAAE,CACpC,CAAC;;IAED;IACA,IAAI,CAACN,YAAY,CAACO,WAAW,CAC5B,IAAI,CAACC,iBAAiB,CAAEJ,KAAM,CAC/B,CAAC;IAED,OAAO,IAAI,CAACJ,YAAY;EACzB;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCQ,iBAAiBA,CAAEJ,KAAK,EAAG;IAC1B,IAAIK,QAAQ,GAAG,CAAC,CAAC;;IAEjB;;IAEA;IACAA,QAAQ,GAAGC,MAAM,CAACC,MAAM,CAAE,CAAC,CAAC,EAAE,IAAI,CAACC,oBAAoB,CAAEH,QAAQ,EAAEL,KAAM,CAAE,CAAC;IAE5E,OAAOK,QAAQ;EAChB;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;AACA;EACCI,wBAAwBA,CAAEC,YAAY,EAAEC,gBAAgB,EAAG;IAC1D;AACF;AACA;IACE,IAAI,CAACb,kBAAkB,CAAEY,YAAY,CAAE,GAAGC,gBAAgB;EAC3D;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACCH,oBAAoBA,CAAEH,QAAQ,EAAEL,KAAK,EAAG;IACvC,IAAIY,gBAAgB,GAAGP,QAAQ;IAC/Bb,OAAO,CAAE,IAAI,CAACM,kBAAkB,EAAE,CAAEa,gBAAgB,EAAED,YAAY,KAAM;MACvE,IAAI;QACH,MAAMG,gBAAgB,GAAGP,MAAM,CAACC,MAAM,CAAE,CAAC,CAAC,EAAEK,gBAAiB,CAAC;QAC9DA,gBAAgB,GAAGD,gBAAgB,CAAEE,gBAAgB,EAAEb,KAAM,CAAC;MAC/D,CAAC,CAAC,OAAQc,aAAa,EAAG;QACzBC,OAAO,CAACC,IAAI,CAAE,0BAA2BF,aAAa,CAACG,OAAO,kCAAkC,GAChG,mBAAoBP,YAAY,6BAA+B,CAAC;MACjE;IACD,CAAE,CAAC;IACH,OAAOE,gBAAgB;EACxB;AACD;AAEA,eAAenB,WAAW","ignoreList":[]}

package/build/parsedPaper/build/linguisticParsing/Sentence.js

@@ -0,0 +1,89 @@
+/**
+ * Represents a sentence in a text.
+ */
+class Sentence {
+  /**
+   * Represents a sentence within a text.
+   *
+   * @param {string} text     The text of this sentence.
+   * @param {number} startIndex   The start index of this sentence.
+   * @param {number} endIndex     The end index of this sentence.
+   */
+  constructor(text, startIndex = 0, endIndex = 0) {
+    this.text = text;
+    this.words = [];
+    this.startIndex = startIndex;
+    this.endIndex = endIndex;
+  }
+
+  /**
+   * Sets a text for this sentence.
+   *
+   * @param {string} text The text to be set for the sentence.
+   *
+   * @returns {void}
+   */
+  setText(text) {
+    this.text = text;
+  }
+
+  /**
+   * Appends text to this sentence.
+   *
+   * @param {string} text The text to be added to the sentence.
+   *
+   * @returns {void}
+   */
+  appendText(text) {
+    this.text += text;
+  }
+
+  /**
+   * Returns the text of this sentence.
+   *
+   * @returns {string} The text of this senence.
+   */
+  getText() {
+    return this.text;
+  }
+
+  /**
+   * Sets the start index of this sentence.
+   *
+   * @param {number} startIndex The start index of this sentence.
+   * @returns {void}
+   */
+  setStartIndex(startIndex) {
+    this.startIndex = startIndex;
+  }
+
+  /**
+   * Sets the end index of this sentence.
+   *
+   * @param {number} endIndex The end index of this sentence.
+   * @returns {void}
+   */
+  setEndIndex(endIndex) {
+    this.endIndex = endIndex;
+  }
+
+  /**
+   * Returns the start index of this sentence.
+   *
+   * @returns {number} The start index of this sentence.
+   */
+  getStartIndex() {
+    return this.startIndex;
+  }
+
+  /**
+   * Returns the end index of this sentence.
+   *
+   * @returns {number} The end index of this sentence.
+   */
+  getEndIndex() {
+    return this.endIndex;
+  }
+}
+export default Sentence;
+//# sourceMappingURL=Sentence.js.map

package/build/parsedPaper/build/linguisticParsing/Sentence.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Sentence.js","names":["Sentence","constructor","text","startIndex","endIndex","words","setText","appendText","getText","setStartIndex","setEndIndex","getStartIndex","getEndIndex"],"sources":["../../../../src/parsedPaper/build/linguisticParsing/Sentence.js"],"sourcesContent":["/**\n * Represents a sentence in a text.\n */\nclass Sentence {\n\t/**\n\t * Represents a sentence within a text.\n\t *\n\t * @param {string} text     The text of this sentence.\n\t * @param {number} startIndex   The start index of this sentence.\n\t * @param {number} endIndex     The end index of this sentence.\n\t */\n\tconstructor( text, startIndex = 0, endIndex = 0 ) {\n\t\tthis.text = text;\n\t\tthis.words = [];\n\t\tthis.startIndex = startIndex;\n\t\tthis.endIndex = endIndex;\n\t}\n\n\t/**\n\t * Sets a text for this sentence.\n\t *\n\t * @param {string} text The text to be set for the sentence.\n\t *\n\t * @returns {void}\n\t */\n\tsetText( text ) {\n\t\tthis.text = text;\n\t}\n\n\t/**\n\t * Appends text to this sentence.\n\t *\n\t * @param {string} text The text to be added to the sentence.\n\t *\n\t * @returns {void}\n\t */\n\tappendText( text ) {\n\t\tthis.text += text;\n\t}\n\n\t/**\n\t * Returns the text of this sentence.\n\t *\n\t * @returns {string} The text of this senence.\n\t */\n\tgetText() {\n\t\treturn this.text;\n\t}\n\n\t/**\n\t * Sets the start index of this sentence.\n\t *\n\t * @param {number} startIndex The start index of this sentence.\n\t * @returns {void}\n\t */\n\tsetStartIndex( startIndex ) {\n\t\tthis.startIndex = startIndex;\n\t}\n\n\t/**\n\t * Sets the end index of this sentence.\n\t *\n\t * @param {number} endIndex The end index of this sentence.\n\t * @returns {void}\n\t */\n\tsetEndIndex( endIndex ) {\n\t\tthis.endIndex = endIndex;\n\t}\n\n\t/**\n\t * Returns the start index of this sentence.\n\t *\n\t * @returns {number} The start index of this sentence.\n\t */\n\tgetStartIndex() {\n\t\treturn this.startIndex;\n\t}\n\n\t/**\n\t * Returns the end index of this sentence.\n\t *\n\t * @returns {number} The end index of this sentence.\n\t */\n\tgetEndIndex() {\n\t\treturn this.endIndex;\n\t}\n}\n\nexport default Sentence;\n"],"mappings":"AAAA;AACA;AACA;AACA,MAAMA,QAAQ,CAAC;EACd;AACD;AACA;AACA;AACA;AACA;AACA;EACCC,WAAWA,CAAEC,IAAI,EAAEC,UAAU,GAAG,CAAC,EAAEC,QAAQ,GAAG,CAAC,EAAG;IACjD,IAAI,CAACF,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACG,KAAK,GAAG,EAAE;IACf,IAAI,CAACF,UAAU,GAAGA,UAAU;IAC5B,IAAI,CAACC,QAAQ,GAAGA,QAAQ;EACzB;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCE,OAAOA,CAAEJ,IAAI,EAAG;IACf,IAAI,CAACA,IAAI,GAAGA,IAAI;EACjB;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCK,UAAUA,CAAEL,IAAI,EAAG;IAClB,IAAI,CAACA,IAAI,IAAIA,IAAI;EAClB;;EAEA;AACD;AACA;AACA;AACA;EACCM,OAAOA,CAAA,EAAG;IACT,OAAO,IAAI,CAACN,IAAI;EACjB;;EAEA;AACD;AACA;AACA;AACA;AACA;EACCO,aAAaA,CAAEN,UAAU,EAAG;IAC3B,IAAI,CAACA,UAAU,GAAGA,UAAU;EAC7B;;EAEA;AACD;AACA;AACA;AACA;AACA;EACCO,WAAWA,CAAEN,QAAQ,EAAG;IACvB,IAAI,CAACA,QAAQ,GAAGA,QAAQ;EACzB;;EAEA;AACD;AACA;AACA;AACA;EACCO,aAAaA,CAAA,EAAG;IACf,OAAO,IAAI,CAACR,UAAU;EACvB;;EAEA;AACD;AACA;AACA;AACA;EACCS,WAAWA,CAAA,EAAG;IACb,OAAO,IAAI,CAACR,QAAQ;EACrB;AACD;AAEA,eAAeJ,QAAQ","ignoreList":[]}

package/build/parsedPaper/build/linguisticParsing/SentenceTokenizer.js

@@ -0,0 +1,287 @@
+import { isNaN, isUndefined } from "lodash";
+import core from "tokenizer2/core";
+import { normalize as normalizeQuotes } from "../../../languageProcessing/helpers/sanitize/quotes";
+import Sentence from "./Sentence";
+
+// All characters that indicate a sentence delimiter.
+const fullStop = ".";
+// The \u2026 character is an ellipsis
+const sentenceDelimiters = "?!;\u2026";
+const fullStopRegex = new RegExp("^[" + fullStop + "]$");
+const sentenceDelimiterRegex = new RegExp("^[" + sentenceDelimiters + "]$");
+const sentenceRegex = new RegExp("^[^" + fullStop + sentenceDelimiters + "\\(\\)\\[\\]]+$");
+const blockStartRegex = /^\s*[[({]\s*$/;
+const blockEndRegex = /^\s*[\])}]\s*$/;
+const whiteSpaceStartRegex = /^\s*/;
+const whiteSpaceEndRegex = /\s*$/;
+
+/**
+ * Class for tokenizing a (html) text into sentences.
+ */
+export default class SentenceTokenizer {
+  /**
+   * Returns whether or not a certain character is a number.
+   *
+   * @param {string} character The character to check.
+   * @returns {boolean} Whether or not the character is a capital letter.
+   */
+  isNumber(character) {
+    return !isNaN(parseInt(character, 10));
+  }
+
+  /**
+   * Returns whether or not a given character is quotation mark.
+   *
+   * @param {string} character The character to check.
+   *
+   * @returns {boolean} Whether or not the given character is a quotation mark.
+   */
+  isQuotation(character) {
+    character = normalizeQuotes(character);
+    return "'" === character || "\"" === character;
+  }
+
+  /**
+   * Returns whether or not a given character is a punctuation mark that can be at the beginning
+   * of a sentence, like ¿ and ¡ used in Spanish.
+   *
+   * @param {string} character The character to check.
+   *
+   * @returns {boolean} Whether or not the given character is a punctuation mark.
+   */
+  isPunctuation(character) {
+    return "¿" === character || "¡" === character;
+  }
+
+  /**
+   * Removes duplicate whitespace from a given text.
+   *
+   * @param {string} text The text with duplicate whitespace.
+   * @returns {string} The text without duplicate whitespace.
+   */
+  removeDuplicateWhitespace(text) {
+    return text.replace(/\s+/, " ");
+  }
+
+  /**
+   * Returns whether or not a certain character is a capital letter.
+   *
+   * @param {string} character The character to check.
+   *
+   * @returns {boolean} Whether or not the character is a capital letter.
+   */
+  isCapitalLetter(character) {
+    return character !== character.toLocaleLowerCase();
+  }
+
+  /**
+   * Retrieves the next two characters from an array with the two next tokens.
+   *
+   * @param {Array} nextTokens The two next tokens. Might be undefined.
+   *
+   * @returns {string} The next two characters.
+   */
+  getNextTwoCharacters(nextTokens) {
+    let next = "";
+    if (!isUndefined(nextTokens[0])) {
+      next += nextTokens[0].src;
+    }
+    if (!isUndefined(nextTokens[1])) {
+      next += nextTokens[1].src;
+    }
+    next = this.removeDuplicateWhitespace(next);
+    return next;
+  }
+
+  /**
+   * Checks if the sentenceBeginning beginning is a valid beginning.
+   *
+   * @param {string} sentenceBeginning The beginning of the sentence to validate.
+   *
+   * @returns {boolean} Returns true if it is a valid beginning, false if it is not.
+   */
+  isValidSentenceBeginning(sentenceBeginning) {
+    return this.isCapitalLetter(sentenceBeginning) || this.isNumber(sentenceBeginning) || this.isQuotation(sentenceBeginning) || this.isPunctuation(sentenceBeginning);
+  }
+
+  /**
+   * Checks if the token is a valid sentence ending.
+   *
+   * @param {Object} token The token to validate.
+   *
+   * @returns {boolean} Returns true if the token is valid ending, false if it is not.
+   */
+  isSentenceStart(token) {
+    return !isUndefined(token) && ("html-start" === token.type || "html-end" === token.type || "block-start" === token.type);
+  }
+
+  /**
+   * Creates a tokenizer.
+   *
+   * @returns {Object} The tokenizer and the tokens.
+   */
+  createTokenizer() {
+    const tokens = [];
+    const tokenizer = core(function (token) {
+      tokens.push(token);
+    });
+    tokenizer.addRule(fullStopRegex, "full-stop");
+    tokenizer.addRule(blockStartRegex, "block-start");
+    tokenizer.addRule(blockEndRegex, "block-end");
+    tokenizer.addRule(sentenceDelimiterRegex, "sentence-delimiter");
+    tokenizer.addRule(sentenceRegex, "sentence");
+    return {
+      tokenizer,
+      tokens
+    };
+  }
+
+  /**
+   * Tokenizes the given text using the given tokenizer.
+   *
+   * @param {Tokenizer} tokenizer The tokenizer to use.
+   * @param {string}    text      The text to tokenize.
+   *
+   * @returns {void}
+   */
+  tokenize(tokenizer, text) {
+    tokenizer.onText(text);
+    try {
+      tokenizer.end();
+    } catch (e) {
+      console.error("Tokenizer end error:", e, e.tokenizer2);
+    }
+  }
+
+  /**
+   * Determines the start and end indices of a set of sentences form a text.
+   *
+   * @param {Sentence[]} sentences A set of sentences for which to determine indices.
+   *
+   * @returns {void}
+   */
+  determineIndices(sentences) {
+    let currentIndex = 0;
+    for (const sentence of sentences) {
+      const startIndex = currentIndex;
+      sentence.setStartIndex(currentIndex);
+      const endIndex = startIndex + sentence.text.length - 1;
+      sentence.setEndIndex(endIndex);
+      currentIndex = endIndex + 1;
+    }
+  }
+
+  /**
+   * Trims the white space from the beginning of a sentence and adjusts the sentence start index accordingly.
+   *
+   * @param {Sentence} sentence The sentence for which to trim the white space at the start.
+   *
+   * @returns {void}
+   */
+  trimWhiteSpaceAtStart(sentence) {
+    const whiteSpaceLength = sentence.text.match(whiteSpaceStartRegex)[0].length;
+    sentence.setText(sentence.getText().slice(whiteSpaceLength));
+    sentence.setStartIndex(sentence.getStartIndex() + whiteSpaceLength);
+  }
+
+  /**
+   * Trims the white space from the end of a sentence and adjusts the sentence end index accordingly.
+   *
+   * @param {Sentence} sentence The sentence for which to trim the white space at the end.
+   *
+   * @returns {void}
+   */
+  trimWhiteSpaceAtEnd(sentence) {
+    const whiteSpaceLength = sentence.text.match(whiteSpaceEndRegex)[0].length;
+    sentence.setText(sentence.getText().slice(0, sentence.getText().length - whiteSpaceLength));
+    sentence.setEndIndex(sentence.getEndIndex() - whiteSpaceLength);
+  }
+
+  /**
+   * Trims white space from the beginning and end of sentences and adjusts the indices
+   * of the sentence beginnings and ends accordingly.
+   *
+   * @param {Sentence[]} sentences The sentences for which to trim the whitespace.
+   *
+   * @returns {void}
+   */
+  trimWhiteSpaces(sentences) {
+    for (const sentence of sentences) {
+      this.trimWhiteSpaceAtStart(sentence);
+      this.trimWhiteSpaceAtEnd(sentence);
+    }
+  }
+
+  /**
+   * Returns an array of sentence objects for a given array of tokens; assumes that the text has already been split into blocks.
+   *
+   * @param {Object[]} tokenArray The tokens from the sentence tokenizer.
+   *
+   * @returns {Sentence[]} An array of sentence objects.
+   */
+  getSentencesFromTokens(tokenArray) {
+    const tokenSentences = [];
+    let currentSentence = new Sentence("", 0, 0),
+      nextSentenceStart;
+    tokenArray.forEach((token, i) => {
+      let hasNextSentence, nextCharacters;
+      const nextToken = tokenArray[i + 1];
+      const secondToNextToken = tokenArray[i + 2];
+      switch (token.type) {
+        case "sentence":
+          currentSentence.appendText(token.src);
+          break;
+        case "sentence-delimiter":
+          currentSentence.appendText(token.src);
+          if (!isUndefined(nextToken) && "block-end" !== nextToken.type && "sentence-delimiter" !== nextToken.type) {
+            tokenSentences.push(currentSentence);
+            currentSentence = new Sentence("");
+          }
+          break;
+        case "full-stop":
+          currentSentence.appendText(token.src);
+          nextCharacters = this.getNextTwoCharacters([nextToken, secondToNextToken]);
+
+          // For a new sentence we need to check the next two characters.
+          hasNextSentence = nextCharacters.length >= 2;
+          nextSentenceStart = hasNextSentence ? nextCharacters[1] : "";
+          // If the next character is a number, never split. For example: IPv4-numbers.
+          if (hasNextSentence && this.isNumber(nextCharacters[0])) {
+            break;
+          }
+          // Only split on sentence delimiters when the next sentence looks like the start of a sentence.
+          if (hasNextSentence && this.isValidSentenceBeginning(nextSentenceStart) || this.isSentenceStart(nextToken)) {
+            tokenSentences.push(currentSentence);
+            currentSentence = new Sentence("");
+          }
+          break;
+        case "block-start":
+          currentSentence.appendText(token.src);
+          break;
+        case "block-end":
+          currentSentence.appendText(token.src);
+          nextCharacters = this.getNextTwoCharacters([nextToken, secondToNextToken]);
+
+          // For a new sentence we need to check the next two characters.
+          hasNextSentence = nextCharacters.length >= 2;
+          nextSentenceStart = hasNextSentence ? nextCharacters[0] : "";
+          // If the next character is a number, never split. For example: IPv4-numbers.
+          if (hasNextSentence && this.isNumber(nextCharacters[0])) {
+            break;
+          }
+          if (hasNextSentence && this.isValidSentenceBeginning(nextSentenceStart) || this.isSentenceStart(nextToken)) {
+            tokenSentences.push(currentSentence);
+            currentSentence = new Sentence("");
+          }
+          break;
+      }
+    });
+    if (currentSentence.getText() !== "") {
+      tokenSentences.push(currentSentence);
+    }
+    this.determineIndices(tokenSentences);
+    this.trimWhiteSpaces(tokenSentences);
+    return tokenSentences;
+  }
+}
+//# sourceMappingURL=SentenceTokenizer.js.map

package/build/parsedPaper/build/linguisticParsing/SentenceTokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SentenceTokenizer.js","names":["isNaN","isUndefined","core","normalize","normalizeQuotes","Sentence","fullStop","sentenceDelimiters","fullStopRegex","RegExp","sentenceDelimiterRegex","sentenceRegex","blockStartRegex","blockEndRegex","whiteSpaceStartRegex","whiteSpaceEndRegex","SentenceTokenizer","isNumber","character","parseInt","isQuotation","isPunctuation","removeDuplicateWhitespace","text","replace","isCapitalLetter","toLocaleLowerCase","getNextTwoCharacters","nextTokens","next","src","isValidSentenceBeginning","sentenceBeginning","isSentenceStart","token","type","createTokenizer","tokens","tokenizer","push","addRule","tokenize","onText","end","e","console","error","tokenizer2","determineIndices","sentences","currentIndex","sentence","startIndex","setStartIndex","endIndex","length","setEndIndex","trimWhiteSpaceAtStart","whiteSpaceLength","match","setText","getText","slice","getStartIndex","trimWhiteSpaceAtEnd","getEndIndex","trimWhiteSpaces","getSentencesFromTokens","tokenArray","tokenSentences","currentSentence","nextSentenceStart","forEach","i","hasNextSentence","nextCharacters","nextToken","secondToNextToken","appendText"],"sources":["../../../../src/parsedPaper/build/linguisticParsing/SentenceTokenizer.js"],"sourcesContent":["import { isNaN, isUndefined } from \"lodash\";\n\nimport core from \"tokenizer2/core\";\nimport { normalize as normalizeQuotes } from \"@axyseo/languageProcessing/helpers/sanitize/quotes\";\n\nimport Sentence from \"./Sentence\";\n\n// All characters that indicate a sentence delimiter.\nconst fullStop = \".\";\n// The \\u2026 character is an ellipsis\nconst sentenceDelimiters = \"?!;\\u2026\";\n\nconst fullStopRegex = new RegExp( \"^[\" + fullStop + \"]$\" );\nconst sentenceDelimiterRegex = new RegExp( \"^[\" + sentenceDelimiters + \"]$\" );\nconst sentenceRegex = new RegExp( \"^[^\" + fullStop + sentenceDelimiters + \"\\\\(\\\\)\\\\[\\\\]]+$\" );\n\nconst blockStartRegex = /^\\s*[[({]\\s*$/;\nconst blockEndRegex = /^\\s*[\\])}]\\s*$/;\n\nconst whiteSpaceStartRegex = /^\\s*/;\nconst whiteSpaceEndRegex = /\\s*$/;\n\n/**\n * Class for tokenizing a (html) text into sentences.\n */\nexport default class SentenceTokenizer {\n\t/**\n\t * Returns whether or not a certain character is a number.\n\t *\n\t * @param {string} character The character to check.\n\t * @returns {boolean} Whether or not the character is a capital letter.\n\t */\n\tisNumber( character ) {\n\t\treturn ! isNaN( parseInt( character, 10 ) );\n\t}\n\n\t/**\n\t * Returns whether or not a given character is quotation mark.\n\t *\n\t * @param {string} character The character to check.\n\t *\n\t * @returns {boolean} Whether or not the given character is a quotation mark.\n\t */\n\tisQuotation( character ) {\n\t\tcharacter = normalizeQuotes( character );\n\n\t\treturn \"'\" === character ||\n\t\t\t\"\\\"\" === character;\n\t}\n\n\t/**\n\t * Returns whether or not a given character is a punctuation mark that can be at the beginning\n\t * of a sentence, like ¿ and ¡ used in Spanish.\n\t *\n\t * @param {string} character The character to check.\n\t *\n\t * @returns {boolean} Whether or not the given character is a punctuation mark.\n\t */\n\tisPunctuation( character ) {\n\t\treturn \"¿\" === character ||\n\t\t\t\"¡\" === character;\n\t}\n\n\t/**\n\t * Removes duplicate whitespace from a given text.\n\t *\n\t * @param {string} text The text with duplicate whitespace.\n\t * @returns {string} The text without duplicate whitespace.\n\t */\n\tremoveDuplicateWhitespace( text ) {\n\t\treturn text.replace( /\\s+/, \" \" );\n\t}\n\n\t/**\n\t * Returns whether or not a certain character is a capital letter.\n\t *\n\t * @param {string} character The character to check.\n\t *\n\t * @returns {boolean} Whether or not the character is a capital letter.\n\t */\n\tisCapitalLetter( character ) {\n\t\treturn character !== character.toLocaleLowerCase();\n\t}\n\n\t/**\n\t * Retrieves the next two characters from an array with the two next tokens.\n\t *\n\t * @param {Array} nextTokens The two next tokens. Might be undefined.\n\t *\n\t * @returns {string} The next two characters.\n\t */\n\tgetNextTwoCharacters( nextTokens ) {\n\t\tlet next = \"\";\n\n\t\tif ( ! isUndefined( nextTokens[ 0 ] ) ) {\n\t\t\tnext += nextTokens[ 0 ].src;\n\t\t}\n\n\t\tif ( ! isUndefined( nextTokens[ 1 ] ) ) {\n\t\t\tnext += nextTokens[ 1 ].src;\n\t\t}\n\n\t\tnext = this.removeDuplicateWhitespace( next );\n\n\t\treturn next;\n\t}\n\n\t/**\n\t * Checks if the sentenceBeginning beginning is a valid beginning.\n\t *\n\t * @param {string} sentenceBeginning The beginning of the sentence to validate.\n\t *\n\t * @returns {boolean} Returns true if it is a valid beginning, false if it is not.\n\t */\n\tisValidSentenceBeginning( sentenceBeginning ) {\n\t\treturn (\n\t\t\tthis.isCapitalLetter( sentenceBeginning ) ||\n\t\t\tthis.isNumber( sentenceBeginning ) ||\n\t\t\tthis.isQuotation( sentenceBeginning ) ||\n\t\t\tthis.isPunctuation( sentenceBeginning )\n\t\t);\n\t}\n\n\t/**\n\t * Checks if the token is a valid sentence ending.\n\t *\n\t * @param {Object} token The token to validate.\n\t *\n\t * @returns {boolean} Returns true if the token is valid ending, false if it is not.\n\t */\n\tisSentenceStart( token ) {\n\t\treturn ( ! isUndefined( token ) && (\n\t\t\t\"html-start\" === token.type ||\n\t\t\t\"html-end\" === token.type ||\n\t\t\t\"block-start\" === token.type\n\t\t) );\n\t}\n\n\t/**\n\t * Creates a tokenizer.\n\t *\n\t * @returns {Object} The tokenizer and the tokens.\n\t */\n\tcreateTokenizer() {\n\t\tconst tokens = [];\n\t\tconst tokenizer = core( function( token ) {\n\t\t\ttokens.push( token );\n\t\t} );\n\n\t\ttokenizer.addRule( fullStopRegex, \"full-stop\" );\n\t\ttokenizer.addRule( blockStartRegex, \"block-start\" );\n\t\ttokenizer.addRule( blockEndRegex, \"block-end\" );\n\t\ttokenizer.addRule( sentenceDelimiterRegex, \"sentence-delimiter\" );\n\t\ttokenizer.addRule( sentenceRegex, \"sentence\" );\n\n\t\treturn {\n\t\t\ttokenizer,\n\t\t\ttokens,\n\t\t};\n\t}\n\n\t/**\n\t * Tokenizes the given text using the given tokenizer.\n\t *\n\t * @param {Tokenizer} tokenizer The tokenizer to use.\n\t * @param {string}    text      The text to tokenize.\n\t *\n\t * @returns {void}\n\t */\n\ttokenize( tokenizer, text ) {\n\t\ttokenizer.onText( text );\n\n\t\ttry {\n\t\t\ttokenizer.end();\n\t\t} catch ( e ) {\n\t\t\tconsole.error( \"Tokenizer end error:\", e, e.tokenizer2 );\n\t\t}\n\t}\n\n\t/**\n\t * Determines the start and end indices of a set of sentences form a text.\n\t *\n\t * @param {Sentence[]} sentences A set of sentences for which to determine indices.\n\t *\n\t * @returns {void}\n\t */\n\tdetermineIndices( sentences ) {\n\t\tlet currentIndex = 0;\n\n\t\tfor ( const sentence of sentences ) {\n\t\t\tconst startIndex = currentIndex;\n\t\t\tsentence.setStartIndex( currentIndex );\n\t\t\tconst endIndex = startIndex + sentence.text.length - 1;\n\t\t\tsentence.setEndIndex( endIndex );\n\t\t\tcurrentIndex = endIndex + 1;\n\t\t}\n\t}\n\n\t/**\n\t * Trims the white space from the beginning of a sentence and adjusts the sentence start index accordingly.\n\t *\n\t * @param {Sentence} sentence The sentence for which to trim the white space at the start.\n\t *\n\t * @returns {void}\n\t */\n\ttrimWhiteSpaceAtStart( sentence ) {\n\t\tconst whiteSpaceLength = sentence.text.match( whiteSpaceStartRegex )[ 0 ].length;\n\t\tsentence.setText( sentence.getText().slice( whiteSpaceLength ) );\n\t\tsentence.setStartIndex( sentence.getStartIndex() + whiteSpaceLength  );\n\t}\n\n\t/**\n\t * Trims the white space from the end of a sentence and adjusts the sentence end index accordingly.\n\t *\n\t * @param {Sentence} sentence The sentence for which to trim the white space at the end.\n\t *\n\t * @returns {void}\n\t */\n\ttrimWhiteSpaceAtEnd( sentence ) {\n\t\tconst whiteSpaceLength = sentence.text.match( whiteSpaceEndRegex )[ 0 ].length;\n\t\tsentence.setText( sentence.getText().slice( 0, sentence.getText().length - whiteSpaceLength ) );\n\t\tsentence.setEndIndex( sentence.getEndIndex() - whiteSpaceLength );\n\t}\n\n\t/**\n\t * Trims white space from the beginning and end of sentences and adjusts the indices\n\t * of the sentence beginnings and ends accordingly.\n\t *\n\t * @param {Sentence[]} sentences The sentences for which to trim the whitespace.\n\t *\n\t * @returns {void}\n\t */\n\ttrimWhiteSpaces( sentences ) {\n\t\tfor ( const sentence of sentences ) {\n\t\t\tthis.trimWhiteSpaceAtStart( sentence );\n\t\t\tthis.trimWhiteSpaceAtEnd( sentence );\n\t\t}\n\t}\n\n\t/**\n\t * Returns an array of sentence objects for a given array of tokens; assumes that the text has already been split into blocks.\n\t *\n\t * @param {Object[]} tokenArray The tokens from the sentence tokenizer.\n\t *\n\t * @returns {Sentence[]} An array of sentence objects.\n\t */\n\tgetSentencesFromTokens( tokenArray ) {\n\t\tconst tokenSentences = [];\n\t\tlet currentSentence = new Sentence( \"\", 0, 0 ),\n\t\t\tnextSentenceStart;\n\n\t\ttokenArray.forEach( ( token, i ) => {\n\t\t\tlet hasNextSentence, nextCharacters;\n\t\t\tconst nextToken = tokenArray[ i + 1 ];\n\t\t\tconst secondToNextToken = tokenArray[ i + 2 ];\n\n\t\t\tswitch ( token.type ) {\n\t\t\t\tcase \"sentence\":\n\t\t\t\t\tcurrentSentence.appendText( token.src );\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase \"sentence-delimiter\":\n\t\t\t\t\tcurrentSentence.appendText( token.src );\n\t\t\t\t\tif ( ! isUndefined( nextToken ) && \"block-end\" !== nextToken.type && \"sentence-delimiter\" !== nextToken.type ) {\n\t\t\t\t\t\ttokenSentences.push( currentSentence );\n\t\t\t\t\t\tcurrentSentence = new Sentence( \"\" );\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase \"full-stop\":\n\t\t\t\t\tcurrentSentence.appendText( token.src );\n\n\t\t\t\t\tnextCharacters = this.getNextTwoCharacters( [ nextToken, secondToNextToken ] );\n\n\t\t\t\t\t// For a new sentence we need to check the next two characters.\n\t\t\t\t\thasNextSentence = nextCharacters.length >= 2;\n\t\t\t\t\tnextSentenceStart = hasNextSentence ? nextCharacters[ 1 ] : \"\";\n\t\t\t\t\t// If the next character is a number, never split. For example: IPv4-numbers.\n\t\t\t\t\tif ( hasNextSentence && this.isNumber( nextCharacters[ 0 ] ) ) {\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t\t// Only split on sentence delimiters when the next sentence looks like the start of a sentence.\n\t\t\t\t\tif ( ( hasNextSentence && this.isValidSentenceBeginning( nextSentenceStart ) ) || this.isSentenceStart( nextToken ) ) {\n\t\t\t\t\t\ttokenSentences.push( currentSentence );\n\t\t\t\t\t\tcurrentSentence = new Sentence( \"\" );\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase \"block-start\":\n\t\t\t\t\tcurrentSentence.appendText( token.src );\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase \"block-end\":\n\t\t\t\t\tcurrentSentence.appendText( token.src );\n\t\t\t\t\tnextCharacters = this.getNextTwoCharacters( [ nextToken, secondToNextToken ] );\n\n\t\t\t\t\t// For a new sentence we need to check the next two characters.\n\t\t\t\t\thasNextSentence = nextCharacters.length >= 2;\n\t\t\t\t\tnextSentenceStart = hasNextSentence ? nextCharacters[ 0 ] : \"\";\n\t\t\t\t\t// If the next character is a number, never split. For example: IPv4-numbers.\n\t\t\t\t\tif ( hasNextSentence && this.isNumber( nextCharacters[ 0 ] ) ) {\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\n\t\t\t\t\tif ( ( hasNextSentence && this.isValidSentenceBeginning( nextSentenceStart ) ) || this.isSentenceStart( nextToken ) ) {\n\t\t\t\t\t\ttokenSentences.push( currentSentence );\n\t\t\t\t\t\tcurrentSentence = new Sentence( \"\" );\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\t\t\t}\n\t\t} );\n\n\t\tif ( currentSentence.getText()  !== \"\"  ) {\n\t\t\ttokenSentences.push( currentSentence );\n\t\t}\n\n\t\tthis.determineIndices( tokenSentences );\n\n\t\tthis.trimWhiteSpaces( tokenSentences );\n\n\t\treturn tokenSentences;\n\t}\n}\n"],"mappings":"AAAA,SAASA,KAAK,EAAEC,WAAW,QAAQ,QAAQ;AAE3C,OAAOC,IAAI,MAAM,iBAAiB;AAClC,SAASC,SAAS,IAAIC,eAAe;AAErC,OAAOC,QAAQ;;AAEf;AACA,MAAMC,QAAQ,GAAG,GAAG;AACpB;AACA,MAAMC,kBAAkB,GAAG,WAAW;AAEtC,MAAMC,aAAa,GAAG,IAAIC,MAAM,CAAE,IAAI,GAAGH,QAAQ,GAAG,IAAK,CAAC;AAC1D,MAAMI,sBAAsB,GAAG,IAAID,MAAM,CAAE,IAAI,GAAGF,kBAAkB,GAAG,IAAK,CAAC;AAC7E,MAAMI,aAAa,GAAG,IAAIF,MAAM,CAAE,KAAK,GAAGH,QAAQ,GAAGC,kBAAkB,GAAG,iBAAkB,CAAC;AAE7F,MAAMK,eAAe,GAAG,eAAe;AACvC,MAAMC,aAAa,GAAG,gBAAgB;AAEtC,MAAMC,oBAAoB,GAAG,MAAM;AACnC,MAAMC,kBAAkB,GAAG,MAAM;;AAEjC;AACA;AACA;AACA,eAAe,MAAMC,iBAAiB,CAAC;EACtC;AACD;AACA;AACA;AACA;AACA;EACCC,QAAQA,CAAEC,SAAS,EAAG;IACrB,OAAO,CAAElB,KAAK,CAAEmB,QAAQ,CAAED,SAAS,EAAE,EAAG,CAAE,CAAC;EAC5C;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCE,WAAWA,CAAEF,SAAS,EAAG;IACxBA,SAAS,GAAGd,eAAe,CAAEc,SAAU,CAAC;IAExC,OAAO,GAAG,KAAKA,SAAS,IACvB,IAAI,KAAKA,SAAS;EACpB;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;AACA;EACCG,aAAaA,CAAEH,SAAS,EAAG;IAC1B,OAAO,GAAG,KAAKA,SAAS,IACvB,GAAG,KAAKA,SAAS;EACnB;;EAEA;AACD;AACA;AACA;AACA;AACA;EACCI,yBAAyBA,CAAEC,IAAI,EAAG;IACjC,OAAOA,IAAI,CAACC,OAAO,CAAE,KAAK,EAAE,GAAI,CAAC;EAClC;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCC,eAAeA,CAAEP,SAAS,EAAG;IAC5B,OAAOA,SAAS,KAAKA,SAAS,CAACQ,iBAAiB,CAAC,CAAC;EACnD;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCC,oBAAoBA,CAAEC,UAAU,EAAG;IAClC,IAAIC,IAAI,GAAG,EAAE;IAEb,IAAK,CAAE5B,WAAW,CAAE2B,UAAU,CAAE,CAAC,CAAG,CAAC,EAAG;MACvCC,IAAI,IAAID,UAAU,CAAE,CAAC,CAAE,CAACE,GAAG;IAC5B;IAEA,IAAK,CAAE7B,WAAW,CAAE2B,UAAU,CAAE,CAAC,CAAG,CAAC,EAAG;MACvCC,IAAI,IAAID,UAAU,CAAE,CAAC,CAAE,CAACE,GAAG;IAC5B;IAEAD,IAAI,GAAG,IAAI,CAACP,yBAAyB,CAAEO,IAAK,CAAC;IAE7C,OAAOA,IAAI;EACZ;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCE,wBAAwBA,CAAEC,iBAAiB,EAAG;IAC7C,OACC,IAAI,CAACP,eAAe,CAAEO,iBAAkB,CAAC,IACzC,IAAI,CAACf,QAAQ,CAAEe,iBAAkB,CAAC,IAClC,IAAI,CAACZ,WAAW,CAAEY,iBAAkB,CAAC,IACrC,IAAI,CAACX,aAAa,CAAEW,iBAAkB,CAAC;EAEzC;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCC,eAAeA,CAAEC,KAAK,EAAG;IACxB,OAAS,CAAEjC,WAAW,CAAEiC,KAAM,CAAC,KAC9B,YAAY,KAAKA,KAAK,CAACC,IAAI,IAC3B,UAAU,KAAKD,KAAK,CAACC,IAAI,IACzB,aAAa,KAAKD,KAAK,CAACC,IAAI,CAC5B;EACF;;EAEA;AACD;AACA;AACA;AACA;EACCC,eAAeA,CAAA,EAAG;IACjB,MAAMC,MAAM,GAAG,EAAE;IACjB,MAAMC,SAAS,GAAGpC,IAAI,CAAE,UAAUgC,KAAK,EAAG;MACzCG,MAAM,CAACE,IAAI,CAAEL,KAAM,CAAC;IACrB,CAAE,CAAC;IAEHI,SAAS,CAACE,OAAO,CAAEhC,aAAa,EAAE,WAAY,CAAC;IAC/C8B,SAAS,CAACE,OAAO,CAAE5B,eAAe,EAAE,aAAc,CAAC;IACnD0B,SAAS,CAACE,OAAO,CAAE3B,aAAa,EAAE,WAAY,CAAC;IAC/CyB,SAAS,CAACE,OAAO,CAAE9B,sBAAsB,EAAE,oBAAqB,CAAC;IACjE4B,SAAS,CAACE,OAAO,CAAE7B,aAAa,EAAE,UAAW,CAAC;IAE9C,OAAO;MACN2B,SAAS;MACTD;IACD,CAAC;EACF;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;AACA;EACCI,QAAQA,CAAEH,SAAS,EAAEf,IAAI,EAAG;IAC3Be,SAAS,CAACI,MAAM,CAAEnB,IAAK,CAAC;IAExB,IAAI;MACHe,SAAS,CAACK,GAAG,CAAC,CAAC;IAChB,CAAC,CAAC,OAAQC,CAAC,EAAG;MACbC,OAAO,CAACC,KAAK,CAAE,sBAAsB,EAAEF,CAAC,EAAEA,CAAC,CAACG,UAAW,CAAC;IACzD;EACD;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCC,gBAAgBA,CAAEC,SAAS,EAAG;IAC7B,IAAIC,YAAY,GAAG,CAAC;IAEpB,KAAM,MAAMC,QAAQ,IAAIF,SAAS,EAAG;MACnC,MAAMG,UAAU,GAAGF,YAAY;MAC/BC,QAAQ,CAACE,aAAa,CAAEH,YAAa,CAAC;MACtC,MAAMI,QAAQ,GAAGF,UAAU,GAAGD,QAAQ,CAAC5B,IAAI,CAACgC,MAAM,GAAG,CAAC;MACtDJ,QAAQ,CAACK,WAAW,CAAEF,QAAS,CAAC;MAChCJ,YAAY,GAAGI,QAAQ,GAAG,CAAC;IAC5B;EACD;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCG,qBAAqBA,CAAEN,QAAQ,EAAG;IACjC,MAAMO,gBAAgB,GAAGP,QAAQ,CAAC5B,IAAI,CAACoC,KAAK,CAAE7C,oBAAqB,CAAC,CAAE,CAAC,CAAE,CAACyC,MAAM;IAChFJ,QAAQ,CAACS,OAAO,CAAET,QAAQ,CAACU,OAAO,CAAC,CAAC,CAACC,KAAK,CAAEJ,gBAAiB,CAAE,CAAC;IAChEP,QAAQ,CAACE,aAAa,CAAEF,QAAQ,CAACY,aAAa,CAAC,CAAC,GAAGL,gBAAkB,CAAC;EACvE;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCM,mBAAmBA,CAAEb,QAAQ,EAAG;IAC/B,MAAMO,gBAAgB,GAAGP,QAAQ,CAAC5B,IAAI,CAACoC,KAAK,CAAE5C,kBAAmB,CAAC,CAAE,CAAC,CAAE,CAACwC,MAAM;IAC9EJ,QAAQ,CAACS,OAAO,CAAET,QAAQ,CAACU,OAAO,CAAC,CAAC,CAACC,KAAK,CAAE,CAAC,EAAEX,QAAQ,CAACU,OAAO,CAAC,CAAC,CAACN,MAAM,GAAGG,gBAAiB,CAAE,CAAC;IAC/FP,QAAQ,CAACK,WAAW,CAAEL,QAAQ,CAACc,WAAW,CAAC,CAAC,GAAGP,gBAAiB,CAAC;EAClE;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;AACA;EACCQ,eAAeA,CAAEjB,SAAS,EAAG;IAC5B,KAAM,MAAME,QAAQ,IAAIF,SAAS,EAAG;MACnC,IAAI,CAACQ,qBAAqB,CAAEN,QAAS,CAAC;MACtC,IAAI,CAACa,mBAAmB,CAAEb,QAAS,CAAC;IACrC;EACD;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCgB,sBAAsBA,CAAEC,UAAU,EAAG;IACpC,MAAMC,cAAc,GAAG,EAAE;IACzB,IAAIC,eAAe,GAAG,IAAIjE,QAAQ,CAAE,EAAE,EAAE,CAAC,EAAE,CAAE,CAAC;MAC7CkE,iBAAiB;IAElBH,UAAU,CAACI,OAAO,CAAE,CAAEtC,KAAK,EAAEuC,CAAC,KAAM;MACnC,IAAIC,eAAe,EAAEC,cAAc;MACnC,MAAMC,SAAS,GAAGR,UAAU,CAAEK,CAAC,GAAG,CAAC,CAAE;MACrC,MAAMI,iBAAiB,GAAGT,UAAU,CAAEK,CAAC,GAAG,CAAC,CAAE;MAE7C,QAASvC,KAAK,CAACC,IAAI;QAClB,KAAK,UAAU;UACdmC,eAAe,CAACQ,UAAU,CAAE5C,KAAK,CAACJ,GAAI,CAAC;UACvC;QAED,KAAK,oBAAoB;UACxBwC,eAAe,CAACQ,UAAU,CAAE5C,KAAK,CAACJ,GAAI,CAAC;UACvC,IAAK,CAAE7B,WAAW,CAAE2E,SAAU,CAAC,IAAI,WAAW,KAAKA,SAAS,CAACzC,IAAI,IAAI,oBAAoB,KAAKyC,SAAS,CAACzC,IAAI,EAAG;YAC9GkC,cAAc,CAAC9B,IAAI,CAAE+B,eAAgB,CAAC;YACtCA,eAAe,GAAG,IAAIjE,QAAQ,CAAE,EAAG,CAAC;UACrC;UACA;QAED,KAAK,WAAW;UACfiE,eAAe,CAACQ,UAAU,CAAE5C,KAAK,CAACJ,GAAI,CAAC;UAEvC6C,cAAc,GAAG,IAAI,CAAChD,oBAAoB,CAAE,CAAEiD,SAAS,EAAEC,iBAAiB,CAAG,CAAC;;UAE9E;UACAH,eAAe,GAAGC,cAAc,CAACpB,MAAM,IAAI,CAAC;UAC5CgB,iBAAiB,GAAGG,eAAe,GAAGC,cAAc,CAAE,CAAC,CAAE,GAAG,EAAE;UAC9D;UACA,IAAKD,eAAe,IAAI,IAAI,CAACzD,QAAQ,CAAE0D,cAAc,CAAE,CAAC,CAAG,CAAC,EAAG;YAC9D;UACD;UACA;UACA,IAAOD,eAAe,IAAI,IAAI,CAAC3C,wBAAwB,CAAEwC,iBAAkB,CAAC,IAAM,IAAI,CAACtC,eAAe,CAAE2C,SAAU,CAAC,EAAG;YACrHP,cAAc,CAAC9B,IAAI,CAAE+B,eAAgB,CAAC;YACtCA,eAAe,GAAG,IAAIjE,QAAQ,CAAE,EAAG,CAAC;UACrC;UACA;QAED,KAAK,aAAa;UACjBiE,eAAe,CAACQ,UAAU,CAAE5C,KAAK,CAACJ,GAAI,CAAC;UACvC;QAED,KAAK,WAAW;UACfwC,eAAe,CAACQ,UAAU,CAAE5C,KAAK,CAACJ,GAAI,CAAC;UACvC6C,cAAc,GAAG,IAAI,CAAChD,oBAAoB,CAAE,CAAEiD,SAAS,EAAEC,iBAAiB,CAAG,CAAC;;UAE9E;UACAH,eAAe,GAAGC,cAAc,CAACpB,MAAM,IAAI,CAAC;UAC5CgB,iBAAiB,GAAGG,eAAe,GAAGC,cAAc,CAAE,CAAC,CAAE,GAAG,EAAE;UAC9D;UACA,IAAKD,eAAe,IAAI,IAAI,CAACzD,QAAQ,CAAE0D,cAAc,CAAE,CAAC,CAAG,CAAC,EAAG;YAC9D;UACD;UAEA,IAAOD,eAAe,IAAI,IAAI,CAAC3C,wBAAwB,CAAEwC,iBAAkB,CAAC,IAAM,IAAI,CAACtC,eAAe,CAAE2C,SAAU,CAAC,EAAG;YACrHP,cAAc,CAAC9B,IAAI,CAAE+B,eAAgB,CAAC;YACtCA,eAAe,GAAG,IAAIjE,QAAQ,CAAE,EAAG,CAAC;UACrC;UACA;MACF;IACD,CAAE,CAAC;IAEH,IAAKiE,eAAe,CAACT,OAAO,CAAC,CAAC,KAAM,EAAE,EAAI;MACzCQ,cAAc,CAAC9B,IAAI,CAAE+B,eAAgB,CAAC;IACvC;IAEA,IAAI,CAACtB,gBAAgB,CAAEqB,cAAe,CAAC;IAEvC,IAAI,CAACH,eAAe,CAAEG,cAAe,CAAC;IAEtC,OAAOA,cAAc;EACtB;AACD","ignoreList":[]}

package/build/parsedPaper/build/linguisticParsing/parseText.js

@@ -0,0 +1,20 @@
+import SentenceTokenizer from "./SentenceTokenizer";
+
+/**
+ * Parses a text into sentences.
+ *
+ * @param {string} text The text to parse.
+ *
+ * @returns {string[]} An array of sentence objects.
+ */
+const parseTextIntoSentences = function (text) {
+  const sentenceTokenizer = new SentenceTokenizer();
+  const {
+    tokenizer,
+    tokens
+  } = sentenceTokenizer.createTokenizer();
+  sentenceTokenizer.tokenize(tokenizer, text);
+  return sentenceTokenizer.getSentencesFromTokens(tokens);
+};
+export { parseTextIntoSentences };
+//# sourceMappingURL=parseText.js.map

package/build/parsedPaper/build/linguisticParsing/parseText.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parseText.js","names":["SentenceTokenizer","parseTextIntoSentences","text","sentenceTokenizer","tokenizer","tokens","createTokenizer","tokenize","getSentencesFromTokens"],"sources":["../../../../src/parsedPaper/build/linguisticParsing/parseText.js"],"sourcesContent":["import SentenceTokenizer from \"./SentenceTokenizer\";\n\n/**\n * Parses a text into sentences.\n *\n * @param {string} text The text to parse.\n *\n * @returns {string[]} An array of sentence objects.\n */\nconst parseTextIntoSentences = function( text ) {\n\tconst  sentenceTokenizer = new SentenceTokenizer();\n\tconst { tokenizer, tokens } = sentenceTokenizer.createTokenizer();\n\n\tsentenceTokenizer.tokenize( tokenizer, text );\n\treturn sentenceTokenizer.getSentencesFromTokens( tokens );\n};\n\nexport {\n\tparseTextIntoSentences,\n};\n"],"mappings":"AAAA,OAAOA,iBAAiB;;AAExB;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMC,sBAAsB,GAAG,SAAAA,CAAUC,IAAI,EAAG;EAC/C,MAAOC,iBAAiB,GAAG,IAAIH,iBAAiB,CAAC,CAAC;EAClD,MAAM;IAAEI,SAAS;IAAEC;EAAO,CAAC,GAAGF,iBAAiB,CAACG,eAAe,CAAC,CAAC;EAEjEH,iBAAiB,CAACI,QAAQ,CAAEH,SAAS,EAAEF,IAAK,CAAC;EAC7C,OAAOC,iBAAiB,CAACK,sBAAsB,CAAEH,MAAO,CAAC;AAC1D,CAAC;AAED,SACCJ,sBAAsB","ignoreList":[]}

package/build/parsedPaper/build/tree/TreeBuilder.js

@@ -0,0 +1,76 @@
+import buildTree from "./html/buildTree";
+
+/**
+ * Builds a tree representation of a source text.
+ *
+ * Currently supported languages:
+ *  - HTML (`"html"`)
+ *
+ * @memberOf module:parsedPaper/builder
+ */
+class TreeBuilder {
+  /**
+   * Creates a new TreeBuilder.
+   */
+  constructor() {
+    /**
+     * A registry holding the parse function to be called
+     * for each supported formatting language.
+     *
+     * @type {Object<string,Function>}
+     * @private
+     */
+    this._buildFunctions = {
+      html: buildTree
+    };
+    this.build = this.build.bind(this);
+    this.register = this.register.bind(this);
+  }
+
+  /**
+   * Parses the given source text to a tree representation.
+   *
+   * Chooses the right parser depending on the given formatting language in which the text has been written (defaults to HTML).
+   *
+   * @param {string} sourceText                The source text that needs to be parsed to a tree representation.
+   * @param {Object} [options]                 Parse options.
+   * @param {string} [options.language="html"] The formatting language in which the source text has been written, e.g. `"html"` for HTML.
+   *
+   * @returns {module:parsedPaper/structure.Node} The tree representation as parsed from the source text.
+   */
+  build(sourceText, options = {
+    language: "html"
+  }) {
+    return this._buildFunctions[options.language](sourceText);
+  }
+
+  /**
+   * Registers a build function for the given formatting language.
+   *
+   * Call this method if you want to add support for another formatting language.
+   *
+   * @example
+   *   // Create a new build function.
+   *   const myBuilder = sourceText => {
+   *       const node = new Paragraph();
+   *       node.textContainer.appendText( sourceText );
+   *       return node;
+   *   };
+   *
+   *   // Register the build function for the language "my-language".
+   *   treeBuilder.register( "my-language", myBuilder );
+   *
+   *   // Build the tree using the registered builder.
+   *   const tree = treeBuilder.build( "some input", { language: "my-language" } );
+   *
+   * @param {string}                                                            language      The language to register.
+   * @param {function( sourceText: string ): module:parsedPaper/structure.Node} buildFunction The build function for the given language.
+   *
+   * @returns {void}
+   */
+  register(language, buildFunction) {
+    this._buildFunctions[language] = buildFunction;
+  }
+}
+export default TreeBuilder;
+//# sourceMappingURL=TreeBuilder.js.map

package/build/parsedPaper/build/tree/TreeBuilder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TreeBuilder.js","names":["buildTree","TreeBuilder","constructor","_buildFunctions","html","build","bind","register","sourceText","options","language","buildFunction"],"sources":["../../../../src/parsedPaper/build/tree/TreeBuilder.js"],"sourcesContent":["import buildTree from \"./html/buildTree\";\n\n/**\n * Builds a tree representation of a source text.\n *\n * Currently supported languages:\n *  - HTML (`\"html\"`)\n *\n * @memberOf module:parsedPaper/builder\n */\nclass TreeBuilder {\n\t/**\n\t * Creates a new TreeBuilder.\n\t */\n\tconstructor() {\n\t\t/**\n\t\t * A registry holding the parse function to be called\n\t\t * for each supported formatting language.\n\t\t *\n\t\t * @type {Object<string,Function>}\n\t\t * @private\n\t\t */\n\t\tthis._buildFunctions = {\n\t\t\thtml: buildTree,\n\t\t};\n\n\t\tthis.build = this.build.bind( this );\n\t\tthis.register = this.register.bind( this );\n\t}\n\n\t/**\n\t * Parses the given source text to a tree representation.\n\t *\n\t * Chooses the right parser depending on the given formatting language in which the text has been written (defaults to HTML).\n\t *\n\t * @param {string} sourceText                The source text that needs to be parsed to a tree representation.\n\t * @param {Object} [options]                 Parse options.\n\t * @param {string} [options.language=\"html\"] The formatting language in which the source text has been written, e.g. `\"html\"` for HTML.\n\t *\n\t * @returns {module:parsedPaper/structure.Node} The tree representation as parsed from the source text.\n\t */\n\tbuild( sourceText, options = { language: \"html\" } ) {\n\t\treturn this._buildFunctions[ options.language ]( sourceText );\n\t}\n\n\t/**\n\t * Registers a build function for the given formatting language.\n\t *\n\t * Call this method if you want to add support for another formatting language.\n\t *\n\t * @example\n\t *   // Create a new build function.\n\t *   const myBuilder = sourceText => {\n\t *       const node = new Paragraph();\n\t *       node.textContainer.appendText( sourceText );\n\t *       return node;\n\t *   };\n\t *\n\t *   // Register the build function for the language \"my-language\".\n\t *   treeBuilder.register( \"my-language\", myBuilder );\n\t *\n\t *   // Build the tree using the registered builder.\n\t *   const tree = treeBuilder.build( \"some input\", { language: \"my-language\" } );\n\t *\n\t * @param {string}                                                            language      The language to register.\n\t * @param {function( sourceText: string ): module:parsedPaper/structure.Node} buildFunction The build function for the given language.\n\t *\n\t * @returns {void}\n\t */\n\tregister( language, buildFunction ) {\n\t\tthis._buildFunctions[ language ] = buildFunction;\n\t}\n}\n\nexport default TreeBuilder;\n"],"mappings":"AAAA,OAAOA,SAAS;;AAEhB;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMC,WAAW,CAAC;EACjB;AACD;AACA;EACCC,WAAWA,CAAA,EAAG;IACb;AACF;AACA;AACA;AACA;AACA;AACA;IACE,IAAI,CAACC,eAAe,GAAG;MACtBC,IAAI,EAAEJ;IACP,CAAC;IAED,IAAI,CAACK,KAAK,GAAG,IAAI,CAACA,KAAK,CAACC,IAAI,CAAE,IAAK,CAAC;IACpC,IAAI,CAACC,QAAQ,GAAG,IAAI,CAACA,QAAQ,CAACD,IAAI,CAAE,IAAK,CAAC;EAC3C;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACCD,KAAKA,CAAEG,UAAU,EAAEC,OAAO,GAAG;IAAEC,QAAQ,EAAE;EAAO,CAAC,EAAG;IACnD,OAAO,IAAI,CAACP,eAAe,CAAEM,OAAO,CAACC,QAAQ,CAAE,CAAEF,UAAW,CAAC;EAC9D;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACCD,QAAQA,CAAEG,QAAQ,EAAEC,aAAa,EAAG;IACnC,IAAI,CAACR,eAAe,CAAEO,QAAQ,CAAE,GAAGC,aAAa;EACjD;AACD;AAEA,eAAeV,WAAW","ignoreList":[]}