axyseo 2.0.0-alpha.1

package/build/parsedPaper/build/linguisticParsing/SentenceTokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SentenceTokenizer.js","names":["_lodash","require","_core","_interopRequireDefault","_quotes","_Sentence","e","__esModule","default","fullStop","sentenceDelimiters","fullStopRegex","RegExp","sentenceDelimiterRegex","sentenceRegex","blockStartRegex","blockEndRegex","whiteSpaceStartRegex","whiteSpaceEndRegex","SentenceTokenizer","isNumber","character","isNaN","parseInt","isQuotation","normalizeQuotes","isPunctuation","removeDuplicateWhitespace","text","replace","isCapitalLetter","toLocaleLowerCase","getNextTwoCharacters","nextTokens","next","isUndefined","src","isValidSentenceBeginning","sentenceBeginning","isSentenceStart","token","type","createTokenizer","tokens","tokenizer","core","push","addRule","tokenize","onText","end","console","error","tokenizer2","determineIndices","sentences","currentIndex","sentence","startIndex","setStartIndex","endIndex","length","setEndIndex","trimWhiteSpaceAtStart","whiteSpaceLength","match","setText","getText","slice","getStartIndex","trimWhiteSpaceAtEnd","getEndIndex","trimWhiteSpaces","getSentencesFromTokens","tokenArray","tokenSentences","currentSentence","Sentence","nextSentenceStart","forEach","i","hasNextSentence","nextCharacters","nextToken","secondToNextToken","appendText","exports"],"sources":["../../../../src/parsedPaper/build/linguisticParsing/SentenceTokenizer.js"],"sourcesContent":["import { isNaN, isUndefined } from \"lodash\";\n\nimport core from \"tokenizer2/core\";\nimport { normalize as normalizeQuotes } from \"../../../languageProcessing/helpers/sanitize/quotes\";\n\nimport Sentence from \"./Sentence\";\n\n// All characters that indicate a sentence delimiter.\nconst fullStop = \".\";\n// The \\u2026 character is an ellipsis\nconst sentenceDelimiters = \"?!;\\u2026\";\n\nconst fullStopRegex = new RegExp( \"^[\" + fullStop + \"]$\" );\nconst sentenceDelimiterRegex = new RegExp( \"^[\" + sentenceDelimiters + \"]$\" );\nconst sentenceRegex = new RegExp( \"^[^\" + fullStop + sentenceDelimiters + \"\\\\(\\\\)\\\\[\\\\]]+$\" );\n\nconst blockStartRegex = /^\\s*[[({]\\s*$/;\nconst blockEndRegex = /^\\s*[\\])}]\\s*$/;\n\nconst whiteSpaceStartRegex = /^\\s*/;\nconst whiteSpaceEndRegex = /\\s*$/;\n\n/**\n * Class for tokenizing a (html) text into sentences.\n */\nexport default class SentenceTokenizer {\n\t/**\n\t * Returns whether or not a certain character is a number.\n\t *\n\t * @param {string} character The character to check.\n\t * @returns {boolean} Whether or not the character is a capital letter.\n\t */\n\tisNumber( character ) {\n\t\treturn ! isNaN( parseInt( character, 10 ) );\n\t}\n\n\t/**\n\t * Returns whether or not a given character is quotation mark.\n\t *\n\t * @param {string} character The character to check.\n\t *\n\t * @returns {boolean} Whether or not the given character is a quotation mark.\n\t */\n\tisQuotation( character ) {\n\t\tcharacter = normalizeQuotes( character );\n\n\t\treturn \"'\" === character ||\n\t\t\t\"\\\"\" === character;\n\t}\n\n\t/**\n\t * Returns whether or not a given character is a punctuation mark that can be at the beginning\n\t * of a sentence, like ¿ and ¡ used in Spanish.\n\t *\n\t * @param {string} character The character to check.\n\t *\n\t * @returns {boolean} Whether or not the given character is a punctuation mark.\n\t */\n\tisPunctuation( character ) {\n\t\treturn \"¿\" === character ||\n\t\t\t\"¡\" === character;\n\t}\n\n\t/**\n\t * Removes duplicate whitespace from a given text.\n\t *\n\t * @param {string} text The text with duplicate whitespace.\n\t * @returns {string} The text without duplicate whitespace.\n\t */\n\tremoveDuplicateWhitespace( text ) {\n\t\treturn text.replace( /\\s+/, \" \" );\n\t}\n\n\t/**\n\t * Returns whether or not a certain character is a capital letter.\n\t *\n\t * @param {string} character The character to check.\n\t *\n\t * @returns {boolean} Whether or not the character is a capital letter.\n\t */\n\tisCapitalLetter( character ) {\n\t\treturn character !== character.toLocaleLowerCase();\n\t}\n\n\t/**\n\t * Retrieves the next two characters from an array with the two next tokens.\n\t *\n\t * @param {Array} nextTokens The two next tokens. Might be undefined.\n\t *\n\t * @returns {string} The next two characters.\n\t */\n\tgetNextTwoCharacters( nextTokens ) {\n\t\tlet next = \"\";\n\n\t\tif ( ! isUndefined( nextTokens[ 0 ] ) ) {\n\t\t\tnext += nextTokens[ 0 ].src;\n\t\t}\n\n\t\tif ( ! isUndefined( nextTokens[ 1 ] ) ) {\n\t\t\tnext += nextTokens[ 1 ].src;\n\t\t}\n\n\t\tnext = this.removeDuplicateWhitespace( next );\n\n\t\treturn next;\n\t}\n\n\t/**\n\t * Checks if the sentenceBeginning beginning is a valid beginning.\n\t *\n\t * @param {string} sentenceBeginning The beginning of the sentence to validate.\n\t *\n\t * @returns {boolean} Returns true if it is a valid beginning, false if it is not.\n\t */\n\tisValidSentenceBeginning( sentenceBeginning ) {\n\t\treturn (\n\t\t\tthis.isCapitalLetter( sentenceBeginning ) ||\n\t\t\tthis.isNumber( sentenceBeginning ) ||\n\t\t\tthis.isQuotation( sentenceBeginning ) ||\n\t\t\tthis.isPunctuation( sentenceBeginning )\n\t\t);\n\t}\n\n\t/**\n\t * Checks if the token is a valid sentence ending.\n\t *\n\t * @param {Object} token The token to validate.\n\t *\n\t * @returns {boolean} Returns true if the token is valid ending, false if it is not.\n\t */\n\tisSentenceStart( token ) {\n\t\treturn ( ! isUndefined( token ) && (\n\t\t\t\"html-start\" === token.type ||\n\t\t\t\"html-end\" === token.type ||\n\t\t\t\"block-start\" === token.type\n\t\t) );\n\t}\n\n\t/**\n\t * Creates a tokenizer.\n\t *\n\t * @returns {Object} The tokenizer and the tokens.\n\t */\n\tcreateTokenizer() {\n\t\tconst tokens = [];\n\t\tconst tokenizer = core( function( token ) {\n\t\t\ttokens.push( token );\n\t\t} );\n\n\t\ttokenizer.addRule( fullStopRegex, \"full-stop\" );\n\t\ttokenizer.addRule( blockStartRegex, \"block-start\" );\n\t\ttokenizer.addRule( blockEndRegex, \"block-end\" );\n\t\ttokenizer.addRule( sentenceDelimiterRegex, \"sentence-delimiter\" );\n\t\ttokenizer.addRule( sentenceRegex, \"sentence\" );\n\n\t\treturn {\n\t\t\ttokenizer,\n\t\t\ttokens,\n\t\t};\n\t}\n\n\t/**\n\t * Tokenizes the given text using the given tokenizer.\n\t *\n\t * @param {Tokenizer} tokenizer The tokenizer to use.\n\t * @param {string}    text      The text to tokenize.\n\t *\n\t * @returns {void}\n\t */\n\ttokenize( tokenizer, text ) {\n\t\ttokenizer.onText( text );\n\n\t\ttry {\n\t\t\ttokenizer.end();\n\t\t} catch ( e ) {\n\t\t\tconsole.error( \"Tokenizer end error:\", e, e.tokenizer2 );\n\t\t}\n\t}\n\n\t/**\n\t * Determines the start and end indices of a set of sentences form a text.\n\t *\n\t * @param {Sentence[]} sentences A set of sentences for which to determine indices.\n\t *\n\t * @returns {void}\n\t */\n\tdetermineIndices( sentences ) {\n\t\tlet currentIndex = 0;\n\n\t\tfor ( const sentence of sentences ) {\n\t\t\tconst startIndex = currentIndex;\n\t\t\tsentence.setStartIndex( currentIndex );\n\t\t\tconst endIndex = startIndex + sentence.text.length - 1;\n\t\t\tsentence.setEndIndex( endIndex );\n\t\t\tcurrentIndex = endIndex + 1;\n\t\t}\n\t}\n\n\t/**\n\t * Trims the white space from the beginning of a sentence and adjusts the sentence start index accordingly.\n\t *\n\t * @param {Sentence} sentence The sentence for which to trim the white space at the start.\n\t *\n\t * @returns {void}\n\t */\n\ttrimWhiteSpaceAtStart( sentence ) {\n\t\tconst whiteSpaceLength = sentence.text.match( whiteSpaceStartRegex )[ 0 ].length;\n\t\tsentence.setText( sentence.getText().slice( whiteSpaceLength ) );\n\t\tsentence.setStartIndex( sentence.getStartIndex() + whiteSpaceLength  );\n\t}\n\n\t/**\n\t * Trims the white space from the end of a sentence and adjusts the sentence end index accordingly.\n\t *\n\t * @param {Sentence} sentence The sentence for which to trim the white space at the end.\n\t *\n\t * @returns {void}\n\t */\n\ttrimWhiteSpaceAtEnd( sentence ) {\n\t\tconst whiteSpaceLength = sentence.text.match( whiteSpaceEndRegex )[ 0 ].length;\n\t\tsentence.setText( sentence.getText().slice( 0, sentence.getText().length - whiteSpaceLength ) );\n\t\tsentence.setEndIndex( sentence.getEndIndex() - whiteSpaceLength );\n\t}\n\n\t/**\n\t * Trims white space from the beginning and end of sentences and adjusts the indices\n\t * of the sentence beginnings and ends accordingly.\n\t *\n\t * @param {Sentence[]} sentences The sentences for which to trim the whitespace.\n\t *\n\t * @returns {void}\n\t */\n\ttrimWhiteSpaces( sentences ) {\n\t\tfor ( const sentence of sentences ) {\n\t\t\tthis.trimWhiteSpaceAtStart( sentence );\n\t\t\tthis.trimWhiteSpaceAtEnd( sentence );\n\t\t}\n\t}\n\n\t/**\n\t * Returns an array of sentence objects for a given array of tokens; assumes that the text has already been split into blocks.\n\t *\n\t * @param {Object[]} tokenArray The tokens from the sentence tokenizer.\n\t *\n\t * @returns {Sentence[]} An array of sentence objects.\n\t */\n\tgetSentencesFromTokens( tokenArray ) {\n\t\tconst tokenSentences = [];\n\t\tlet currentSentence = new Sentence( \"\", 0, 0 ),\n\t\t\tnextSentenceStart;\n\n\t\ttokenArray.forEach( ( token, i ) => {\n\t\t\tlet hasNextSentence, nextCharacters;\n\t\t\tconst nextToken = tokenArray[ i + 1 ];\n\t\t\tconst secondToNextToken = tokenArray[ i + 2 ];\n\n\t\t\tswitch ( token.type ) {\n\t\t\t\tcase \"sentence\":\n\t\t\t\t\tcurrentSentence.appendText( token.src );\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase \"sentence-delimiter\":\n\t\t\t\t\tcurrentSentence.appendText( token.src );\n\t\t\t\t\tif ( ! isUndefined( nextToken ) && \"block-end\" !== nextToken.type && \"sentence-delimiter\" !== nextToken.type ) {\n\t\t\t\t\t\ttokenSentences.push( currentSentence );\n\t\t\t\t\t\tcurrentSentence = new Sentence( \"\" );\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase \"full-stop\":\n\t\t\t\t\tcurrentSentence.appendText( token.src );\n\n\t\t\t\t\tnextCharacters = this.getNextTwoCharacters( [ nextToken, secondToNextToken ] );\n\n\t\t\t\t\t// For a new sentence we need to check the next two characters.\n\t\t\t\t\thasNextSentence = nextCharacters.length >= 2;\n\t\t\t\t\tnextSentenceStart = hasNextSentence ? nextCharacters[ 1 ] : \"\";\n\t\t\t\t\t// If the next character is a number, never split. For example: IPv4-numbers.\n\t\t\t\t\tif ( hasNextSentence && this.isNumber( nextCharacters[ 0 ] ) ) {\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t\t// Only split on sentence delimiters when the next sentence looks like the start of a sentence.\n\t\t\t\t\tif ( ( hasNextSentence && this.isValidSentenceBeginning( nextSentenceStart ) ) || this.isSentenceStart( nextToken ) ) {\n\t\t\t\t\t\ttokenSentences.push( currentSentence );\n\t\t\t\t\t\tcurrentSentence = new Sentence( \"\" );\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase \"block-start\":\n\t\t\t\t\tcurrentSentence.appendText( token.src );\n\t\t\t\t\tbreak;\n\n\t\t\t\tcase \"block-end\":\n\t\t\t\t\tcurrentSentence.appendText( token.src );\n\t\t\t\t\tnextCharacters = this.getNextTwoCharacters( [ nextToken, secondToNextToken ] );\n\n\t\t\t\t\t// For a new sentence we need to check the next two characters.\n\t\t\t\t\thasNextSentence = nextCharacters.length >= 2;\n\t\t\t\t\tnextSentenceStart = hasNextSentence ? nextCharacters[ 0 ] : \"\";\n\t\t\t\t\t// If the next character is a number, never split. For example: IPv4-numbers.\n\t\t\t\t\tif ( hasNextSentence && this.isNumber( nextCharacters[ 0 ] ) ) {\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\n\t\t\t\t\tif ( ( hasNextSentence && this.isValidSentenceBeginning( nextSentenceStart ) ) || this.isSentenceStart( nextToken ) ) {\n\t\t\t\t\t\ttokenSentences.push( currentSentence );\n\t\t\t\t\t\tcurrentSentence = new Sentence( \"\" );\n\t\t\t\t\t}\n\t\t\t\t\tbreak;\n\t\t\t}\n\t\t} );\n\n\t\tif ( currentSentence.getText()  !== \"\"  ) {\n\t\t\ttokenSentences.push( currentSentence );\n\t\t}\n\n\t\tthis.determineIndices( tokenSentences );\n\n\t\tthis.trimWhiteSpaces( tokenSentences );\n\n\t\treturn tokenSentences;\n\t}\n}\n"],"mappings":";;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AAEA,IAAAC,KAAA,GAAAC,sBAAA,CAAAF,OAAA;AACA,IAAAG,OAAA,GAAAH,OAAA;AAEA,IAAAI,SAAA,GAAAF,sBAAA,CAAAF,OAAA;AAAkC,SAAAE,uBAAAG,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAElC;AACA,MAAMG,QAAQ,GAAG,GAAG;AACpB;AACA,MAAMC,kBAAkB,GAAG,WAAW;AAEtC,MAAMC,aAAa,GAAG,IAAIC,MAAM,CAAE,IAAI,GAAGH,QAAQ,GAAG,IAAK,CAAC;AAC1D,MAAMI,sBAAsB,GAAG,IAAID,MAAM,CAAE,IAAI,GAAGF,kBAAkB,GAAG,IAAK,CAAC;AAC7E,MAAMI,aAAa,GAAG,IAAIF,MAAM,CAAE,KAAK,GAAGH,QAAQ,GAAGC,kBAAkB,GAAG,iBAAkB,CAAC;AAE7F,MAAMK,eAAe,GAAG,eAAe;AACvC,MAAMC,aAAa,GAAG,gBAAgB;AAEtC,MAAMC,oBAAoB,GAAG,MAAM;AACnC,MAAMC,kBAAkB,GAAG,MAAM;;AAEjC;AACA;AACA;AACe,MAAMC,iBAAiB,CAAC;EACtC;AACD;AACA;AACA;AACA;AACA;EACCC,QAAQA,CAAEC,SAAS,EAAG;IACrB,OAAO,CAAE,IAAAC,aAAK,EAAEC,QAAQ,CAAEF,SAAS,EAAE,EAAG,CAAE,CAAC;EAC5C;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCG,WAAWA,CAAEH,SAAS,EAAG;IACxBA,SAAS,GAAG,IAAAI,iBAAe,EAAEJ,SAAU,CAAC;IAExC,OAAO,GAAG,KAAKA,SAAS,IACvB,IAAI,KAAKA,SAAS;EACpB;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;AACA;EACCK,aAAaA,CAAEL,SAAS,EAAG;IAC1B,OAAO,GAAG,KAAKA,SAAS,IACvB,GAAG,KAAKA,SAAS;EACnB;;EAEA;AACD;AACA;AACA;AACA;AACA;EACCM,yBAAyBA,CAAEC,IAAI,EAAG;IACjC,OAAOA,IAAI,CAACC,OAAO,CAAE,KAAK,EAAE,GAAI,CAAC;EAClC;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCC,eAAeA,CAAET,SAAS,EAAG;IAC5B,OAAOA,SAAS,KAAKA,SAAS,CAACU,iBAAiB,CAAC,CAAC;EACnD;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCC,oBAAoBA,CAAEC,UAAU,EAAG;IAClC,IAAIC,IAAI,GAAG,EAAE;IAEb,IAAK,CAAE,IAAAC,mBAAW,EAAEF,UAAU,CAAE,CAAC,CAAG,CAAC,EAAG;MACvCC,IAAI,IAAID,UAAU,CAAE,CAAC,CAAE,CAACG,GAAG;IAC5B;IAEA,IAAK,CAAE,IAAAD,mBAAW,EAAEF,UAAU,CAAE,CAAC,CAAG,CAAC,EAAG;MACvCC,IAAI,IAAID,UAAU,CAAE,CAAC,CAAE,CAACG,GAAG;IAC5B;IAEAF,IAAI,GAAG,IAAI,CAACP,yBAAyB,CAAEO,IAAK,CAAC;IAE7C,OAAOA,IAAI;EACZ;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCG,wBAAwBA,CAAEC,iBAAiB,EAAG;IAC7C,OACC,IAAI,CAACR,eAAe,CAAEQ,iBAAkB,CAAC,IACzC,IAAI,CAAClB,QAAQ,CAAEkB,iBAAkB,CAAC,IAClC,IAAI,CAACd,WAAW,CAAEc,iBAAkB,CAAC,IACrC,IAAI,CAACZ,aAAa,CAAEY,iBAAkB,CAAC;EAEzC;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCC,eAAeA,CAAEC,KAAK,EAAG;IACxB,OAAS,CAAE,IAAAL,mBAAW,EAAEK,KAAM,CAAC,KAC9B,YAAY,KAAKA,KAAK,CAACC,IAAI,IAC3B,UAAU,KAAKD,KAAK,CAACC,IAAI,IACzB,aAAa,KAAKD,KAAK,CAACC,IAAI,CAC5B;EACF;;EAEA;AACD;AACA;AACA;AACA;EACCC,eAAeA,CAAA,EAAG;IACjB,MAAMC,MAAM,GAAG,EAAE;IACjB,MAAMC,SAAS,GAAG,IAAAC,aAAI,EAAE,UAAUL,KAAK,EAAG;MACzCG,MAAM,CAACG,IAAI,CAAEN,KAAM,CAAC;IACrB,CAAE,CAAC;IAEHI,SAAS,CAACG,OAAO,CAAEpC,aAAa,EAAE,WAAY,CAAC;IAC/CiC,SAAS,CAACG,OAAO,CAAEhC,eAAe,EAAE,aAAc,CAAC;IACnD6B,SAAS,CAACG,OAAO,CAAE/B,aAAa,EAAE,WAAY,CAAC;IAC/C4B,SAAS,CAACG,OAAO,CAAElC,sBAAsB,EAAE,oBAAqB,CAAC;IACjE+B,SAAS,CAACG,OAAO,CAAEjC,aAAa,EAAE,UAAW,CAAC;IAE9C,OAAO;MACN8B,SAAS;MACTD;IACD,CAAC;EACF;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;AACA;EACCK,QAAQA,CAAEJ,SAAS,EAAEhB,IAAI,EAAG;IAC3BgB,SAAS,CAACK,MAAM,CAAErB,IAAK,CAAC;IAExB,IAAI;MACHgB,SAAS,CAACM,GAAG,CAAC,CAAC;IAChB,CAAC,CAAC,OAAQ5C,CAAC,EAAG;MACb6C,OAAO,CAACC,KAAK,CAAE,sBAAsB,EAAE9C,CAAC,EAAEA,CAAC,CAAC+C,UAAW,CAAC;IACzD;EACD;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCC,gBAAgBA,CAAEC,SAAS,EAAG;IAC7B,IAAIC,YAAY,GAAG,CAAC;IAEpB,KAAM,MAAMC,QAAQ,IAAIF,SAAS,EAAG;MACnC,MAAMG,UAAU,GAAGF,YAAY;MAC/BC,QAAQ,CAACE,aAAa,CAAEH,YAAa,CAAC;MACtC,MAAMI,QAAQ,GAAGF,UAAU,GAAGD,QAAQ,CAAC7B,IAAI,CAACiC,MAAM,GAAG,CAAC;MACtDJ,QAAQ,CAACK,WAAW,CAAEF,QAAS,CAAC;MAChCJ,YAAY,GAAGI,QAAQ,GAAG,CAAC;IAC5B;EACD;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCG,qBAAqBA,CAAEN,QAAQ,EAAG;IACjC,MAAMO,gBAAgB,GAAGP,QAAQ,CAAC7B,IAAI,CAACqC,KAAK,CAAEhD,oBAAqB,CAAC,CAAE,CAAC,CAAE,CAAC4C,MAAM;IAChFJ,QAAQ,CAACS,OAAO,CAAET,QAAQ,CAACU,OAAO,CAAC,CAAC,CAACC,KAAK,CAAEJ,gBAAiB,CAAE,CAAC;IAChEP,QAAQ,CAACE,aAAa,CAAEF,QAAQ,CAACY,aAAa,CAAC,CAAC,GAAGL,gBAAkB,CAAC;EACvE;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCM,mBAAmBA,CAAEb,QAAQ,EAAG;IAC/B,MAAMO,gBAAgB,GAAGP,QAAQ,CAAC7B,IAAI,CAACqC,KAAK,CAAE/C,kBAAmB,CAAC,CAAE,CAAC,CAAE,CAAC2C,MAAM;IAC9EJ,QAAQ,CAACS,OAAO,CAAET,QAAQ,CAACU,OAAO,CAAC,CAAC,CAACC,KAAK,CAAE,CAAC,EAAEX,QAAQ,CAACU,OAAO,CAAC,CAAC,CAACN,MAAM,GAAGG,gBAAiB,CAAE,CAAC;IAC/FP,QAAQ,CAACK,WAAW,CAAEL,QAAQ,CAACc,WAAW,CAAC,CAAC,GAAGP,gBAAiB,CAAC;EAClE;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;AACA;EACCQ,eAAeA,CAAEjB,SAAS,EAAG;IAC5B,KAAM,MAAME,QAAQ,IAAIF,SAAS,EAAG;MACnC,IAAI,CAACQ,qBAAqB,CAAEN,QAAS,CAAC;MACtC,IAAI,CAACa,mBAAmB,CAAEb,QAAS,CAAC;IACrC;EACD;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;EACCgB,sBAAsBA,CAAEC,UAAU,EAAG;IACpC,MAAMC,cAAc,GAAG,EAAE;IACzB,IAAIC,eAAe,GAAG,IAAIC,iBAAQ,CAAE,EAAE,EAAE,CAAC,EAAE,CAAE,CAAC;MAC7CC,iBAAiB;IAElBJ,UAAU,CAACK,OAAO,CAAE,CAAEvC,KAAK,EAAEwC,CAAC,KAAM;MACnC,IAAIC,eAAe,EAAEC,cAAc;MACnC,MAAMC,SAAS,GAAGT,UAAU,CAAEM,CAAC,GAAG,CAAC,CAAE;MACrC,MAAMI,iBAAiB,GAAGV,UAAU,CAAEM,CAAC,GAAG,CAAC,CAAE;MAE7C,QAASxC,KAAK,CAACC,IAAI;QAClB,KAAK,UAAU;UACdmC,eAAe,CAACS,UAAU,CAAE7C,KAAK,CAACJ,GAAI,CAAC;UACvC;QAED,KAAK,oBAAoB;UACxBwC,eAAe,CAACS,UAAU,CAAE7C,KAAK,CAACJ,GAAI,CAAC;UACvC,IAAK,CAAE,IAAAD,mBAAW,EAAEgD,SAAU,CAAC,IAAI,WAAW,KAAKA,SAAS,CAAC1C,IAAI,IAAI,oBAAoB,KAAK0C,SAAS,CAAC1C,IAAI,EAAG;YAC9GkC,cAAc,CAAC7B,IAAI,CAAE8B,eAAgB,CAAC;YACtCA,eAAe,GAAG,IAAIC,iBAAQ,CAAE,EAAG,CAAC;UACrC;UACA;QAED,KAAK,WAAW;UACfD,eAAe,CAACS,UAAU,CAAE7C,KAAK,CAACJ,GAAI,CAAC;UAEvC8C,cAAc,GAAG,IAAI,CAAClD,oBAAoB,CAAE,CAAEmD,SAAS,EAAEC,iBAAiB,CAAG,CAAC;;UAE9E;UACAH,eAAe,GAAGC,cAAc,CAACrB,MAAM,IAAI,CAAC;UAC5CiB,iBAAiB,GAAGG,eAAe,GAAGC,cAAc,CAAE,CAAC,CAAE,GAAG,EAAE;UAC9D;UACA,IAAKD,eAAe,IAAI,IAAI,CAAC7D,QAAQ,CAAE8D,cAAc,CAAE,CAAC,CAAG,CAAC,EAAG;YAC9D;UACD;UACA;UACA,IAAOD,eAAe,IAAI,IAAI,CAAC5C,wBAAwB,CAAEyC,iBAAkB,CAAC,IAAM,IAAI,CAACvC,eAAe,CAAE4C,SAAU,CAAC,EAAG;YACrHR,cAAc,CAAC7B,IAAI,CAAE8B,eAAgB,CAAC;YACtCA,eAAe,GAAG,IAAIC,iBAAQ,CAAE,EAAG,CAAC;UACrC;UACA;QAED,KAAK,aAAa;UACjBD,eAAe,CAACS,UAAU,CAAE7C,KAAK,CAACJ,GAAI,CAAC;UACvC;QAED,KAAK,WAAW;UACfwC,eAAe,CAACS,UAAU,CAAE7C,KAAK,CAACJ,GAAI,CAAC;UACvC8C,cAAc,GAAG,IAAI,CAAClD,oBAAoB,CAAE,CAAEmD,SAAS,EAAEC,iBAAiB,CAAG,CAAC;;UAE9E;UACAH,eAAe,GAAGC,cAAc,CAACrB,MAAM,IAAI,CAAC;UAC5CiB,iBAAiB,GAAGG,eAAe,GAAGC,cAAc,CAAE,CAAC,CAAE,GAAG,EAAE;UAC9D;UACA,IAAKD,eAAe,IAAI,IAAI,CAAC7D,QAAQ,CAAE8D,cAAc,CAAE,CAAC,CAAG,CAAC,EAAG;YAC9D;UACD;UAEA,IAAOD,eAAe,IAAI,IAAI,CAAC5C,wBAAwB,CAAEyC,iBAAkB,CAAC,IAAM,IAAI,CAACvC,eAAe,CAAE4C,SAAU,CAAC,EAAG;YACrHR,cAAc,CAAC7B,IAAI,CAAE8B,eAAgB,CAAC;YACtCA,eAAe,GAAG,IAAIC,iBAAQ,CAAE,EAAG,CAAC;UACrC;UACA;MACF;IACD,CAAE,CAAC;IAEH,IAAKD,eAAe,CAACT,OAAO,CAAC,CAAC,KAAM,EAAE,EAAI;MACzCQ,cAAc,CAAC7B,IAAI,CAAE8B,eAAgB,CAAC;IACvC;IAEA,IAAI,CAACtB,gBAAgB,CAAEqB,cAAe,CAAC;IAEvC,IAAI,CAACH,eAAe,CAAEG,cAAe,CAAC;IAEtC,OAAOA,cAAc;EACtB;AACD;AAACW,OAAA,CAAA9E,OAAA,GAAAW,iBAAA","ignoreList":[]}

package/build/parsedPaper/build/linguisticParsing/parseText.js

@@ -0,0 +1,26 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.parseTextIntoSentences = void 0;
+var _SentenceTokenizer = _interopRequireDefault(require("./SentenceTokenizer"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Parses a text into sentences.
+ *
+ * @param {string} text The text to parse.
+ *
+ * @returns {string[]} An array of sentence objects.
+ */
+const parseTextIntoSentences = function (text) {
+  const sentenceTokenizer = new _SentenceTokenizer.default();
+  const {
+    tokenizer,
+    tokens
+  } = sentenceTokenizer.createTokenizer();
+  sentenceTokenizer.tokenize(tokenizer, text);
+  return sentenceTokenizer.getSentencesFromTokens(tokens);
+};
+exports.parseTextIntoSentences = parseTextIntoSentences;
+//# sourceMappingURL=parseText.js.map

package/build/parsedPaper/build/linguisticParsing/parseText.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parseText.js","names":["_SentenceTokenizer","_interopRequireDefault","require","e","__esModule","default","parseTextIntoSentences","text","sentenceTokenizer","SentenceTokenizer","tokenizer","tokens","createTokenizer","tokenize","getSentencesFromTokens","exports"],"sources":["../../../../src/parsedPaper/build/linguisticParsing/parseText.js"],"sourcesContent":["import SentenceTokenizer from \"./SentenceTokenizer\";\n\n/**\n * Parses a text into sentences.\n *\n * @param {string} text The text to parse.\n *\n * @returns {string[]} An array of sentence objects.\n */\nconst parseTextIntoSentences = function( text ) {\n\tconst  sentenceTokenizer = new SentenceTokenizer();\n\tconst { tokenizer, tokens } = sentenceTokenizer.createTokenizer();\n\n\tsentenceTokenizer.tokenize( tokenizer, text );\n\treturn sentenceTokenizer.getSentencesFromTokens( tokens );\n};\n\nexport {\n\tparseTextIntoSentences,\n};\n"],"mappings":";;;;;;AAAA,IAAAA,kBAAA,GAAAC,sBAAA,CAAAC,OAAA;AAAoD,SAAAD,uBAAAE,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAEpD;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMG,sBAAsB,GAAG,SAAAA,CAAUC,IAAI,EAAG;EAC/C,MAAOC,iBAAiB,GAAG,IAAIC,0BAAiB,CAAC,CAAC;EAClD,MAAM;IAAEC,SAAS;IAAEC;EAAO,CAAC,GAAGH,iBAAiB,CAACI,eAAe,CAAC,CAAC;EAEjEJ,iBAAiB,CAACK,QAAQ,CAAEH,SAAS,EAAEH,IAAK,CAAC;EAC7C,OAAOC,iBAAiB,CAACM,sBAAsB,CAAEH,MAAO,CAAC;AAC1D,CAAC;AAACI,OAAA,CAAAT,sBAAA,GAAAA,sBAAA","ignoreList":[]}

package/build/parsedPaper/build/tree/TreeBuilder.js

@@ -0,0 +1,82 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _buildTree = _interopRequireDefault(require("./html/buildTree"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Builds a tree representation of a source text.
+ *
+ * Currently supported languages:
+ *  - HTML (`"html"`)
+ *
+ * @memberOf module:parsedPaper/builder
+ */
+class TreeBuilder {
+  /**
+   * Creates a new TreeBuilder.
+   */
+  constructor() {
+    /**
+     * A registry holding the parse function to be called
+     * for each supported formatting language.
+     *
+     * @type {Object<string,Function>}
+     * @private
+     */
+    this._buildFunctions = {
+      html: _buildTree.default
+    };
+    this.build = this.build.bind(this);
+    this.register = this.register.bind(this);
+  }
+
+  /**
+   * Parses the given source text to a tree representation.
+   *
+   * Chooses the right parser depending on the given formatting language in which the text has been written (defaults to HTML).
+   *
+   * @param {string} sourceText                The source text that needs to be parsed to a tree representation.
+   * @param {Object} [options]                 Parse options.
+   * @param {string} [options.language="html"] The formatting language in which the source text has been written, e.g. `"html"` for HTML.
+   *
+   * @returns {module:parsedPaper/structure.Node} The tree representation as parsed from the source text.
+   */
+  build(sourceText, options = {
+    language: "html"
+  }) {
+    return this._buildFunctions[options.language](sourceText);
+  }
+
+  /**
+   * Registers a build function for the given formatting language.
+   *
+   * Call this method if you want to add support for another formatting language.
+   *
+   * @example
+   *   // Create a new build function.
+   *   const myBuilder = sourceText => {
+   *       const node = new Paragraph();
+   *       node.textContainer.appendText( sourceText );
+   *       return node;
+   *   };
+   *
+   *   // Register the build function for the language "my-language".
+   *   treeBuilder.register( "my-language", myBuilder );
+   *
+   *   // Build the tree using the registered builder.
+   *   const tree = treeBuilder.build( "some input", { language: "my-language" } );
+   *
+   * @param {string}                                                            language      The language to register.
+   * @param {function( sourceText: string ): module:parsedPaper/structure.Node} buildFunction The build function for the given language.
+   *
+   * @returns {void}
+   */
+  register(language, buildFunction) {
+    this._buildFunctions[language] = buildFunction;
+  }
+}
+var _default = exports.default = TreeBuilder;
+//# sourceMappingURL=TreeBuilder.js.map

package/build/parsedPaper/build/tree/TreeBuilder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TreeBuilder.js","names":["_buildTree","_interopRequireDefault","require","e","__esModule","default","TreeBuilder","constructor","_buildFunctions","html","buildTree","build","bind","register","sourceText","options","language","buildFunction","_default","exports"],"sources":["../../../../src/parsedPaper/build/tree/TreeBuilder.js"],"sourcesContent":["import buildTree from \"./html/buildTree\";\n\n/**\n * Builds a tree representation of a source text.\n *\n * Currently supported languages:\n *  - HTML (`\"html\"`)\n *\n * @memberOf module:parsedPaper/builder\n */\nclass TreeBuilder {\n\t/**\n\t * Creates a new TreeBuilder.\n\t */\n\tconstructor() {\n\t\t/**\n\t\t * A registry holding the parse function to be called\n\t\t * for each supported formatting language.\n\t\t *\n\t\t * @type {Object<string,Function>}\n\t\t * @private\n\t\t */\n\t\tthis._buildFunctions = {\n\t\t\thtml: buildTree,\n\t\t};\n\n\t\tthis.build = this.build.bind( this );\n\t\tthis.register = this.register.bind( this );\n\t}\n\n\t/**\n\t * Parses the given source text to a tree representation.\n\t *\n\t * Chooses the right parser depending on the given formatting language in which the text has been written (defaults to HTML).\n\t *\n\t * @param {string} sourceText                The source text that needs to be parsed to a tree representation.\n\t * @param {Object} [options]                 Parse options.\n\t * @param {string} [options.language=\"html\"] The formatting language in which the source text has been written, e.g. `\"html\"` for HTML.\n\t *\n\t * @returns {module:parsedPaper/structure.Node} The tree representation as parsed from the source text.\n\t */\n\tbuild( sourceText, options = { language: \"html\" } ) {\n\t\treturn this._buildFunctions[ options.language ]( sourceText );\n\t}\n\n\t/**\n\t * Registers a build function for the given formatting language.\n\t *\n\t * Call this method if you want to add support for another formatting language.\n\t *\n\t * @example\n\t *   // Create a new build function.\n\t *   const myBuilder = sourceText => {\n\t *       const node = new Paragraph();\n\t *       node.textContainer.appendText( sourceText );\n\t *       return node;\n\t *   };\n\t *\n\t *   // Register the build function for the language \"my-language\".\n\t *   treeBuilder.register( \"my-language\", myBuilder );\n\t *\n\t *   // Build the tree using the registered builder.\n\t *   const tree = treeBuilder.build( \"some input\", { language: \"my-language\" } );\n\t *\n\t * @param {string}                                                            language      The language to register.\n\t * @param {function( sourceText: string ): module:parsedPaper/structure.Node} buildFunction The build function for the given language.\n\t *\n\t * @returns {void}\n\t */\n\tregister( language, buildFunction ) {\n\t\tthis._buildFunctions[ language ] = buildFunction;\n\t}\n}\n\nexport default TreeBuilder;\n"],"mappings":";;;;;;AAAA,IAAAA,UAAA,GAAAC,sBAAA,CAAAC,OAAA;AAAyC,SAAAD,uBAAAE,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAEzC;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMG,WAAW,CAAC;EACjB;AACD;AACA;EACCC,WAAWA,CAAA,EAAG;IACb;AACF;AACA;AACA;AACA;AACA;AACA;IACE,IAAI,CAACC,eAAe,GAAG;MACtBC,IAAI,EAAEC;IACP,CAAC;IAED,IAAI,CAACC,KAAK,GAAG,IAAI,CAACA,KAAK,CAACC,IAAI,CAAE,IAAK,CAAC;IACpC,IAAI,CAACC,QAAQ,GAAG,IAAI,CAACA,QAAQ,CAACD,IAAI,CAAE,IAAK,CAAC;EAC3C;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACCD,KAAKA,CAAEG,UAAU,EAAEC,OAAO,GAAG;IAAEC,QAAQ,EAAE;EAAO,CAAC,EAAG;IACnD,OAAO,IAAI,CAACR,eAAe,CAAEO,OAAO,CAACC,QAAQ,CAAE,CAAEF,UAAW,CAAC;EAC9D;;EAEA;AACD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACCD,QAAQA,CAAEG,QAAQ,EAAEC,aAAa,EAAG;IACnC,IAAI,CAACT,eAAe,CAAEQ,QAAQ,CAAE,GAAGC,aAAa;EACjD;AACD;AAAC,IAAAC,QAAA,GAAAC,OAAA,CAAAd,OAAA,GAEcC,WAAW","ignoreList":[]}

package/build/parsedPaper/build/tree/cleanup/calculateTextIndices.js

@@ -0,0 +1,183 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _lodash = require("lodash");
+var _htmlConstants = require("../html/htmlConstants");
+/**
+ * Gathers all elements that can be closed given the position of the current element in the source code.
+ *
+ * Elements that can be closed are all elements that are opened before this element, but in which this element is
+ * not nested.
+ * E.g.
+ * ```html
+ * <strong>Hello</strong><em>World<b>!!!</b></em>`
+ * ```
+ * with `<b>!!!</b>` as the current element,
+ * means that the `<strong>` needs to be closed, but `<em>` **not**.
+ *
+ * @param {module:parsedPaper/structure.FormattingElement} currentElement The current element.
+ * @param {module:parsedPaper/structure.FormattingElement[]} openElements The elements that are currently open.
+ *
+ * @returns {module:parsedPaper/structure.FormattingElement[]} The elements that can be closed.
+ */
+const elementsThatCanBeClosed = function (currentElement, openElements) {
+  return openElements.filter(el => {
+    const endTag = el.sourceCodeLocation.endTag;
+    return endTag.endOffset <= currentElement.sourceCodeLocation.startOffset;
+  });
+};
+
+/**
+ * Closes the elements that can be closed given the position of the current element within the source code.
+ *
+ * This does two things:
+ *  1. The closed element's text end index is calculated based on the current offset.
+ *  2. The closed element's end tag lengths are counted towards the current offset, to make sure that the computed position
+ * of the formatting elements are still correct.
+ *
+ * @param {module:parsedPaper/structure.FormattingElement[]} elementsToClose The list of open elements that need to be closed
+ * @param {number} currentOffset                                      The current offset when parsing the formatting elements
+ *
+ * @returns {number} The updated current offset
+ *
+ * @private
+ */
+const closeElements = function (elementsToClose, currentOffset) {
+  // Sort, so we close all elements in the right order.
+  elementsToClose.sort((a, b) => a.sourceCodeLocation.endTag.endOffset - b.sourceCodeLocation.endTag.endOffset);
+  elementsToClose.forEach(elementToClose => {
+    const endTag = elementToClose.sourceCodeLocation.endTag;
+    // Set the end position as seen in the text.
+    elementToClose.textEndIndex = endTag.startOffset - currentOffset;
+    /*
+      Add the end tag length of the to be closed element to the total offset.
+     */
+    const endTagLength = endTag.endOffset - endTag.startOffset;
+    currentOffset += endTagLength;
+  });
+  return currentOffset;
+};
+
+/**
+ * Adds the content length of the given element (the part between the tags) to the current offset
+ * and adds the content to the element as a parameter.
+ *
+ * @param {module:parsedPaper/structure.FormattingElement} element The element of which to add the content length.
+ * @param {number} currentOffset                            The current offset to which to add the length to.
+ *
+ * @returns {number} The updated current offset
+ */
+const handleIgnoredContent = function (element, currentOffset) {
+  // Has 0 length in text, so end = start.
+  element.textEndIndex = element.textStartIndex;
+
+  // Update current offset.
+  const end = element.sourceCodeLocation.endTag ? element.sourceCodeLocation.endTag.startOffset : element.sourceCodeLocation.endOffset;
+  const start = element.sourceCodeLocation.startTag ? element.sourceCodeLocation.startTag.endOffset : element.sourceCodeLocation.startOffset;
+  currentOffset += end - start;
+  return currentOffset;
+};
+
+/**
+ * Sets the start and end text positions of a comment.
+ *
+ * @param {module:parsedPaper/structure.FormattingElement}	element			The formatting element to assign start and end text positions to.
+ * @param {int}												currentOffset	A sum of all characters in the source code that don't get rendered
+ * 																			(e.g., tags, comments).
+ *
+ * @returns {number} The length of the comment.
+ *
+ * @private
+ */
+const computeCommentStartEndTextIndices = function (element, currentOffset) {
+  element.textStartIndex = element.sourceCodeLocation.startOffset - currentOffset;
+  element.textEndIndex = element.textStartIndex;
+  return element.sourceCodeLocation.endOffset - element.sourceCodeLocation.startOffset;
+};
+
+/**
+ * Sets the start and end text positions of one formatting element.
+ *
+ * @param {module:parsedPaper/structure.FormattingElement}	element			The formatting element to assign start and end text positions to.
+ * @param {int}												currentOffset	A sum of all characters in the source code that don't get rendered
+ * 																			(e.g., tags, comments).
+ *
+ * @returns {int} The updated currentOffset.
+ *
+ * @private
+ */
+const computeElementStartTextIndex = function (element, currentOffset) {
+  const startTag = element.sourceCodeLocation.startTag;
+
+  // For example: "<strong>".length
+  const startTagLength = startTag.endOffset - startTag.startOffset;
+  currentOffset += startTagLength;
+
+  // Set start position of element in heading's / paragraph's text.
+  element.textStartIndex = startTag.endOffset - currentOffset;
+
+  /*
+    Elements that have no end tags (e.g., void element like <img/> or self-closing elements) can be closed immediately.
+    The text length of those elements will be automatically 0.
+   */
+  if (!element.sourceCodeLocation.endTag) {
+    element.textEndIndex = element.textStartIndex;
+  }
+  return currentOffset;
+};
+
+/**
+ * Sets the start and end position of the text in formatting elements of the given node.
+ *
+ * @param {module:parsedPaper/structure.LeafNode} node The node containing a TextContainer
+ *
+ * @returns {void}
+ *
+ * @private
+ */
+const calculateTextIndices = function (node) {
+  if (!node.textContainer.formatting || node.textContainer.formatting.length === 0) {
+    return;
+  }
+  const openElements = [];
+
+  /*
+    Keeps track of the current total size of the start and end tags (and the ignored content)
+    These should not be counted towards the start and end position of the elements in the text.
+   */
+  let currentOffset = node.sourceCodeLocation.startTag ? node.sourceCodeLocation.startTag.endOffset : node.sourceCodeLocation.startOffset;
+  node.textContainer.formatting.forEach(element => {
+    // Close elements that can be closed and remove them from the list of open elements.
+    const elementsToClose = elementsThatCanBeClosed(element, openElements);
+    currentOffset = closeElements(elementsToClose, currentOffset);
+    (0, _lodash.pullAll)(openElements, elementsToClose);
+
+    // Comments are self-closing formatting elements that are completely ignored in rendering.
+    if (element.type === "#comment") {
+      currentOffset += computeCommentStartEndTextIndices(element, currentOffset);
+      return;
+    }
+    currentOffset = computeElementStartTextIndex(element, currentOffset);
+
+    // If there is an endTag, the element should be closed in one of the next iterations of the loop.
+    if (element.sourceCodeLocation.endTag) {
+      openElements.push(element);
+    }
+
+    /*
+      If this element is an ignored element its contents are not in the text,
+      so its content should be added to the respective formatting element instead,
+      and the current offset should be updated.
+     */
+    if (_htmlConstants.ignoredHtmlElements.includes(element.type)) {
+      currentOffset = handleIgnoredContent(element, currentOffset);
+    }
+  });
+  // Close all remaining elements.
+  closeElements(openElements, currentOffset);
+};
+var _default = exports.default = calculateTextIndices;
+//# sourceMappingURL=calculateTextIndices.js.map

package/build/parsedPaper/build/tree/cleanup/calculateTextIndices.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"calculateTextIndices.js","names":["_lodash","require","_htmlConstants","elementsThatCanBeClosed","currentElement","openElements","filter","el","endTag","sourceCodeLocation","endOffset","startOffset","closeElements","elementsToClose","currentOffset","sort","a","b","forEach","elementToClose","textEndIndex","endTagLength","handleIgnoredContent","element","textStartIndex","end","start","startTag","computeCommentStartEndTextIndices","computeElementStartTextIndex","startTagLength","calculateTextIndices","node","textContainer","formatting","length","pullAll","type","push","ignoredHtmlElements","includes","_default","exports","default"],"sources":["../../../../../src/parsedPaper/build/tree/cleanup/calculateTextIndices.js"],"sourcesContent":["import { pullAll } from \"lodash\";\n\nimport { ignoredHtmlElements } from \"../html/htmlConstants\";\n\n/**\n * Gathers all elements that can be closed given the position of the current element in the source code.\n *\n * Elements that can be closed are all elements that are opened before this element, but in which this element is\n * not nested.\n * E.g.\n * ```html\n * <strong>Hello</strong><em>World<b>!!!</b></em>`\n * ```\n * with `<b>!!!</b>` as the current element,\n * means that the `<strong>` needs to be closed, but `<em>` **not**.\n *\n * @param {module:parsedPaper/structure.FormattingElement} currentElement The current element.\n * @param {module:parsedPaper/structure.FormattingElement[]} openElements The elements that are currently open.\n *\n * @returns {module:parsedPaper/structure.FormattingElement[]} The elements that can be closed.\n */\nconst elementsThatCanBeClosed = function( currentElement, openElements ) {\n\treturn openElements.filter( el => {\n\t\tconst endTag = el.sourceCodeLocation.endTag;\n\t\treturn endTag.endOffset <= currentElement.sourceCodeLocation.startOffset;\n\t} );\n};\n\n/**\n * Closes the elements that can be closed given the position of the current element within the source code.\n *\n * This does two things:\n *  1. The closed element's text end index is calculated based on the current offset.\n *  2. The closed element's end tag lengths are counted towards the current offset, to make sure that the computed position\n * of the formatting elements are still correct.\n *\n * @param {module:parsedPaper/structure.FormattingElement[]} elementsToClose The list of open elements that need to be closed\n * @param {number} currentOffset                                      The current offset when parsing the formatting elements\n *\n * @returns {number} The updated current offset\n *\n * @private\n */\nconst closeElements = function( elementsToClose, currentOffset ) {\n\t// Sort, so we close all elements in the right order.\n\telementsToClose.sort( ( a, b ) => a.sourceCodeLocation.endTag.endOffset - b.sourceCodeLocation.endTag.endOffset );\n\n\telementsToClose.forEach( elementToClose => {\n\t\tconst endTag = elementToClose.sourceCodeLocation.endTag;\n\t\t// Set the end position as seen in the text.\n\t\telementToClose.textEndIndex = endTag.startOffset - currentOffset;\n\t\t/*\n\t\t  Add the end tag length of the to be closed element to the total offset.\n\t\t */\n\t\tconst endTagLength = endTag.endOffset - endTag.startOffset;\n\t\tcurrentOffset += endTagLength;\n\t} );\n\n\treturn currentOffset;\n};\n\n/**\n * Adds the content length of the given element (the part between the tags) to the current offset\n * and adds the content to the element as a parameter.\n *\n * @param {module:parsedPaper/structure.FormattingElement} element The element of which to add the content length.\n * @param {number} currentOffset                            The current offset to which to add the length to.\n *\n * @returns {number} The updated current offset\n */\nconst handleIgnoredContent = function( element, currentOffset ) {\n\t// Has 0 length in text, so end = start.\n\telement.textEndIndex = element.textStartIndex;\n\n\t// Update current offset.\n\tconst end = element.sourceCodeLocation.endTag ? element.sourceCodeLocation.endTag.startOffset : element.sourceCodeLocation.endOffset;\n\tconst start = element.sourceCodeLocation.startTag ? element.sourceCodeLocation.startTag.endOffset : element.sourceCodeLocation.startOffset;\n\n\tcurrentOffset += end - start;\n\n\treturn currentOffset;\n};\n\n\n/**\n * Sets the start and end text positions of a comment.\n *\n * @param {module:parsedPaper/structure.FormattingElement}\telement\t\t\tThe formatting element to assign start and end text positions to.\n * @param {int}\t\t\t\t\t\t\t\t\t\t\t\tcurrentOffset\tA sum of all characters in the source code that don't get rendered\n * \t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t(e.g., tags, comments).\n *\n * @returns {number} The length of the comment.\n *\n * @private\n */\nconst computeCommentStartEndTextIndices = function( element, currentOffset ) {\n\telement.textStartIndex = element.sourceCodeLocation.startOffset - currentOffset;\n\telement.textEndIndex = element.textStartIndex;\n\n\treturn element.sourceCodeLocation.endOffset - element.sourceCodeLocation.startOffset;\n};\n\n/**\n * Sets the start and end text positions of one formatting element.\n *\n * @param {module:parsedPaper/structure.FormattingElement}\telement\t\t\tThe formatting element to assign start and end text positions to.\n * @param {int}\t\t\t\t\t\t\t\t\t\t\t\tcurrentOffset\tA sum of all characters in the source code that don't get rendered\n * \t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t(e.g., tags, comments).\n *\n * @returns {int} The updated currentOffset.\n *\n * @private\n */\nconst computeElementStartTextIndex = function( element, currentOffset ) {\n\tconst startTag = element.sourceCodeLocation.startTag;\n\n\t// For example: \"<strong>\".length\n\tconst startTagLength = startTag.endOffset - startTag.startOffset;\n\n\tcurrentOffset += startTagLength;\n\n\t// Set start position of element in heading's / paragraph's text.\n\telement.textStartIndex = startTag.endOffset - currentOffset;\n\n\t/*\n\t  Elements that have no end tags (e.g., void element like <img/> or self-closing elements) can be closed immediately.\n\t  The text length of those elements will be automatically 0.\n\t */\n\tif ( ! element.sourceCodeLocation.endTag ) {\n\t\telement.textEndIndex = element.textStartIndex;\n\t}\n\n\treturn currentOffset;\n};\n\n/**\n * Sets the start and end position of the text in formatting elements of the given node.\n *\n * @param {module:parsedPaper/structure.LeafNode} node The node containing a TextContainer\n *\n * @returns {void}\n *\n * @private\n */\nconst calculateTextIndices = function( node ) {\n\tif ( ! node.textContainer.formatting || node.textContainer.formatting.length === 0 ) {\n\t\treturn;\n\t}\n\n\tconst openElements = [];\n\n\t/*\n\t  Keeps track of the current total size of the start and end tags (and the ignored content)\n\t  These should not be counted towards the start and end position of the elements in the text.\n\t */\n\tlet currentOffset = node.sourceCodeLocation.startTag ? node.sourceCodeLocation.startTag.endOffset : node.sourceCodeLocation.startOffset;\n\n\tnode.textContainer.formatting.forEach( element => {\n\t\t// Close elements that can be closed and remove them from the list of open elements.\n\t\tconst elementsToClose = elementsThatCanBeClosed( element, openElements );\n\t\tcurrentOffset = closeElements( elementsToClose, currentOffset );\n\t\tpullAll( openElements, elementsToClose );\n\n\t\t// Comments are self-closing formatting elements that are completely ignored in rendering.\n\t\tif ( element.type === \"#comment\" ) {\n\t\t\tcurrentOffset += computeCommentStartEndTextIndices( element, currentOffset );\n\t\t\treturn;\n\t\t}\n\n\t\tcurrentOffset = computeElementStartTextIndex( element, currentOffset );\n\n\t\t// If there is an endTag, the element should be closed in one of the next iterations of the loop.\n\t\tif (  element.sourceCodeLocation.endTag ) {\n\t\t\topenElements.push( element );\n\t\t}\n\n\t\t/*\n\t\t  If this element is an ignored element its contents are not in the text,\n\t\t  so its content should be added to the respective formatting element instead,\n\t\t  and the current offset should be updated.\n\t\t */\n\t\tif ( ignoredHtmlElements.includes( element.type ) ) {\n\t\t\tcurrentOffset = handleIgnoredContent( element, currentOffset );\n\t\t}\n\t} );\n\t// Close all remaining elements.\n\tcloseElements( openElements, currentOffset );\n};\n\nexport default calculateTextIndices;\n"],"mappings":";;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AAEA,IAAAC,cAAA,GAAAD,OAAA;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAME,uBAAuB,GAAG,SAAAA,CAAUC,cAAc,EAAEC,YAAY,EAAG;EACxE,OAAOA,YAAY,CAACC,MAAM,CAAEC,EAAE,IAAI;IACjC,MAAMC,MAAM,GAAGD,EAAE,CAACE,kBAAkB,CAACD,MAAM;IAC3C,OAAOA,MAAM,CAACE,SAAS,IAAIN,cAAc,CAACK,kBAAkB,CAACE,WAAW;EACzE,CAAE,CAAC;AACJ,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMC,aAAa,GAAG,SAAAA,CAAUC,eAAe,EAAEC,aAAa,EAAG;EAChE;EACAD,eAAe,CAACE,IAAI,CAAE,CAAEC,CAAC,EAAEC,CAAC,KAAMD,CAAC,CAACP,kBAAkB,CAACD,MAAM,CAACE,SAAS,GAAGO,CAAC,CAACR,kBAAkB,CAACD,MAAM,CAACE,SAAU,CAAC;EAEjHG,eAAe,CAACK,OAAO,CAAEC,cAAc,IAAI;IAC1C,MAAMX,MAAM,GAAGW,cAAc,CAACV,kBAAkB,CAACD,MAAM;IACvD;IACAW,cAAc,CAACC,YAAY,GAAGZ,MAAM,CAACG,WAAW,GAAGG,aAAa;IAChE;AACF;AACA;IACE,MAAMO,YAAY,GAAGb,MAAM,CAACE,SAAS,GAAGF,MAAM,CAACG,WAAW;IAC1DG,aAAa,IAAIO,YAAY;EAC9B,CAAE,CAAC;EAEH,OAAOP,aAAa;AACrB,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMQ,oBAAoB,GAAG,SAAAA,CAAUC,OAAO,EAAET,aAAa,EAAG;EAC/D;EACAS,OAAO,CAACH,YAAY,GAAGG,OAAO,CAACC,cAAc;;EAE7C;EACA,MAAMC,GAAG,GAAGF,OAAO,CAACd,kBAAkB,CAACD,MAAM,GAAGe,OAAO,CAACd,kBAAkB,CAACD,MAAM,CAACG,WAAW,GAAGY,OAAO,CAACd,kBAAkB,CAACC,SAAS;EACpI,MAAMgB,KAAK,GAAGH,OAAO,CAACd,kBAAkB,CAACkB,QAAQ,GAAGJ,OAAO,CAACd,kBAAkB,CAACkB,QAAQ,CAACjB,SAAS,GAAGa,OAAO,CAACd,kBAAkB,CAACE,WAAW;EAE1IG,aAAa,IAAIW,GAAG,GAAGC,KAAK;EAE5B,OAAOZ,aAAa;AACrB,CAAC;;AAGD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMc,iCAAiC,GAAG,SAAAA,CAAUL,OAAO,EAAET,aAAa,EAAG;EAC5ES,OAAO,CAACC,cAAc,GAAGD,OAAO,CAACd,kBAAkB,CAACE,WAAW,GAAGG,aAAa;EAC/ES,OAAO,CAACH,YAAY,GAAGG,OAAO,CAACC,cAAc;EAE7C,OAAOD,OAAO,CAACd,kBAAkB,CAACC,SAAS,GAAGa,OAAO,CAACd,kBAAkB,CAACE,WAAW;AACrF,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMkB,4BAA4B,GAAG,SAAAA,CAAUN,OAAO,EAAET,aAAa,EAAG;EACvE,MAAMa,QAAQ,GAAGJ,OAAO,CAACd,kBAAkB,CAACkB,QAAQ;;EAEpD;EACA,MAAMG,cAAc,GAAGH,QAAQ,CAACjB,SAAS,GAAGiB,QAAQ,CAAChB,WAAW;EAEhEG,aAAa,IAAIgB,cAAc;;EAE/B;EACAP,OAAO,CAACC,cAAc,GAAGG,QAAQ,CAACjB,SAAS,GAAGI,aAAa;;EAE3D;AACD;AACA;AACA;EACC,IAAK,CAAES,OAAO,CAACd,kBAAkB,CAACD,MAAM,EAAG;IAC1Ce,OAAO,CAACH,YAAY,GAAGG,OAAO,CAACC,cAAc;EAC9C;EAEA,OAAOV,aAAa;AACrB,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMiB,oBAAoB,GAAG,SAAAA,CAAUC,IAAI,EAAG;EAC7C,IAAK,CAAEA,IAAI,CAACC,aAAa,CAACC,UAAU,IAAIF,IAAI,CAACC,aAAa,CAACC,UAAU,CAACC,MAAM,KAAK,CAAC,EAAG;IACpF;EACD;EAEA,MAAM9B,YAAY,GAAG,EAAE;;EAEvB;AACD;AACA;AACA;EACC,IAAIS,aAAa,GAAGkB,IAAI,CAACvB,kBAAkB,CAACkB,QAAQ,GAAGK,IAAI,CAACvB,kBAAkB,CAACkB,QAAQ,CAACjB,SAAS,GAAGsB,IAAI,CAACvB,kBAAkB,CAACE,WAAW;EAEvIqB,IAAI,CAACC,aAAa,CAACC,UAAU,CAAChB,OAAO,CAAEK,OAAO,IAAI;IACjD;IACA,MAAMV,eAAe,GAAGV,uBAAuB,CAAEoB,OAAO,EAAElB,YAAa,CAAC;IACxES,aAAa,GAAGF,aAAa,CAAEC,eAAe,EAAEC,aAAc,CAAC;IAC/D,IAAAsB,eAAO,EAAE/B,YAAY,EAAEQ,eAAgB,CAAC;;IAExC;IACA,IAAKU,OAAO,CAACc,IAAI,KAAK,UAAU,EAAG;MAClCvB,aAAa,IAAIc,iCAAiC,CAAEL,OAAO,EAAET,aAAc,CAAC;MAC5E;IACD;IAEAA,aAAa,GAAGe,4BAA4B,CAAEN,OAAO,EAAET,aAAc,CAAC;;IAEtE;IACA,IAAMS,OAAO,CAACd,kBAAkB,CAACD,MAAM,EAAG;MACzCH,YAAY,CAACiC,IAAI,CAAEf,OAAQ,CAAC;IAC7B;;IAEA;AACF;AACA;AACA;AACA;IACE,IAAKgB,kCAAmB,CAACC,QAAQ,CAAEjB,OAAO,CAACc,IAAK,CAAC,EAAG;MACnDvB,aAAa,GAAGQ,oBAAoB,CAAEC,OAAO,EAAET,aAAc,CAAC;IAC/D;EACD,CAAE,CAAC;EACH;EACAF,aAAa,CAAEP,YAAY,EAAES,aAAc,CAAC;AAC7C,CAAC;AAAC,IAAA2B,QAAA,GAAAC,OAAA,CAAAC,OAAA,GAEaZ,oBAAoB","ignoreList":[]}

package/build/parsedPaper/build/tree/cleanup/getElementContent.js

@@ -0,0 +1,27 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+/**
+ * Gets the content of an element (the part _between_ the opening and closing tag) from the HTML source code.
+ *
+ * @param {module:parsedPaper/structure.Node|module:parsedPaper/structure.FormattingElement} element The element to parse the contents of
+ * @param {string} html                                                                The source code to parse the contents from
+ *
+ * @returns {string} The element's contents.
+ *
+ * @private
+ */
+const getElementContent = function (element, html) {
+  const location = element.location;
+  if (location) {
+    const start = location.startTag ? location.startTag.endOffset : location.startOffset;
+    const end = location.endTag ? location.endTag.startOffset : location.endOffset;
+    return html.slice(start, end);
+  }
+  return "";
+};
+var _default = exports.default = getElementContent;
+//# sourceMappingURL=getElementContent.js.map

package/build/parsedPaper/build/tree/cleanup/getElementContent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getElementContent.js","names":["getElementContent","element","html","location","start","startTag","endOffset","startOffset","end","endTag","slice","_default","exports","default"],"sources":["../../../../../src/parsedPaper/build/tree/cleanup/getElementContent.js"],"sourcesContent":["/**\n * Gets the content of an element (the part _between_ the opening and closing tag) from the HTML source code.\n *\n * @param {module:parsedPaper/structure.Node|module:parsedPaper/structure.FormattingElement} element The element to parse the contents of\n * @param {string} html                                                                The source code to parse the contents from\n *\n * @returns {string} The element's contents.\n *\n * @private\n */\nconst getElementContent = function( element, html ) {\n\tconst location = element.location;\n\tif ( location ) {\n\t\tconst start = location.startTag ? location.startTag.endOffset : location.startOffset;\n\t\tconst end = location.endTag ? location.endTag.startOffset : location.endOffset;\n\t\treturn html.slice( start, end );\n\t}\n\treturn \"\";\n};\n\nexport default getElementContent;\n"],"mappings":";;;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMA,iBAAiB,GAAG,SAAAA,CAAUC,OAAO,EAAEC,IAAI,EAAG;EACnD,MAAMC,QAAQ,GAAGF,OAAO,CAACE,QAAQ;EACjC,IAAKA,QAAQ,EAAG;IACf,MAAMC,KAAK,GAAGD,QAAQ,CAACE,QAAQ,GAAGF,QAAQ,CAACE,QAAQ,CAACC,SAAS,GAAGH,QAAQ,CAACI,WAAW;IACpF,MAAMC,GAAG,GAAGL,QAAQ,CAACM,MAAM,GAAGN,QAAQ,CAACM,MAAM,CAACF,WAAW,GAAGJ,QAAQ,CAACG,SAAS;IAC9E,OAAOJ,IAAI,CAACQ,KAAK,CAAEN,KAAK,EAAEI,GAAI,CAAC;EAChC;EACA,OAAO,EAAE;AACV,CAAC;AAAC,IAAAG,QAAA,GAAAC,OAAA,CAAAC,OAAA,GAEab,iBAAiB","ignoreList":[]}

package/build/parsedPaper/build/tree/cleanup/postParsing.js

@@ -0,0 +1,43 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _tree = require("../../../structure/tree");
+var _calculateTextIndices = _interopRequireDefault(require("./calculateTextIndices"));
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Cleans up a node in the tree.
+ *
+ * @param {module:parsedPaper/structure.Node} node The node that needs to be cleaned.
+ *
+ * @returns {module:parsedPaper/structure.Node} The cleaned up node.
+ *
+ * @private
+ */
+const cleanUpNode = function (node) {
+  // Clean up formatting elements in headings and paragraphs.
+  if (node instanceof _tree.LeafNode) {
+    // Start and end position in leaf node's (header's or paragraph's) text without formatting.
+    (0, _calculateTextIndices.default)(node);
+  }
+  return node;
+};
+
+/**
+ * Cleans up the given tree after parsing of the HTML source code
+ * by setting the start and end index of each formatting element in a leaf node's text.
+ *
+ * @param {module:parsedPaper/structure.Node} tree The tree structure to be cleaned.
+ *
+ * @returns {module:parsedPaper/structure.Node} The cleaned up tree.
+ *
+ * @private
+ */
+const cleanUpTree = function (tree) {
+  tree.map(node => cleanUpNode(node));
+  return tree;
+};
+var _default = exports.default = cleanUpTree;
+//# sourceMappingURL=postParsing.js.map

package/build/parsedPaper/build/tree/cleanup/postParsing.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"postParsing.js","names":["_tree","require","_calculateTextIndices","_interopRequireDefault","e","__esModule","default","cleanUpNode","node","LeafNode","calculateTextIndices","cleanUpTree","tree","map","_default","exports"],"sources":["../../../../../src/parsedPaper/build/tree/cleanup/postParsing.js"],"sourcesContent":["import { LeafNode } from \"../../../structure/tree\";\nimport calculateTextIndices from \"./calculateTextIndices\";\n\n/**\n * Cleans up a node in the tree.\n *\n * @param {module:parsedPaper/structure.Node} node The node that needs to be cleaned.\n *\n * @returns {module:parsedPaper/structure.Node} The cleaned up node.\n *\n * @private\n */\nconst cleanUpNode = function( node ) {\n\t// Clean up formatting elements in headings and paragraphs.\n\tif ( node instanceof LeafNode ) {\n\t\t// Start and end position in leaf node's (header's or paragraph's) text without formatting.\n\t\tcalculateTextIndices( node );\n\t}\n\treturn node;\n};\n\n/**\n * Cleans up the given tree after parsing of the HTML source code\n * by setting the start and end index of each formatting element in a leaf node's text.\n *\n * @param {module:parsedPaper/structure.Node} tree The tree structure to be cleaned.\n *\n * @returns {module:parsedPaper/structure.Node} The cleaned up tree.\n *\n * @private\n */\nconst cleanUpTree = function( tree ) {\n\ttree.map( node => cleanUpNode( node ) );\n\treturn tree;\n};\n\nexport default cleanUpTree;\n"],"mappings":";;;;;;AAAA,IAAAA,KAAA,GAAAC,OAAA;AACA,IAAAC,qBAAA,GAAAC,sBAAA,CAAAF,OAAA;AAA0D,SAAAE,uBAAAC,CAAA,WAAAA,CAAA,IAAAA,CAAA,CAAAC,UAAA,GAAAD,CAAA,KAAAE,OAAA,EAAAF,CAAA;AAE1D;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMG,WAAW,GAAG,SAAAA,CAAUC,IAAI,EAAG;EACpC;EACA,IAAKA,IAAI,YAAYC,cAAQ,EAAG;IAC/B;IACA,IAAAC,6BAAoB,EAAEF,IAAK,CAAC;EAC7B;EACA,OAAOA,IAAI;AACZ,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMG,WAAW,GAAG,SAAAA,CAAUC,IAAI,EAAG;EACpCA,IAAI,CAACC,GAAG,CAAEL,IAAI,IAAID,WAAW,CAAEC,IAAK,CAAE,CAAC;EACvC,OAAOI,IAAI;AACZ,CAAC;AAAC,IAAAE,QAAA,GAAAC,OAAA,CAAAT,OAAA,GAEaK,WAAW","ignoreList":[]}