axyseo 2.0.0-alpha.1

package/build/parse/build/private/filterHelpers.js

@@ -0,0 +1,52 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.elementHasClass = elementHasClass;
+exports.elementHasID = elementHasID;
+exports.elementHasName = elementHasName;
+/**
+ * Filter helpers for the html tree filter.
+ * All helpers are functions that return a callback with the element as the only argument.
+ */
+
+/**
+ * Creates a callback that checks if an element has a specific name.
+ *
+ * @param {string} name The name to check.
+ *
+ * @returns {function(*)} A function that returns true if the element has a specific name.
+ */
+function elementHasName(name) {
+  return element => {
+    return element.name === name;
+  };
+}
+
+/**
+ * Creates a callback that checks the class of an element.
+ *
+ * @param {string} className The classname to filter out.
+ *
+ * @returns {function(*): boolean} A function that returns true if a Node has a certain class.
+ */
+function elementHasClass(className) {
+  return blockElement => {
+    return !!blockElement.attributes.class && blockElement.attributes.class.has(className);
+  };
+}
+
+/**
+ * Creates a callback that checks if an element has a certain ID.
+ *
+ * @param {string} id The ID we want to check against.
+ *
+ * @returns {function(*): boolean}  A function that returns true if an element has a certain ID.
+ */
+function elementHasID(id) {
+  return element => {
+    return element.attributes.id === id;
+  };
+}
+//# sourceMappingURL=filterHelpers.js.map

package/build/parse/build/private/filterHelpers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"filterHelpers.js","names":["elementHasName","name","element","elementHasClass","className","blockElement","attributes","class","has","elementHasID","id"],"sources":["../../../../src/parse/build/private/filterHelpers.js"],"sourcesContent":["/**\n * Filter helpers for the html tree filter.\n * All helpers are functions that return a callback with the element as the only argument.\n */\n\n\n/**\n * Creates a callback that checks if an element has a specific name.\n *\n * @param {string} name The name to check.\n *\n * @returns {function(*)} A function that returns true if the element has a specific name.\n */\nexport function elementHasName( name ) {\n\treturn ( element ) => {\n\t\treturn element.name === name;\n\t};\n}\n\n/**\n * Creates a callback that checks the class of an element.\n *\n * @param {string} className The classname to filter out.\n *\n * @returns {function(*): boolean} A function that returns true if a Node has a certain class.\n */\nexport function elementHasClass( className ) {\n\treturn ( blockElement ) => {\n\t\treturn !! blockElement.attributes.class && blockElement.attributes.class.has( className );\n\t};\n}\n\n/**\n * Creates a callback that checks if an element has a certain ID.\n *\n * @param {string} id The ID we want to check against.\n *\n * @returns {function(*): boolean}  A function that returns true if an element has a certain ID.\n */\nexport function elementHasID( id ) {\n\treturn ( element ) => {\n\t\treturn element.attributes.id === id;\n\t};\n}\n"],"mappings":";;;;;;;;AAAA;AACA;AACA;AACA;;AAGA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,SAASA,cAAcA,CAAEC,IAAI,EAAG;EACtC,OAASC,OAAO,IAAM;IACrB,OAAOA,OAAO,CAACD,IAAI,KAAKA,IAAI;EAC7B,CAAC;AACF;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,SAASE,eAAeA,CAAEC,SAAS,EAAG;EAC5C,OAASC,YAAY,IAAM;IAC1B,OAAO,CAAC,CAAEA,YAAY,CAACC,UAAU,CAACC,KAAK,IAAIF,YAAY,CAACC,UAAU,CAACC,KAAK,CAACC,GAAG,CAAEJ,SAAU,CAAC;EAC1F,CAAC;AACF;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,SAASK,YAAYA,CAAEC,EAAE,EAAG;EAClC,OAASR,OAAO,IAAM;IACrB,OAAOA,OAAO,CAACI,UAAU,CAACI,EAAE,KAAKA,EAAE;EACpC,CAAC;AACF","ignoreList":[]}

package/build/parse/build/private/filterTree.js

@@ -0,0 +1,47 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = filterTree;
+var _structure = require("../../structure");
+/**
+ * Checks if a node should be kept or discarded.
+ * @param {Node} 		node 		A node.
+ * @param {Function[]} 	filters 	An array of filter callbacks.
+ * @returns {boolean} True if the node should be kept, false if the node should be discarded.
+ */
+function shouldRemoveNode(node, filters) {
+  // Always keep text nodes.
+  if (node.name === "#text") {
+    return false;
+  }
+
+  // If any of the filters returns true, the node will be discarded.
+  return filters.some(filter => filter(node));
+}
+
+/**
+ * A recursive function that removes all nodes based on an array of filters.
+ * @param {Node} 		node 		A node. (Could be the entire tree.)
+ * @param {Function[]} 	filters 	An array of callbacks. If a callback returns true, the node is discarded.
+ * @returns {Node|undefined} A Node with all undesired subtrees removed.
+ */
+function filterTree(node, filters) {
+  // Returns undefined when the node should be disregarded.
+  if (shouldRemoveNode(node, filters)) {
+    return;
+  }
+
+  // Recursively filters the node's children.
+  if (node.childNodes) {
+    node.childNodes = node.childNodes.filter(childNode => filterTree(childNode, filters));
+
+    // Drops implicit paragraphs if all their child nodes have been removed.
+    if (node.childNodes.length === 0 && node instanceof _structure.Paragraph && node.isImplicit) {
+      return;
+    }
+  }
+  return node;
+}
+//# sourceMappingURL=filterTree.js.map

package/build/parse/build/private/filterTree.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"filterTree.js","names":["_structure","require","shouldRemoveNode","node","filters","name","some","filter","filterTree","childNodes","childNode","length","Paragraph","isImplicit"],"sources":["../../../../src/parse/build/private/filterTree.js"],"sourcesContent":["import { Paragraph } from \"../../structure\";\n\n/**\n * Checks if a node should be kept or discarded.\n * @param {Node} \t\tnode \t\tA node.\n * @param {Function[]} \tfilters \tAn array of filter callbacks.\n * @returns {boolean} True if the node should be kept, false if the node should be discarded.\n */\nfunction shouldRemoveNode( node, filters ) {\n\t// Always keep text nodes.\n\tif ( node.name === \"#text\" ) {\n\t\treturn false;\n\t}\n\n\t// If any of the filters returns true, the node will be discarded.\n\treturn filters.some( filter => filter( node ) );\n}\n\n/**\n * A recursive function that removes all nodes based on an array of filters.\n * @param {Node} \t\tnode \t\tA node. (Could be the entire tree.)\n * @param {Function[]} \tfilters \tAn array of callbacks. If a callback returns true, the node is discarded.\n * @returns {Node|undefined} A Node with all undesired subtrees removed.\n */\nexport default function filterTree( node, filters ) {\n\t// Returns undefined when the node should be disregarded.\n\tif ( shouldRemoveNode( node, filters ) ) {\n\t\treturn;\n\t}\n\n\t// Recursively filters the node's children.\n\tif ( node.childNodes ) {\n\t\tnode.childNodes = node.childNodes.filter( childNode => filterTree( childNode, filters ) );\n\n\t\t// Drops implicit paragraphs if all their child nodes have been removed.\n\t\tif ( node.childNodes.length === 0 && node instanceof Paragraph && node.isImplicit ) {\n\t\t\treturn;\n\t\t}\n\t}\n\n\treturn node;\n}\n"],"mappings":";;;;;;AAAA,IAAAA,UAAA,GAAAC,OAAA;AAEA;AACA;AACA;AACA;AACA;AACA;AACA,SAASC,gBAAgBA,CAAEC,IAAI,EAAEC,OAAO,EAAG;EAC1C;EACA,IAAKD,IAAI,CAACE,IAAI,KAAK,OAAO,EAAG;IAC5B,OAAO,KAAK;EACb;;EAEA;EACA,OAAOD,OAAO,CAACE,IAAI,CAAEC,MAAM,IAAIA,MAAM,CAAEJ,IAAK,CAAE,CAAC;AAChD;;AAEA;AACA;AACA;AACA;AACA;AACA;AACe,SAASK,UAAUA,CAAEL,IAAI,EAAEC,OAAO,EAAG;EACnD;EACA,IAAKF,gBAAgB,CAAEC,IAAI,EAAEC,OAAQ,CAAC,EAAG;IACxC;EACD;;EAEA;EACA,IAAKD,IAAI,CAACM,UAAU,EAAG;IACtBN,IAAI,CAACM,UAAU,GAAGN,IAAI,CAACM,UAAU,CAACF,MAAM,CAAEG,SAAS,IAAIF,UAAU,CAAEE,SAAS,EAAEN,OAAQ,CAAE,CAAC;;IAEzF;IACA,IAAKD,IAAI,CAACM,UAAU,CAACE,MAAM,KAAK,CAAC,IAAIR,IAAI,YAAYS,oBAAS,IAAIT,IAAI,CAACU,UAAU,EAAG;MACnF;IACD;EACD;EAEA,OAAOV,IAAI;AACZ","ignoreList":[]}

package/build/parse/build/private/getTextElementPositions.js

@@ -0,0 +1,184 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = getTextElementPositions;
+var _structure = require("../../structure");
+var _alwaysFilterElements = require("./alwaysFilterElements");
+/**
+ * Gets the start and end positions of all descendant nodes' tags and stores them in an array.
+ * Each object in the array represents an opening or closing tag for nodes that have tags, or the full node for nodes
+ * without tags (such as 'comment' nodes).
+ * The startOffset and endOffset properties of the objects correspond to the start and end positions of the tags, or the
+ * start and end positions of the full node for nodes without tags.
+ * Extracting this data into a separate array makes it easier to work with it (e.g. sort it and loop over it).
+ *
+ * @param {Node[]} descendantNodes	The descendant nodes to get positions from.
+ *
+ * @returns {SourceCodeRange[]}	An array of the locations of each start and end tag in the source code, or the start and
+ * 								end locations of the full node for nodes without tags.
+ *
+ */
+function getDescendantPositions(descendantNodes) {
+  const descendantTagPositions = [];
+  descendantNodes.forEach(node => {
+    /*
+     * For nodes whose content we don't want to analyze, we should add the length of the full node to the array, to
+     * still take into account the whole element's length when calculating positions.
+     */
+    if (_alwaysFilterElements.canBeChildOfParagraph.includes(node.name)) {
+      descendantTagPositions.push(node.sourceCodeLocation);
+    } else {
+      if (node.sourceCodeLocation.startTag) {
+        const startRange = {
+          startOffset: node.sourceCodeLocation.startTag.startOffset,
+          endOffset: node.sourceCodeLocation.startTag.endOffset
+        };
+        /*
+         * Here, we need to account for the fact that earlier (in innerText.js), we treated a <br> as a newline character.
+         * Therefore, we need to subtract 1 from the endOffset to not count it twice.
+         */
+        if (node.name === "br") {
+          startRange.endOffset = startRange.endOffset - 1;
+        }
+        descendantTagPositions.push(startRange);
+      }
+      /*
+       * Check whether node has an end tag before adding it to the array.
+       * Some nodes, such as the 'img' node, only have a start tag.
+       */
+      if (node.sourceCodeLocation.endTag) {
+        descendantTagPositions.push(node.sourceCodeLocation.endTag);
+      }
+    }
+  });
+  // Sort the tag position objects by the start tag position in ascending order.
+  descendantTagPositions.sort((a, b) => a.startOffset - b.startOffset);
+  return descendantTagPositions;
+}
+
+/**
+ * Adjusts the end position of the text element to account for descendant node tags that overlap with the text element.
+ * We want to add the length of tags that are directly preceding the start of the text, or that are between the start
+ * and end of the text, to the end position of the text element.
+ * For example, if the text is "<span><em>Hello</em>, world!</span>", the length of all tags except for the closing
+ * </span> tag should be added to the end position of the sentence.
+ *
+ * @param {SourceCodeRange[]}	descendantTagPositions	The positions of the descendant nodes' tags.
+ * @param {number}				textElementStart		The start position of a text element.
+ * @param {number}				textElementEnd			The end position of a text element.
+ *
+ * @returns {number}	The adjusted end position of the text element.
+ */
+function adjustElementEnd(descendantTagPositions, textElementStart, textElementEnd) {
+  /*
+   * If the start position of a descendant's node tag is between the start and end position of the text element, or is
+   * the same as the start position of the text element, add the tag's length to the end position of the text element.
+   */
+  descendantTagPositions.forEach(position => {
+    if (position.startOffset >= textElementStart && position.startOffset < textElementEnd) {
+      textElementEnd += position.endOffset - position.startOffset;
+    }
+  });
+  return textElementEnd;
+}
+
+/**
+ * Adjusts the start position of the text element to exclude descendant node tags at the beginning of the text element.
+ * For example, the start position of the text element `<span>Hello!</span>`
+ * should be adjusted to start at the `H`, not at the start tag of the opening `<span>` tag.
+ * @param {SourceCodeRange[]} descendantTagPositions The positions of the descendant nodes' tags.
+ * @param {Number} textElementStart The start position of a text element.
+ *
+ * @returns {Number} The adjusted start position of the text element.
+ */
+function adjustTextElementStart(descendantTagPositions, textElementStart) {
+  /*
+   * If the start position of a descendant's node tag is between the start and end position of the text element, or is
+   * the same as the start/end position of the text element, add the tag's length to the end position of the text element.
+   */
+  descendantTagPositions.forEach(position => {
+    if (position.startOffset === textElementStart) {
+      textElementStart += position.endOffset - position.startOffset;
+    }
+  });
+  return textElementStart;
+}
+
+/**
+ * Retrieves the initial start offset for the first text element.
+ * Normally, that is the end offset of the start tag of the parent node.
+ * In case of implicit paragraphs, we use the start offset of the node as a whole.
+ *
+ * @param {Paragraph|Heading} node The node to retrieve the start position from.
+ * @returns {number} The start offset for the first text element.
+ */
+const getTextElementStart = node => {
+  return node instanceof _structure.Paragraph && node.isImplicit ? node.sourceCodeLocation.startOffset : node.sourceCodeLocation.startTag.endOffset;
+};
+
+/**
+ * Gets the start and end positions of text elements (sentences or tokens) in the source code.
+ *
+ * @param {Paragraph|Heading} 		node  			The paragraph or heading node.
+ * @param {Sentence[]|Token[]} 		textElements 	The sentences or tokens in the node.
+ * @param {number} 					startOffset 	The start position of the node in the source code.
+ * Defaults to -1 which signals that it should not be used.
+ *
+ * @returns {Sentence[]|Token[]} The sentences or tokens, with their positions in the source code.
+ */
+function getTextElementPositions(node, textElements, startOffset = -1) {
+  // We cannot calculate positions if there are no text elements, or if we don't know the node's source code location.
+  if (textElements.length === 0 || !node.sourceCodeLocation) {
+    return textElements;
+  }
+
+  /*
+   * Set the (initial) start offset of the first text element.
+   * If the start offset has been provided (in the case of Tokens), use that.
+   * Otherwise, determine the start offset from the parent Paragraph or Heading node.
+   */
+  let textElementStart = startOffset >= 0 ? startOffset : getTextElementStart(node);
+  let textElementEnd;
+  let descendantTagPositions = [];
+
+  /*
+   * Check if the node has any descendant nodes that have a sourceCodeLocation property (all nodes other than Text nodes
+   * should have this property). If such nodes exist, store the positions of each node's opening and closing tags in
+   * an array. These positions will have to be taken into account when calculating the position of the text elements.
+   */
+  if (node.findAll) {
+    const descendantNodes = node.findAll(descendantNode => descendantNode.sourceCodeLocation, true);
+    if (descendantNodes.length > 0) {
+      descendantTagPositions = getDescendantPositions(descendantNodes);
+    }
+  }
+  textElements.forEach(textElement => {
+    // Set the end position to the start position + the length of the textElement.
+    textElementEnd = textElementStart + textElement.text.length;
+
+    /*
+     * If there are descendant tags, possibly adjust the textElementEnd and textElementStart.
+     * The textElementEnd should be adjusted to include the length of any descendant tags until the end of the text, in
+     * addition to the length of the text itself. Descendant tags AFTER the end of the text should not be included.
+     * The textElementStart should be adjusted so that it is where the actual text starts, not including any descendant
+     * tags preceding the text.
+     */
+    if (descendantTagPositions.length > 0) {
+      textElementEnd = adjustElementEnd(descendantTagPositions, textElementStart, textElementEnd);
+      textElementStart = adjustTextElementStart(descendantTagPositions, textElementStart);
+    }
+
+    // Add the start and end positions to the textElement object.
+    textElement.sourceCodeRange = {
+      startOffset: textElementStart,
+      endOffset: textElementEnd
+    };
+
+    // Start position of the next textElement is the end position of current textElement.
+    textElementStart = textElementEnd;
+  });
+  return textElements;
+}
+//# sourceMappingURL=getTextElementPositions.js.map

package/build/parse/build/private/getTextElementPositions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getTextElementPositions.js","names":["_structure","require","_alwaysFilterElements","getDescendantPositions","descendantNodes","descendantTagPositions","forEach","node","excludeFromAnalysis","includes","name","push","sourceCodeLocation","startTag","startRange","startOffset","endOffset","endTag","sort","a","b","adjustElementEnd","textElementStart","textElementEnd","position","adjustTextElementStart","getTextElementStart","Paragraph","isImplicit","getTextElementPositions","textElements","length","findAll","descendantNode","textElement","text","sourceCodeRange"],"sources":["../../../../src/parse/build/private/getTextElementPositions.js"],"sourcesContent":["import { Paragraph } from \"../../structure\";\nimport { canBeChildOfParagraph as excludeFromAnalysis } from \"./alwaysFilterElements\";\n\n/**\n * Gets the start and end positions of all descendant nodes' tags and stores them in an array.\n * Each object in the array represents an opening or closing tag for nodes that have tags, or the full node for nodes\n * without tags (such as 'comment' nodes).\n * The startOffset and endOffset properties of the objects correspond to the start and end positions of the tags, or the\n * start and end positions of the full node for nodes without tags.\n * Extracting this data into a separate array makes it easier to work with it (e.g. sort it and loop over it).\n *\n * @param {Node[]} descendantNodes\tThe descendant nodes to get positions from.\n *\n * @returns {SourceCodeRange[]}\tAn array of the locations of each start and end tag in the source code, or the start and\n * \t\t\t\t\t\t\t\tend locations of the full node for nodes without tags.\n *\n */\nfunction getDescendantPositions( descendantNodes ) {\n\tconst descendantTagPositions = [];\n\tdescendantNodes.forEach( ( node ) => {\n\t\t/*\n\t\t * For nodes whose content we don't want to analyze, we should add the length of the full node to the array, to\n\t\t * still take into account the whole element's length when calculating positions.\n\t\t */\n\t\tif ( excludeFromAnalysis.includes( node.name ) ) {\n\t\t\tdescendantTagPositions.push( node.sourceCodeLocation );\n\t\t} else {\n\t\t\tif ( node.sourceCodeLocation.startTag ) {\n\t\t\t\tconst startRange = {\n\t\t\t\t\tstartOffset: node.sourceCodeLocation.startTag.startOffset,\n\t\t\t\t\tendOffset: node.sourceCodeLocation.startTag.endOffset,\n\t\t\t\t};\n\t\t\t\t/*\n\t\t\t\t * Here, we need to account for the fact that earlier (in innerText.js), we treated a <br> as a newline character.\n\t\t\t\t * Therefore, we need to subtract 1 from the endOffset to not count it twice.\n\t\t\t\t */\n\t\t\t\tif ( node.name === \"br\" ) {\n\t\t\t\t\tstartRange.endOffset = startRange.endOffset - 1;\n\t\t\t\t}\n\t\t\t\tdescendantTagPositions.push( startRange );\n\t\t\t}\n\t\t\t/*\n\t\t\t * Check whether node has an end tag before adding it to the array.\n\t\t\t * Some nodes, such as the 'img' node, only have a start tag.\n\t\t\t */\n\t\t\tif ( node.sourceCodeLocation.endTag ) {\n\t\t\t\tdescendantTagPositions.push( node.sourceCodeLocation.endTag );\n\t\t\t}\n\t\t}\n\t} );\n\t// Sort the tag position objects by the start tag position in ascending order.\n\tdescendantTagPositions.sort( ( a, b ) => a.startOffset - b.startOffset );\n\n\treturn descendantTagPositions;\n}\n\n/**\n * Adjusts the end position of the text element to account for descendant node tags that overlap with the text element.\n * We want to add the length of tags that are directly preceding the start of the text, or that are between the start\n * and end of the text, to the end position of the text element.\n * For example, if the text is \"<span><em>Hello</em>, world!</span>\", the length of all tags except for the closing\n * </span> tag should be added to the end position of the sentence.\n *\n * @param {SourceCodeRange[]}\tdescendantTagPositions\tThe positions of the descendant nodes' tags.\n * @param {number}\t\t\t\ttextElementStart\t\tThe start position of a text element.\n * @param {number}\t\t\t\ttextElementEnd\t\t\tThe end position of a text element.\n *\n * @returns {number}\tThe adjusted end position of the text element.\n */\nfunction adjustElementEnd( descendantTagPositions, textElementStart, textElementEnd ) {\n\t/*\n\t * If the start position of a descendant's node tag is between the start and end position of the text element, or is\n\t * the same as the start position of the text element, add the tag's length to the end position of the text element.\n\t */\n\tdescendantTagPositions.forEach( ( position ) => {\n\t\tif ( position.startOffset >= textElementStart && position.startOffset < textElementEnd ) {\n\t\t\ttextElementEnd += ( position.endOffset - position.startOffset );\n\t\t}\n\t} );\n\n\treturn textElementEnd;\n}\n\n/**\n * Adjusts the start position of the text element to exclude descendant node tags at the beginning of the text element.\n * For example, the start position of the text element `<span>Hello!</span>`\n * should be adjusted to start at the `H`, not at the start tag of the opening `<span>` tag.\n * @param {SourceCodeRange[]} descendantTagPositions The positions of the descendant nodes' tags.\n * @param {Number} textElementStart The start position of a text element.\n *\n * @returns {Number} The adjusted start position of the text element.\n */\nfunction adjustTextElementStart( descendantTagPositions, textElementStart ) {\n\t/*\n\t * If the start position of a descendant's node tag is between the start and end position of the text element, or is\n\t * the same as the start/end position of the text element, add the tag's length to the end position of the text element.\n\t */\n\tdescendantTagPositions.forEach( ( position ) => {\n\t\tif ( position.startOffset === textElementStart ) {\n\t\t\ttextElementStart += ( position.endOffset - position.startOffset );\n\t\t}\n\t} );\n\n\treturn textElementStart;\n}\n\n/**\n * Retrieves the initial start offset for the first text element.\n * Normally, that is the end offset of the start tag of the parent node.\n * In case of implicit paragraphs, we use the start offset of the node as a whole.\n *\n * @param {Paragraph|Heading} node The node to retrieve the start position from.\n * @returns {number} The start offset for the first text element.\n */\nconst getTextElementStart = ( node ) => {\n\treturn node instanceof Paragraph && node.isImplicit\n\t\t? node.sourceCodeLocation.startOffset\n\t\t: node.sourceCodeLocation.startTag.endOffset;\n};\n\n/**\n * Gets the start and end positions of text elements (sentences or tokens) in the source code.\n *\n * @param {Paragraph|Heading} \t\tnode  \t\t\tThe paragraph or heading node.\n * @param {Sentence[]|Token[]} \t\ttextElements \tThe sentences or tokens in the node.\n * @param {number} \t\t\t\t\tstartOffset \tThe start position of the node in the source code.\n * Defaults to -1 which signals that it should not be used.\n *\n * @returns {Sentence[]|Token[]} The sentences or tokens, with their positions in the source code.\n */\nexport default function getTextElementPositions( node, textElements, startOffset = -1 ) {\n\t// We cannot calculate positions if there are no text elements, or if we don't know the node's source code location.\n\tif (  textElements.length === 0 || ! node.sourceCodeLocation ) {\n\t\treturn textElements;\n\t}\n\n\t/*\n\t * Set the (initial) start offset of the first text element.\n\t * If the start offset has been provided (in the case of Tokens), use that.\n\t * Otherwise, determine the start offset from the parent Paragraph or Heading node.\n\t */\n\tlet textElementStart = startOffset >= 0 ? startOffset : getTextElementStart( node );\n\n\tlet textElementEnd;\n\tlet descendantTagPositions = [];\n\n\t/*\n\t * Check if the node has any descendant nodes that have a sourceCodeLocation property (all nodes other than Text nodes\n\t * should have this property). If such nodes exist, store the positions of each node's opening and closing tags in\n\t * an array. These positions will have to be taken into account when calculating the position of the text elements.\n\t */\n\tif ( node.findAll ) {\n\t\tconst descendantNodes = node.findAll( descendantNode => descendantNode.sourceCodeLocation, true );\n\t\tif ( descendantNodes.length > 0 ) {\n\t\t\tdescendantTagPositions = getDescendantPositions( descendantNodes );\n\t\t}\n\t}\n\n\ttextElements.forEach( ( textElement ) => {\n\t\t// Set the end position to the start position + the length of the textElement.\n\t\ttextElementEnd = textElementStart + textElement.text.length;\n\n\t\t/*\n\t\t * If there are descendant tags, possibly adjust the textElementEnd and textElementStart.\n\t\t * The textElementEnd should be adjusted to include the length of any descendant tags until the end of the text, in\n\t\t * addition to the length of the text itself. Descendant tags AFTER the end of the text should not be included.\n\t\t * The textElementStart should be adjusted so that it is where the actual text starts, not including any descendant\n\t\t * tags preceding the text.\n\t\t */\n\t\tif ( descendantTagPositions.length > 0 ) {\n\t\t\ttextElementEnd = adjustElementEnd( descendantTagPositions, textElementStart, textElementEnd );\n\n\t\t\ttextElementStart = adjustTextElementStart( descendantTagPositions, textElementStart );\n\t\t}\n\n\t\t// Add the start and end positions to the textElement object.\n\t\ttextElement.sourceCodeRange = { startOffset: textElementStart, endOffset: textElementEnd };\n\n\t\t// Start position of the next textElement is the end position of current textElement.\n\t\ttextElementStart = textElementEnd;\n\t} );\n\n\treturn textElements;\n}\n"],"mappings":";;;;;;AAAA,IAAAA,UAAA,GAAAC,OAAA;AACA,IAAAC,qBAAA,GAAAD,OAAA;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASE,sBAAsBA,CAAEC,eAAe,EAAG;EAClD,MAAMC,sBAAsB,GAAG,EAAE;EACjCD,eAAe,CAACE,OAAO,CAAIC,IAAI,IAAM;IACpC;AACF;AACA;AACA;IACE,IAAKC,2CAAmB,CAACC,QAAQ,CAAEF,IAAI,CAACG,IAAK,CAAC,EAAG;MAChDL,sBAAsB,CAACM,IAAI,CAAEJ,IAAI,CAACK,kBAAmB,CAAC;IACvD,CAAC,MAAM;MACN,IAAKL,IAAI,CAACK,kBAAkB,CAACC,QAAQ,EAAG;QACvC,MAAMC,UAAU,GAAG;UAClBC,WAAW,EAAER,IAAI,CAACK,kBAAkB,CAACC,QAAQ,CAACE,WAAW;UACzDC,SAAS,EAAET,IAAI,CAACK,kBAAkB,CAACC,QAAQ,CAACG;QAC7C,CAAC;QACD;AACJ;AACA;AACA;QACI,IAAKT,IAAI,CAACG,IAAI,KAAK,IAAI,EAAG;UACzBI,UAAU,CAACE,SAAS,GAAGF,UAAU,CAACE,SAAS,GAAG,CAAC;QAChD;QACAX,sBAAsB,CAACM,IAAI,CAAEG,UAAW,CAAC;MAC1C;MACA;AACH;AACA;AACA;MACG,IAAKP,IAAI,CAACK,kBAAkB,CAACK,MAAM,EAAG;QACrCZ,sBAAsB,CAACM,IAAI,CAAEJ,IAAI,CAACK,kBAAkB,CAACK,MAAO,CAAC;MAC9D;IACD;EACD,CAAE,CAAC;EACH;EACAZ,sBAAsB,CAACa,IAAI,CAAE,CAAEC,CAAC,EAAEC,CAAC,KAAMD,CAAC,CAACJ,WAAW,GAAGK,CAAC,CAACL,WAAY,CAAC;EAExE,OAAOV,sBAAsB;AAC9B;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASgB,gBAAgBA,CAAEhB,sBAAsB,EAAEiB,gBAAgB,EAAEC,cAAc,EAAG;EACrF;AACD;AACA;AACA;EACClB,sBAAsB,CAACC,OAAO,CAAIkB,QAAQ,IAAM;IAC/C,IAAKA,QAAQ,CAACT,WAAW,IAAIO,gBAAgB,IAAIE,QAAQ,CAACT,WAAW,GAAGQ,cAAc,EAAG;MACxFA,cAAc,IAAMC,QAAQ,CAACR,SAAS,GAAGQ,QAAQ,CAACT,WAAa;IAChE;EACD,CAAE,CAAC;EAEH,OAAOQ,cAAc;AACtB;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASE,sBAAsBA,CAAEpB,sBAAsB,EAAEiB,gBAAgB,EAAG;EAC3E;AACD;AACA;AACA;EACCjB,sBAAsB,CAACC,OAAO,CAAIkB,QAAQ,IAAM;IAC/C,IAAKA,QAAQ,CAACT,WAAW,KAAKO,gBAAgB,EAAG;MAChDA,gBAAgB,IAAME,QAAQ,CAACR,SAAS,GAAGQ,QAAQ,CAACT,WAAa;IAClE;EACD,CAAE,CAAC;EAEH,OAAOO,gBAAgB;AACxB;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMI,mBAAmB,GAAKnB,IAAI,IAAM;EACvC,OAAOA,IAAI,YAAYoB,oBAAS,IAAIpB,IAAI,CAACqB,UAAU,GAChDrB,IAAI,CAACK,kBAAkB,CAACG,WAAW,GACnCR,IAAI,CAACK,kBAAkB,CAACC,QAAQ,CAACG,SAAS;AAC9C,CAAC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAASa,uBAAuBA,CAAEtB,IAAI,EAAEuB,YAAY,EAAEf,WAAW,GAAG,CAAC,CAAC,EAAG;EACvF;EACA,IAAMe,YAAY,CAACC,MAAM,KAAK,CAAC,IAAI,CAAExB,IAAI,CAACK,kBAAkB,EAAG;IAC9D,OAAOkB,YAAY;EACpB;;EAEA;AACD;AACA;AACA;AACA;EACC,IAAIR,gBAAgB,GAAGP,WAAW,IAAI,CAAC,GAAGA,WAAW,GAAGW,mBAAmB,CAAEnB,IAAK,CAAC;EAEnF,IAAIgB,cAAc;EAClB,IAAIlB,sBAAsB,GAAG,EAAE;;EAE/B;AACD;AACA;AACA;AACA;EACC,IAAKE,IAAI,CAACyB,OAAO,EAAG;IACnB,MAAM5B,eAAe,GAAGG,IAAI,CAACyB,OAAO,CAAEC,cAAc,IAAIA,cAAc,CAACrB,kBAAkB,EAAE,IAAK,CAAC;IACjG,IAAKR,eAAe,CAAC2B,MAAM,GAAG,CAAC,EAAG;MACjC1B,sBAAsB,GAAGF,sBAAsB,CAAEC,eAAgB,CAAC;IACnE;EACD;EAEA0B,YAAY,CAACxB,OAAO,CAAI4B,WAAW,IAAM;IACxC;IACAX,cAAc,GAAGD,gBAAgB,GAAGY,WAAW,CAACC,IAAI,CAACJ,MAAM;;IAE3D;AACF;AACA;AACA;AACA;AACA;AACA;IACE,IAAK1B,sBAAsB,CAAC0B,MAAM,GAAG,CAAC,EAAG;MACxCR,cAAc,GAAGF,gBAAgB,CAAEhB,sBAAsB,EAAEiB,gBAAgB,EAAEC,cAAe,CAAC;MAE7FD,gBAAgB,GAAGG,sBAAsB,CAAEpB,sBAAsB,EAAEiB,gBAAiB,CAAC;IACtF;;IAEA;IACAY,WAAW,CAACE,eAAe,GAAG;MAAErB,WAAW,EAAEO,gBAAgB;MAAEN,SAAS,EAAEO;IAAe,CAAC;;IAE1F;IACAD,gBAAgB,GAAGC,cAAc;EAClC,CAAE,CAAC;EAEH,OAAOO,YAAY;AACpB","ignoreList":[]}

package/build/parse/build/private/helpers/parseClassAttribute.js

@@ -0,0 +1,15 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = parseClassAttribute;
+/**
+ * When splits a string of html classes into a set html classes.
+ * @param {string} classString The value of the class attribute.
+ * @returns {Set} A set containing all individual values of the class.
+ */
+function parseClassAttribute(classString) {
+  return new Set(classString.split(" "));
+}
+//# sourceMappingURL=parseClassAttribute.js.map

package/build/parse/build/private/helpers/parseClassAttribute.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parseClassAttribute.js","names":["parseClassAttribute","classString","Set","split"],"sources":["../../../../../src/parse/build/private/helpers/parseClassAttribute.js"],"sourcesContent":["\n/**\n * When splits a string of html classes into a set html classes.\n * @param {string} classString The value of the class attribute.\n * @returns {Set} A set containing all individual values of the class.\n */\nexport default function parseClassAttribute( classString ) {\n\treturn new Set( classString.split( \" \" ) );\n}\n"],"mappings":";;;;;;AACA;AACA;AACA;AACA;AACA;AACe,SAASA,mBAAmBA,CAAEC,WAAW,EAAG;EAC1D,OAAO,IAAIC,GAAG,CAAED,WAAW,CAACE,KAAK,CAAE,GAAI,CAAE,CAAC;AAC3C","ignoreList":[]}

package/build/parse/build/private/isPhrasingContent.js

@@ -0,0 +1,30 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+/**
+ * Tag names of HTML elements that are considered phrasing content
+ * in the HTML content model.
+ *
+ * @see https://html.spec.whatwg.org/#phrasing-content
+ *
+ * @type {string[]}
+ */
+const phrasingContentTags = ["b", "big", "i", "small", "tt", "abbr", "acronym", "cite", "code", "dfn", "em", "kbd", "strong", "samp", "time", "var", "a", "bdo", "br", "img", "map", "object", "q", "script", "span", "sub", "sup", "button", "input", "label", "select", "textarea"];
+
+/**
+ * Checks whether a node is considered phrasing content.
+ *
+ * @see https://html.spec.whatwg.org/#phrasing-content
+ *
+ * @param {string} nodeName The name/tag of the node to check if it is phrasing content.
+ *
+ * @returns {boolean} Whether the node is phrasing content.
+ */
+function isPhrasingContent(nodeName) {
+  return phrasingContentTags.includes(nodeName) || nodeName === "#text";
+}
+var _default = exports.default = isPhrasingContent;
+//# sourceMappingURL=isPhrasingContent.js.map

package/build/parse/build/private/isPhrasingContent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"isPhrasingContent.js","names":["phrasingContentTags","isPhrasingContent","nodeName","includes","_default","exports","default"],"sources":["../../../../src/parse/build/private/isPhrasingContent.js"],"sourcesContent":["/**\n * Tag names of HTML elements that are considered phrasing content\n * in the HTML content model.\n *\n * @see https://html.spec.whatwg.org/#phrasing-content\n *\n * @type {string[]}\n */\nconst phrasingContentTags = [\n\t\"b\", \"big\", \"i\", \"small\", \"tt\", \"abbr\", \"acronym\", \"cite\", \"code\", \"dfn\", \"em\", \"kbd\", \"strong\",\n\t\"samp\", \"time\", \"var\", \"a\", \"bdo\", \"br\", \"img\", \"map\", \"object\", \"q\", \"script\", \"span\", \"sub\", \"sup\", \"button\",\n\t\"input\", \"label\", \"select\", \"textarea\",\n];\n\n/**\n * Checks whether a node is considered phrasing content.\n *\n * @see https://html.spec.whatwg.org/#phrasing-content\n *\n * @param {string} nodeName The name/tag of the node to check if it is phrasing content.\n *\n * @returns {boolean} Whether the node is phrasing content.\n */\nfunction isPhrasingContent( nodeName ) {\n\treturn phrasingContentTags.includes( nodeName ) || nodeName === \"#text\";\n}\n\nexport default isPhrasingContent;\n"],"mappings":";;;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMA,mBAAmB,GAAG,CAC3B,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAC/F,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAC9G,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CACtC;;AAED;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASC,iBAAiBA,CAAEC,QAAQ,EAAG;EACtC,OAAOF,mBAAmB,CAACG,QAAQ,CAAED,QAAS,CAAC,IAAIA,QAAQ,KAAK,OAAO;AACxE;AAAC,IAAAE,QAAA,GAAAC,OAAA,CAAAC,OAAA,GAEcL,iBAAiB","ignoreList":[]}

package/build/parse/build/private/parseBlocks.js

@@ -0,0 +1,151 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = _default;
+exports.updateBlocksOffset = updateBlocksOffset;
+var _lodash = require("lodash");
+const blockTokenizer = /<!--\s+wp:([a-z][a-z0-9_-]*\/)?([a-z][a-z0-9_-]*)\s+({(?:(?=([^}]+|}+(?=})|(?!}\s+\/?-->)[^])*)\5|[^]*?)}\s+)?(\/)?-->/g;
+
+/**
+ * Get all the blocks including the inner blocks.
+ *
+ * @param {Paper} paper The paper to get all the blocks from.
+ *
+ * @returns {Object[]} An array of blocks.
+ */
+const getAllBlocks = paper => {
+  const blocks = paper._attributes.wpBlocks;
+  const flattenBlocks = [];
+  if (!(blocks && blocks.length > 0)) {
+    return [];
+  }
+  blocks.forEach(block => {
+    if (block.innerBlocks.length > 0) {
+      const innerBlocks = block.innerBlocks;
+      flattenBlocks.push(block, ...innerBlocks);
+    } else {
+      flattenBlocks.push(block);
+    }
+  });
+  return flattenBlocks;
+};
+
+/**
+ * Gets the offset of each block and set it to the tree.
+ *
+ * @param {Object[]}	blocks An array of blocks.
+ * @param {string}		text The text.
+ *
+ * @returns {void}
+ */
+function updateBlocksOffset(blocks, text) {
+  if (blocks.length === 0) {
+    return;
+  }
+  blocks.forEach((currentBlock, index) => {
+    const matches = blockTokenizer.exec(text);
+    if (currentBlock.name === "core/freeform") {
+      const previousBlock = blocks[index - 1];
+      if (previousBlock) {
+        const previousBlockEndOffset = previousBlock.endOffset;
+        const startedAt = previousBlockEndOffset + 2;
+        currentBlock.startOffset = startedAt;
+        currentBlock.endOffset = startedAt + currentBlock.blockLength;
+        currentBlock.contentOffset = startedAt;
+      } else {
+        currentBlock.startOffset = 0;
+        currentBlock.endOffset = 0 + currentBlock.blockLength;
+        currentBlock.contentOffset = 0;
+      }
+    } else {
+      if (null === matches) {
+        return;
+      }
+      const [match] = matches;
+      const startedAt = matches.index;
+      const length = match.length;
+      currentBlock.startOffset = startedAt;
+      currentBlock.endOffset = startedAt + currentBlock.blockLength;
+      currentBlock.contentOffset = startedAt + length + 1;
+    }
+    if (currentBlock.innerBlocks && currentBlock.innerBlocks.length > 0) {
+      updateBlocksOffset(currentBlock.innerBlocks, text);
+    }
+  });
+}
+
+/**
+ * Gets the client id for each block and set it to the child nodes.
+ * Additionally, when a node has an attribute with 'id' key, also set this id to the first.
+ *
+ * @param {Node}		rootNode The root node.
+ * @param {Object[]}	blocks An array of blocks.
+ * @param {string}		clientId The client id to set.
+ *
+ * @returns {void}
+ */
+function updateClientIdAndAttrIdForSubtree(rootNode, blocks, clientId) {
+  if (!rootNode) {
+    return;
+  }
+  let currentClientId = clientId;
+  if (rootNode.sourceCodeLocation && !(0, _lodash.isUndefined)(rootNode.sourceCodeLocation.startOffset)) {
+    const foundBlock = blocks.find(block => block.contentOffset === rootNode.sourceCodeLocation.startOffset);
+    if (foundBlock) {
+      currentClientId = foundBlock.clientId;
+    }
+  }
+
+  // If the clientId is not undefined, also set this to the root node.
+  if (currentClientId) {
+    rootNode.clientId = currentClientId;
+  }
+
+  // If the node has children, update the clientId for them.
+  (rootNode.childNodes || []).forEach(node => {
+    if (node.attributes && node.attributes.id) {
+      /*
+       * If the node's child has an attribute with 'id' key, also set this id to the first and third child node.
+       * This step is specifically for parsing the Yoast blocks.
+       *
+       * For Yoast blocks, if a node has attribute with 'id' key, this means that this node represents a sub-block.
+       * A sub-block has four child nodes:
+       * 1. The first child node of the sub-block, represents the first section of that sub-block.
+       *    - For example, in Yoast FAQ block, the first child node would represent the "question" section.
+       * 2. The second child node of the sub-block, only contains a new line.
+       * 3. The third child node of the sub-block, represents the second section of that sub-block.
+       *    - For example, in Yoast FAQ block, the second child node would represent the "answer" section.
+       * 4. The fourth child node of the sub-block, only contains a new line.
+       */
+      if (node.childNodes && node.childNodes.length > 3) {
+        node.childNodes[0].attributeId = node.attributes.id;
+        node.childNodes[0].isFirstSection = true;
+        node.childNodes[2].attributeId = node.attributes.id;
+        node.childNodes[2].isFirstSection = false;
+      }
+    }
+    updateClientIdAndAttrIdForSubtree(node, blocks, currentClientId);
+  });
+}
+
+/**
+ * Parses blocks and updates the clientId for the subtree.
+ * Additionally, when a node has an attribute with 'id' key, also set this id to the first and third child node of that node.
+ *
+ * @param {Paper} paper The paper to parse.
+ * @param {Node} node	The node to parse.
+ *
+ * @returns {void}
+ */
+function _default(paper, node) {
+  const blocks = paper._attributes.wpBlocks || [];
+  blockTokenizer.lastIndex = 0;
+  updateBlocksOffset(blocks, paper.getText());
+  const rawBlocks = getAllBlocks(paper);
+
+  // eslint-disable-next-line no-undefined
+  updateClientIdAndAttrIdForSubtree(node, rawBlocks, undefined);
+}
+//# sourceMappingURL=parseBlocks.js.map

package/build/parse/build/private/parseBlocks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parseBlocks.js","names":["_lodash","require","blockTokenizer","getAllBlocks","paper","blocks","_attributes","wpBlocks","flattenBlocks","length","forEach","block","innerBlocks","push","updateBlocksOffset","text","currentBlock","index","matches","exec","name","previousBlock","previousBlockEndOffset","endOffset","startedAt","startOffset","blockLength","contentOffset","match","updateClientIdAndAttrIdForSubtree","rootNode","clientId","currentClientId","sourceCodeLocation","isUndefined","foundBlock","find","childNodes","node","attributes","id","attributeId","isFirstSection","_default","lastIndex","getText","rawBlocks","undefined"],"sources":["../../../../src/parse/build/private/parseBlocks.js"],"sourcesContent":["import { isUndefined } from \"lodash\";\nconst blockTokenizer = /<!--\\s+wp:([a-z][a-z0-9_-]*\\/)?([a-z][a-z0-9_-]*)\\s+({(?:(?=([^}]+|}+(?=})|(?!}\\s+\\/?-->)[^])*)\\5|[^]*?)}\\s+)?(\\/)?-->/g;\n\n/**\n * Get all the blocks including the inner blocks.\n *\n * @param {Paper} paper The paper to get all the blocks from.\n *\n * @returns {Object[]} An array of blocks.\n */\nconst getAllBlocks = ( paper ) => {\n\tconst blocks = paper._attributes.wpBlocks;\n\tconst flattenBlocks = [];\n\tif ( ! ( blocks && blocks.length > 0 ) ) {\n\t\treturn [];\n\t}\n\tblocks.forEach( ( block ) => {\n\t\tif ( block.innerBlocks.length > 0 ) {\n\t\t\tconst innerBlocks = block.innerBlocks;\n\t\t\tflattenBlocks.push(\n\t\t\t\tblock, ...innerBlocks\n\t\t\t);\n\t\t} else {\n\t\t\tflattenBlocks.push( block );\n\t\t}\n\t} );\n\treturn flattenBlocks;\n};\n\n\n/**\n * Gets the offset of each block and set it to the tree.\n *\n * @param {Object[]}\tblocks An array of blocks.\n * @param {string}\t\ttext The text.\n *\n * @returns {void}\n */\nexport function updateBlocksOffset( blocks, text ) {\n\tif ( blocks.length === 0 ) {\n\t\treturn;\n\t}\n\tblocks.forEach( ( currentBlock, index ) => {\n\t\tconst matches = blockTokenizer.exec( text );\n\t\tif ( currentBlock.name === \"core/freeform\" ) {\n\t\t\tconst previousBlock = blocks[ index - 1 ];\n\t\t\tif ( previousBlock ) {\n\t\t\t\tconst previousBlockEndOffset = previousBlock.endOffset;\n\t\t\t\tconst startedAt = previousBlockEndOffset + 2;\n\t\t\t\tcurrentBlock.startOffset = startedAt;\n\t\t\t\tcurrentBlock.endOffset = startedAt + currentBlock.blockLength;\n\t\t\t\tcurrentBlock.contentOffset = startedAt;\n\t\t\t} else {\n\t\t\t\tcurrentBlock.startOffset = 0;\n\t\t\t\tcurrentBlock.endOffset = 0 + currentBlock.blockLength;\n\t\t\t\tcurrentBlock.contentOffset = 0;\n\t\t\t}\n\t\t} else {\n\t\t\tif ( null === matches ) {\n\t\t\t\treturn;\n\t\t\t}\n\t\t\tconst [ match ] = matches;\n\t\t\tconst startedAt = matches.index;\n\t\t\tconst length = match.length;\n\t\t\tcurrentBlock.startOffset = startedAt;\n\t\t\tcurrentBlock.endOffset = startedAt + currentBlock.blockLength;\n\t\t\tcurrentBlock.contentOffset = startedAt + length + 1;\n\t\t}\n\n\t\tif ( currentBlock.innerBlocks && currentBlock.innerBlocks.length > 0 ) {\n\t\t\tupdateBlocksOffset( currentBlock.innerBlocks, text );\n\t\t}\n\t} );\n}\n\n\n/**\n * Gets the client id for each block and set it to the child nodes.\n * Additionally, when a node has an attribute with 'id' key, also set this id to the first.\n *\n * @param {Node}\t\trootNode The root node.\n * @param {Object[]}\tblocks An array of blocks.\n * @param {string}\t\tclientId The client id to set.\n *\n * @returns {void}\n */\nfunction updateClientIdAndAttrIdForSubtree( rootNode, blocks, clientId ) {\n\tif ( ! rootNode ) {\n\t\treturn;\n\t}\n\n\tlet currentClientId = clientId;\n\tif ( rootNode.sourceCodeLocation && ! isUndefined( rootNode.sourceCodeLocation.startOffset ) ) {\n\t\tconst foundBlock = blocks.find( block => block.contentOffset === rootNode.sourceCodeLocation.startOffset );\n\t\tif ( foundBlock ) {\n\t\t\tcurrentClientId = foundBlock.clientId;\n\t\t}\n\t}\n\n\t// If the clientId is not undefined, also set this to the root node.\n\tif ( currentClientId ) {\n\t\trootNode.clientId = currentClientId;\n\t}\n\n\t// If the node has children, update the clientId for them.\n\t( rootNode.childNodes || [] ).forEach( ( node ) => {\n\t\tif ( node.attributes && node.attributes.id ) {\n\t\t\t/*\n\t\t\t * If the node's child has an attribute with 'id' key, also set this id to the first and third child node.\n\t\t\t * This step is specifically for parsing the Yoast blocks.\n\t\t\t *\n\t\t\t * For Yoast blocks, if a node has attribute with 'id' key, this means that this node represents a sub-block.\n\t\t\t * A sub-block has four child nodes:\n\t\t\t * 1. The first child node of the sub-block, represents the first section of that sub-block.\n\t\t\t *    - For example, in Yoast FAQ block, the first child node would represent the \"question\" section.\n\t\t\t * 2. The second child node of the sub-block, only contains a new line.\n\t\t\t * 3. The third child node of the sub-block, represents the second section of that sub-block.\n\t\t\t *    - For example, in Yoast FAQ block, the second child node would represent the \"answer\" section.\n\t\t\t * 4. The fourth child node of the sub-block, only contains a new line.\n\t\t\t */\n\t\t\tif ( node.childNodes && node.childNodes.length > 3 ) {\n\t\t\t\tnode.childNodes[ 0 ].attributeId = node.attributes.id;\n\t\t\t\tnode.childNodes[ 0 ].isFirstSection = true;\n\n\t\t\t\tnode.childNodes[ 2 ].attributeId = node.attributes.id;\n\t\t\t\tnode.childNodes[ 2 ].isFirstSection = false;\n\t\t\t}\n\t\t}\n\t\tupdateClientIdAndAttrIdForSubtree( node, blocks, currentClientId );\n\t} );\n}\n\n/**\n * Parses blocks and updates the clientId for the subtree.\n * Additionally, when a node has an attribute with 'id' key, also set this id to the first and third child node of that node.\n *\n * @param {Paper} paper The paper to parse.\n * @param {Node} node\tThe node to parse.\n *\n * @returns {void}\n */\nexport default function( paper, node ) {\n\tconst blocks = paper._attributes.wpBlocks || [];\n\tblockTokenizer.lastIndex = 0;\n\tupdateBlocksOffset( blocks, paper.getText() );\n\n\tconst rawBlocks = getAllBlocks( paper );\n\n\t// eslint-disable-next-line no-undefined\n\tupdateClientIdAndAttrIdForSubtree( node, rawBlocks, undefined );\n}\n"],"mappings":";;;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AACA,MAAMC,cAAc,GAAG,yHAAyH;;AAEhJ;AACA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMC,YAAY,GAAKC,KAAK,IAAM;EACjC,MAAMC,MAAM,GAAGD,KAAK,CAACE,WAAW,CAACC,QAAQ;EACzC,MAAMC,aAAa,GAAG,EAAE;EACxB,IAAK,EAAIH,MAAM,IAAIA,MAAM,CAACI,MAAM,GAAG,CAAC,CAAE,EAAG;IACxC,OAAO,EAAE;EACV;EACAJ,MAAM,CAACK,OAAO,CAAIC,KAAK,IAAM;IAC5B,IAAKA,KAAK,CAACC,WAAW,CAACH,MAAM,GAAG,CAAC,EAAG;MACnC,MAAMG,WAAW,GAAGD,KAAK,CAACC,WAAW;MACrCJ,aAAa,CAACK,IAAI,CACjBF,KAAK,EAAE,GAAGC,WACX,CAAC;IACF,CAAC,MAAM;MACNJ,aAAa,CAACK,IAAI,CAAEF,KAAM,CAAC;IAC5B;EACD,CAAE,CAAC;EACH,OAAOH,aAAa;AACrB,CAAC;;AAGD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACO,SAASM,kBAAkBA,CAAET,MAAM,EAAEU,IAAI,EAAG;EAClD,IAAKV,MAAM,CAACI,MAAM,KAAK,CAAC,EAAG;IAC1B;EACD;EACAJ,MAAM,CAACK,OAAO,CAAE,CAAEM,YAAY,EAAEC,KAAK,KAAM;IAC1C,MAAMC,OAAO,GAAGhB,cAAc,CAACiB,IAAI,CAAEJ,IAAK,CAAC;IAC3C,IAAKC,YAAY,CAACI,IAAI,KAAK,eAAe,EAAG;MAC5C,MAAMC,aAAa,GAAGhB,MAAM,CAAEY,KAAK,GAAG,CAAC,CAAE;MACzC,IAAKI,aAAa,EAAG;QACpB,MAAMC,sBAAsB,GAAGD,aAAa,CAACE,SAAS;QACtD,MAAMC,SAAS,GAAGF,sBAAsB,GAAG,CAAC;QAC5CN,YAAY,CAACS,WAAW,GAAGD,SAAS;QACpCR,YAAY,CAACO,SAAS,GAAGC,SAAS,GAAGR,YAAY,CAACU,WAAW;QAC7DV,YAAY,CAACW,aAAa,GAAGH,SAAS;MACvC,CAAC,MAAM;QACNR,YAAY,CAACS,WAAW,GAAG,CAAC;QAC5BT,YAAY,CAACO,SAAS,GAAG,CAAC,GAAGP,YAAY,CAACU,WAAW;QACrDV,YAAY,CAACW,aAAa,GAAG,CAAC;MAC/B;IACD,CAAC,MAAM;MACN,IAAK,IAAI,KAAKT,OAAO,EAAG;QACvB;MACD;MACA,MAAM,CAAEU,KAAK,CAAE,GAAGV,OAAO;MACzB,MAAMM,SAAS,GAAGN,OAAO,CAACD,KAAK;MAC/B,MAAMR,MAAM,GAAGmB,KAAK,CAACnB,MAAM;MAC3BO,YAAY,CAACS,WAAW,GAAGD,SAAS;MACpCR,YAAY,CAACO,SAAS,GAAGC,SAAS,GAAGR,YAAY,CAACU,WAAW;MAC7DV,YAAY,CAACW,aAAa,GAAGH,SAAS,GAAGf,MAAM,GAAG,CAAC;IACpD;IAEA,IAAKO,YAAY,CAACJ,WAAW,IAAII,YAAY,CAACJ,WAAW,CAACH,MAAM,GAAG,CAAC,EAAG;MACtEK,kBAAkB,CAAEE,YAAY,CAACJ,WAAW,EAAEG,IAAK,CAAC;IACrD;EACD,CAAE,CAAC;AACJ;;AAGA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAASc,iCAAiCA,CAAEC,QAAQ,EAAEzB,MAAM,EAAE0B,QAAQ,EAAG;EACxE,IAAK,CAAED,QAAQ,EAAG;IACjB;EACD;EAEA,IAAIE,eAAe,GAAGD,QAAQ;EAC9B,IAAKD,QAAQ,CAACG,kBAAkB,IAAI,CAAE,IAAAC,mBAAW,EAAEJ,QAAQ,CAACG,kBAAkB,CAACR,WAAY,CAAC,EAAG;IAC9F,MAAMU,UAAU,GAAG9B,MAAM,CAAC+B,IAAI,CAAEzB,KAAK,IAAIA,KAAK,CAACgB,aAAa,KAAKG,QAAQ,CAACG,kBAAkB,CAACR,WAAY,CAAC;IAC1G,IAAKU,UAAU,EAAG;MACjBH,eAAe,GAAGG,UAAU,CAACJ,QAAQ;IACtC;EACD;;EAEA;EACA,IAAKC,eAAe,EAAG;IACtBF,QAAQ,CAACC,QAAQ,GAAGC,eAAe;EACpC;;EAEA;EACA,CAAEF,QAAQ,CAACO,UAAU,IAAI,EAAE,EAAG3B,OAAO,CAAI4B,IAAI,IAAM;IAClD,IAAKA,IAAI,CAACC,UAAU,IAAID,IAAI,CAACC,UAAU,CAACC,EAAE,EAAG;MAC5C;AACH;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;MACG,IAAKF,IAAI,CAACD,UAAU,IAAIC,IAAI,CAACD,UAAU,CAAC5B,MAAM,GAAG,CAAC,EAAG;QACpD6B,IAAI,CAACD,UAAU,CAAE,CAAC,CAAE,CAACI,WAAW,GAAGH,IAAI,CAACC,UAAU,CAACC,EAAE;QACrDF,IAAI,CAACD,UAAU,CAAE,CAAC,CAAE,CAACK,cAAc,GAAG,IAAI;QAE1CJ,IAAI,CAACD,UAAU,CAAE,CAAC,CAAE,CAACI,WAAW,GAAGH,IAAI,CAACC,UAAU,CAACC,EAAE;QACrDF,IAAI,CAACD,UAAU,CAAE,CAAC,CAAE,CAACK,cAAc,GAAG,KAAK;MAC5C;IACD;IACAb,iCAAiC,CAAES,IAAI,EAAEjC,MAAM,EAAE2B,eAAgB,CAAC;EACnE,CAAE,CAAC;AACJ;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACe,SAAAW,SAAUvC,KAAK,EAAEkC,IAAI,EAAG;EACtC,MAAMjC,MAAM,GAAGD,KAAK,CAACE,WAAW,CAACC,QAAQ,IAAI,EAAE;EAC/CL,cAAc,CAAC0C,SAAS,GAAG,CAAC;EAC5B9B,kBAAkB,CAAET,MAAM,EAAED,KAAK,CAACyC,OAAO,CAAC,CAAE,CAAC;EAE7C,MAAMC,SAAS,GAAG3C,YAAY,CAAEC,KAAM,CAAC;;EAEvC;EACAyB,iCAAiC,CAAES,IAAI,EAAEQ,SAAS,EAAEC,SAAU,CAAC;AAChE","ignoreList":[]}

package/build/parse/build/private/tokenize.js

@@ -0,0 +1,78 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+var _structure = require("../../structure");
+var _getTextElementPositions = _interopRequireDefault(require("./getTextElementPositions"));
+var _htmlEntities = require("../../../helpers/htmlEntities");
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+/**
+ * Splits the sentence into tokens, determines their positions in the source code, and puts them on the sentence.
+ *
+ * @param {Paragraph|Heading} node The paragraph or heading node to split into sentences.
+ * @param {Sentence} sentence The sentence.
+ * @param {LanguageProcessor} languageProcessor The language processor for the current language.
+ *
+ * @returns {Sentence} The sentence, with tokens.
+ */
+function getTokens(node, sentence, languageProcessor) {
+  sentence.tokens = languageProcessor.splitIntoTokens(sentence);
+  sentence.tokens = (0, _getTextElementPositions.default)(node, sentence.tokens, sentence.sourceCodeRange.startOffset);
+  return sentence;
+}
+
+/**
+ * Splits the node's inner text into sentences, and the sentences into tokens,
+ * using the language processor.
+ *
+ * @param {Paragraph|Heading} node 				The paragraph or heading node to split into sentences.
+ * @param {LanguageProcessor} languageProcessor The language processor to use.
+ *
+ * @returns {Sentence[]} The node's sentences.
+ */
+function getSentences(node, languageProcessor) {
+  // Split text into sentences.
+  let sentences = languageProcessor.splitIntoSentences(node.innerText());
+  // Add position information to the sentences.
+  sentences = (0, _getTextElementPositions.default)(node, sentences);
+  // Tokenize sentences into tokens.
+  return sentences.map(sentence => {
+    sentence = getTokens(node, sentence, languageProcessor);
+    // Now positions have been determined, change HTML entities that had earlier been converted to hashed versions back to their short version.
+    // For example, "&" was earlier converted into "#amp;" and is now converted into "&".
+    // We make this change in both the Sentence and the accompanying Tokens.
+    _htmlEntities.hashedHtmlEntities.forEach((character, hashedHtmlEntity) => {
+      // We use a global regex instead of replaceAll to support older browsers.
+      const hashedHtmlEntityRegex = new RegExp(hashedHtmlEntity, "g");
+      sentence.text = sentence.text.replace(hashedHtmlEntityRegex, character);
+      sentence.tokens.map(token => {
+        token.text = token.text.replace(hashedHtmlEntityRegex, character);
+        return token;
+      });
+    });
+    return sentence;
+  });
+}
+
+/**
+ * Splits any Paragraph and Heading nodes in the tree into sentences and tokens.
+ * Excludes overarching Paragraphs, as those will have (implicit) paragraphs as their children.
+ *
+ * @param {Node} tree The tree to process.
+ * @param {LanguageProcessor} languageProcessor The language processor to use.
+ *
+ * @returns {Node} The processed tree.
+ */
+function tokenize(tree, languageProcessor) {
+  if (tree instanceof _structure.Paragraph && tree.name !== "p-overarching" || tree instanceof _structure.Heading) {
+    tree.sentences = getSentences(tree, languageProcessor);
+  }
+  if (tree.childNodes) {
+    tree.childNodes = tree.childNodes.map(child => tokenize(child, languageProcessor));
+  }
+  return tree;
+}
+var _default = exports.default = tokenize;
+//# sourceMappingURL=tokenize.js.map