cherry-muse 1.0.1

package/src/core/hooks/Header.js

@@ -0,0 +1,229 @@
+/**
+ * Copyright (C) 2021 THL A29 Limited, a Tencent company.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ParagraphBase from '@/core/ParagraphBase';
+import { compileRegExp } from '@/utils/regexp';
+import { calculateLinesOfParagraph } from '@/utils/lineFeed';
+
+const ATX_HEADER = 'atx';
+const SETEXT_HEADER = 'setext';
+const toDashChars = /[\s\-_]/;
+const alphabetic = /[A-Za-z]/;
+const numeric = /[0-9]/;
+
+export default class Header extends ParagraphBase {
+  static HOOK_NAME = 'header';
+
+  constructor({ externals, config } = { config: undefined, externals: undefined }) {
+    super({ needCache: true });
+    this.strict = config ? !!config.strict : true;
+    this.RULE = this.rule();
+    this.headerIDCache = [];
+    this.headerIDCounter = {};
+    this.config = config || {};
+    // TODO: AllowCustomID
+  }
+
+  $parseTitleText(html = '') {
+    if (typeof html !== 'string') {
+      return '';
+    }
+    return html.replace(/<.*?>/g, '').replace(/&#60;/g, '<').replace(/&#62;/g, '>');
+  }
+
+  /**
+   * refer:
+   * @see https://github.com/vsch/flexmark-java/blob/8bf621924158dfed8b84120479c82704020a6927/flexmark
+   * /src/main/java/com/vladsch/flexmark/html/renderer/HeaderIdGenerator.java#L90-L113
+   *
+   * @param {string} headerText
+   * @param {boolean} [toLowerCase]
+   * @returns
+   */
+  $generateId(headerText, toLowerCase = true) {
+    const len = headerText.length;
+    let id = '';
+    for (let i = 0; i < len; i++) {
+      const c = headerText.charAt(i);
+      if (alphabetic.test(c)) {
+        id += toLowerCase ? c.toLowerCase() : c;
+      } else if (numeric.test(c)) {
+        id += c;
+      } else if (toDashChars.test(c)) {
+        id += id.length < 1 || id.charAt(id.length - 1) !== '-' ? '-' : '';
+      } else if (c.charCodeAt(0) > 255) {
+        // unicode
+        try {
+          id += encodeURIComponent(c);
+        } catch (error) {
+          // empty
+        }
+      }
+    }
+    return id;
+  }
+
+  generateIDNoDup(headerText) {
+    // 处理被引擎转换过的实体字符
+    const unescapedHeaderText = headerText.replace(/&#60;/g, '<').replace(/&#62;/g, '>');
+    let newId = this.$generateId(unescapedHeaderText, true);
+    const idIndex = this.headerIDCache.indexOf(newId);
+    if (idIndex !== -1) {
+      this.headerIDCounter[idIndex] += 1;
+      newId += `-${this.headerIDCounter[idIndex] + 1}`;
+    } else {
+      const newIndex = this.headerIDCache.push(newId);
+      this.headerIDCounter[newIndex - 1] = 1;
+    }
+    return newId;
+  }
+
+  $wrapHeader(text, level, dataLines, sentenceMakeFunc) {
+    // 需要经过一次escape
+    const processedText = sentenceMakeFunc(text.trim());
+    let { html } = processedText;
+    const customIDRegex = /\s+\{#([A-Za-z0-9-]+)\}$/; // ?<id>
+    const idMatch = html.match(customIDRegex);
+    let anchorID;
+    if (idMatch !== null) {
+      html = html.substring(0, idMatch.index);
+      [, anchorID] = idMatch;
+    }
+    const headerTextRaw = this.$parseTitleText(html);
+    if (!anchorID) {
+      const replaceFootNote = /~fn#([0-9]+)#/g;
+      anchorID = this.generateIDNoDup(headerTextRaw.replace(replaceFootNote, ''));
+    }
+    const safeAnchorID = `safe_${anchorID}`; // transform header id to avoid being sanitized
+    const sign = this.$engine.md5(`${level}-${processedText.sign}-${anchorID}-${dataLines}`);
+    const result = [
+      `<h${level} id="${safeAnchorID}" data-sign="${sign}" data-lines="${dataLines}">`,
+      this.$getAnchor(anchorID, html),
+      // `${html}`,
+      `</h${level}>`,
+    ].join('');
+    return { html: result, sign: `${sign}` };
+  }
+
+  $getAnchor(anchorID, text) {
+    const anchorStyle = this.config.anchorStyle || 'default';
+    if (anchorStyle === 'none') {
+      return '';
+    }
+    return `${text} <a class="anchor" href="#${anchorID}">#</a>`;
+  }
+
+  beforeMakeHtml(str) {
+    let $str = str;
+    if (this.$engine.$cherry.options.engine.global.flowSessionContext) {
+      // 适配流式会话的场景，文章末尾的段横线标题语法（`\n-`）失效
+      $str = $str.replace(/(\n\s*-{1,})\s*$/, '$1 ');
+    }
+    // atx 优先
+    if (this.test($str, ATX_HEADER)) {
+      $str = $str.replace(this.RULE[ATX_HEADER].reg, (match, lines, level, text) => {
+        if (text.trim() === '') {
+          return match;
+        }
+        return this.getCacheWithSpace(this.pushCache(match), match, true);
+      });
+    }
+    // 按照目前的引擎，每个hook只会执行一次，所以需要并行执行替换
+    if (this.test($str, SETEXT_HEADER)) {
+      $str = $str.replace(this.RULE[SETEXT_HEADER].reg, (match, lines, text) => {
+        if (text.trim() === '' || this.isContainsCache(text)) {
+          return match;
+        }
+        return this.getCacheWithSpace(this.pushCache(match), match, true);
+      });
+    }
+    return $str;
+  }
+
+  makeHtml(str, sentenceMakeFunc) {
+    // 先还原
+    let $str = this.restoreCache(str);
+    // atx 优先
+    if (this.test($str, ATX_HEADER)) {
+      $str = $str.replace(this.RULE[ATX_HEADER].reg, (match, lines, level, text) => {
+        // 其中有两行是beforeMake加上的
+        const lineCount = calculateLinesOfParagraph(lines, this.getLineCount(match.replace(/^\n+/, '')));
+        const $text = text.replace(/\s+#+\s*$/, ''); // close tag
+        const { html: result, sign } = this.$wrapHeader($text, level.length, lineCount, sentenceMakeFunc);
+        // 文章的开头不加换行
+        return this.getCacheWithSpace(this.pushCache(result, sign, lineCount), match, true);
+      });
+    }
+    // 按照目前的引擎，每个hook只会执行一次，所以需要并行执行替换
+    if (this.test($str, SETEXT_HEADER)) {
+      $str = $str.replace(this.RULE[SETEXT_HEADER].reg, (match, lines, text, level) => {
+        if (this.isContainsCache(text)) {
+          return match;
+        }
+        // 其中有两行是beforeMake加上的
+        const lineCount = calculateLinesOfParagraph(lines, this.getLineCount(match.replace(/^\n+/, '')));
+        const headerLevel = level[0] === '-' ? 2 : 1; // =: H1, -: H2
+        const { html: result, sign } = this.$wrapHeader(text, headerLevel, lineCount, sentenceMakeFunc);
+        // 文章的开头不加换行
+        return this.getCacheWithSpace(this.pushCache(result, sign, lineCount), match, true);
+      });
+    }
+    return $str;
+  }
+
+  afterMakeHtml(html) {
+    const $html = super.afterMakeHtml(html);
+    this.headerIDCache = [];
+    this.headerIDCounter = {};
+    return $html;
+  }
+
+  test(str, flavor) {
+    return this.RULE[flavor].reg && this.RULE[flavor].reg.test(str);
+  }
+
+  /**
+   * TODO: fix type errors, prefer use `rules` for multiple spec instead
+   * @returns
+   */
+  rule() {
+    // setext Header
+    // TODO: 支持多行标题
+    const setext = {
+      begin: '(?:^|\\n)(\\n*)', // (?<lines>\\n*)
+      content: [
+        '(?:\\h*',
+        '(.+)', // (?<text>.+)
+        ')\\n',
+        '(?:\\h*',
+        '([=]+|[-]+)', // (?<level>[=]+|[-]+)
+        ')',
+      ].join(''),
+      end: '(?=$|\\n)',
+    };
+    setext.reg = compileRegExp(setext, 'g', true);
+
+    // atx header
+    const atx = {
+      begin: '(?:^|\\n)(\\n*)(?:\\h*(#{1,6}))', // (?<lines>\\n*), (?<level>#{1,6})
+      content: '(.+?)', // '(?<text>.+?)'
+      end: '(?=$|\\n)',
+    };
+    this.strict && (atx.begin += '(?=\\h+)'); // (?=\\s+) for strict mode
+    atx.reg = compileRegExp(atx, 'g', true);
+
+    return /** @type {any} */ ({ setext, atx });
+  }
+}

package/src/core/hooks/HighLight.js

@@ -0,0 +1,37 @@
+/**
+ * Copyright (C) 2021 THL A29 Limited, a Tencent company.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import SyntaxBase from '../SyntaxBase.js';
+
+export default class HighLight extends SyntaxBase {
+  static HOOK_NAME = 'highLight';
+
+  makeHtml(str) {
+    if (!this.test(str)) {
+      return str;
+    }
+    return str.replace(this.RULE.reg, '$1<mark>$2</mark>$3');
+  }
+
+  rule() {
+    const ret = {
+      begin: '(^| )==',
+      end: '==( |$|\\n)',
+      content: '([^\\n]+?)',
+    };
+    ret.reg = new RegExp(ret.begin + ret.content + ret.end, 'g');
+    return ret;
+  }
+}

package/src/core/hooks/Hr.js

@@ -0,0 +1,52 @@
+/**
+ * Copyright (C) 2021 THL A29 Limited, a Tencent company.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ParagraphBase from '@/core/ParagraphBase';
+import { prependLineFeedForParagraph } from '@/utils/lineFeed';
+/**
+ * 分割线语法
+ */
+export default class Hr extends ParagraphBase {
+  static HOOK_NAME = 'hr';
+
+  constructor() {
+    super({ needCache: true });
+  }
+
+  beforeMakeHtml(str) {
+    return str.replace(this.RULE.reg, (match, preLines) => {
+      const lineCount = (preLines.match(/\n/g) || []).length + 1;
+      // 计算签名，签名可能会重复，符合预期
+      const sign = `hr${lineCount}`;
+      const placeHolder = this.pushCache(`<hr data-sign="${sign}" data-lines="${lineCount}" />`, sign);
+      return prependLineFeedForParagraph(match, placeHolder);
+    });
+  }
+
+  makeHtml(str, sentenceMakeFunc) {
+    return str;
+  }
+
+  rule() {
+    // 分割线必须有从新行开始，比如以换行结束
+    const ret = {
+      begin: '(?:^|\\n)(\\n*)[ ]*',
+      end: '(?=$|\\n)',
+      content: '((?:-[ \\t]*){3,}|(?:\\*[ \\t]*){3,}|(?:_[ \\t]*){3,})',
+    };
+    ret.reg = new RegExp(ret.begin + ret.content + ret.end, 'g');
+    return ret;
+  }
+}

package/src/core/hooks/HtmlBlock.js

@@ -0,0 +1,159 @@
+/**
+ * Copyright (C) 2021 THL A29 Limited, a Tencent company.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import ParagraphBase from '@/core/ParagraphBase';
+import {
+  whiteList,
+  convertHTMLNumberToName,
+  // isValidScheme, encodeURIOnce,
+  escapeHTMLEntitiesWithoutSemicolon,
+} from '@/utils/sanitize';
+import { sanitizer } from '@/Sanitizer';
+import { isBrowser } from '@/utils/env';
+
+/**
+ * encode unsafe link-related attributes
+ */
+const unsafeAttributes = ['href', 'src'];
+
+sanitizer.addHook('afterSanitizeAttributes', (node) => {
+  unsafeAttributes.forEach((attr) => {
+    if (!node.hasAttribute(attr)) {
+      return;
+    }
+    const value = node.getAttribute(attr);
+    // encode unsafe backslash in link attributes
+    node.setAttribute(attr, value.replace(/\\/g, '%5c'));
+  });
+});
+
+export default class HtmlBlock extends ParagraphBase {
+  static HOOK_NAME = 'htmlBlock';
+  constructor() {
+    super({ needCache: true });
+  }
+
+  // ref: http://www.vfmd.org/vfmd-spec/specification/#procedure-for-detecting-automatic-links
+  isAutoLinkTag(tagMatch) {
+    const REGEX_GROUP = [
+      /^<([a-z][a-z0-9+.-]{1,31}:\/\/[^<> `]+)>$/i,
+      /^<(mailto:[^<> `]+)>$/i,
+      /^<([^()<>[\]:'@\\,"\s`]+@[^()<>[\]:'@\\,"\s`.]+\.[^()<>[\]:'@\\,"\s`]+)>$/i,
+    ];
+    return REGEX_GROUP.some((regex) => regex.test(tagMatch));
+  }
+
+  isHtmlComment(match) {
+    const htmlComment = /^<!--.*?-->$/;
+    return htmlComment.test(match);
+  }
+
+  beforeMakeHtml(str, sentenceMakeFunc) {
+    if (this.$engine.htmlWhiteListAppend) {
+      /**
+       * @property
+       * @type {false | RegExp}
+       */
+      this.htmlWhiteListAppend = new RegExp(`^(${this.$engine.htmlWhiteListAppend})( |$|/)`, 'i');
+      /**
+       * @property
+       * @type {string[]}
+       */
+      this.htmlWhiteList = this.$engine.htmlWhiteListAppend.split('|');
+    } else {
+      this.htmlWhiteListAppend = false;
+      this.htmlWhiteList = [];
+    }
+
+    let $str = str;
+    $str = convertHTMLNumberToName($str);
+    $str = escapeHTMLEntitiesWithoutSemicolon($str);
+    $str = $str.replace(/<[/]?(.*?)>/g, (whole, m1) => {
+      // 匹配到非白名单且非AutoLink语法的尖括号会被转义
+      // 如果是HTML注释，放行
+      if (!whiteList.test(m1) && !this.isAutoLinkTag(whole) && !this.isHtmlComment(whole)) {
+        if (this.htmlWhiteListAppend === false || !this.htmlWhiteListAppend.test(m1)) {
+          return whole.replace(/</g, '&#60;').replace(/>/g, '&#62;');
+        }
+      }
+      // 到达此分支的包含被尖括号包裹的AutoLink语法以及在白名单内的HTML标签
+      // 没有被AutoLink解析并渲染的标签会被DOMPurify过滤掉，正常情况下不会出现遗漏
+      // 临时替换完整的HTML标签首尾为$#60;和$#62;，供下一步剔除损坏的HTML标签
+      return whole.replace(/</g, '$#60;').replace(/>/g, '$#62;');
+    });
+    // 替换所有形如「<abcd」和「</abcd」的左尖括号
+    $str = $str.replace(/<(?=\/?(\w|\n|$))/g, '&#60;');
+    // 还原被替换的尖括号
+    $str = $str.replace(/\$#60;/g, '<').replace(/\$#62;/g, '>');
+    return $str;
+  }
+
+  // beforeMakeHtml(str) {
+  //     return str;
+  // }
+
+  makeHtml(str, sentenceMakeFunc) {
+    return str;
+  }
+
+  afterMakeHtml(str) {
+    let $str = str;
+    const config = {
+      ALLOW_UNKNOWN_PROTOCOLS: true,
+      ADD_ATTR: ['target'],
+    };
+    if (this.htmlWhiteListAppend !== false) {
+      config.ADD_TAGS = this.htmlWhiteList;
+      if (this.htmlWhiteListAppend.test('style') || this.htmlWhiteListAppend.test('ALL')) {
+        $str = $str.replace(/<style(>| [^>]*>).*?<\/style>/gi, (match) => {
+          return match.replace(/<br>/gi, '');
+        });
+      }
+      if (this.htmlWhiteListAppend.test('iframe') || this.htmlWhiteListAppend.test('ALL')) {
+        config.ADD_ATTR = config.ADD_ATTR.concat([
+          'align',
+          'frameborder',
+          'height',
+          'longdesc',
+          'marginheight',
+          'marginwidth',
+          'name',
+          'sandbox',
+          'scrolling',
+          'seamless',
+          'src',
+          'srcdoc',
+          'width',
+        ]);
+        config.SANITIZE_DOM = false;
+        $str = $str.replace(/<iframe(>| [^>]*>).*?<\/iframe>/gi, (match) => {
+          return match.replace(/<br>/gi, '').replace(/\n/g, '');
+        });
+      }
+      if (this.htmlWhiteListAppend.test('script') || this.htmlWhiteListAppend.test('ALL')) {
+        // 如果允许script或者输入了ALL，则不做任何过滤了
+        $str = $str.replace(/<script(>| [^>]*>).*?<\/script>/gi, (match) => {
+          return match.replace(/<br>/gi, '');
+        });
+        return $str;
+      }
+    }
+    // node 环境下不输出sign和lines
+    if (!isBrowser()) {
+      config.FORBID_ATTR = ['data-sign', 'data-lines'];
+    }
+    return sanitizer.sanitize($str, config);
+  }
+}

package/src/core/hooks/Iframe.js

@@ -0,0 +1,80 @@
+import { isValidScheme } from '@/utils/sanitize';
+import { prependLineFeedForParagraph } from '@/utils/lineFeed';
+import SyntaxBase from '@/core/SyntaxBase';
+
+export default class Iframe extends SyntaxBase {
+  static HOOK_NAME = 'iframe';
+
+  rule() {
+    const ret = {
+      begin: /^@@/m, // 严格匹配行首的@@
+      content: /https?:\/\/\S+/, // 匹配到第一个空格/换行立即停止
+      end: /(?=\s|$)/, // 结束条件简化为空格或行尾
+    };
+
+    ret.reg = new RegExp(
+      `${ret.begin.source}\\s*` + // 允许@@后跟空格
+        `(${
+          // 第1捕获组：完整URL（含参数）
+          ret.content.source
+        })${ret.end.source}`,
+      'gm', // 全局+多行模式（确保^生效）
+    );
+    return ret;
+  }
+
+  makeHtml(str, sentenceMakeFunc) {
+    return str.replace(this.RULE.reg, (match, url) => {
+      const replacedUrl = url.trim().replace(/&/g, '&');
+      const attrs = {
+        width: '100%',
+        height: '400px',
+        sandbox: 'allow-scripts',
+      };
+
+      const iframe = `<iframe src="${replacedUrl}"
+       
+      width="${attrs.width}" 
+      height="${attrs.height}"
+      sandbox="${attrs.sandbox}"
+   
+      frameborder="0"
+      loading="lazy"
+      allow="fullscreen"
+    ></iframe>`;
+      return iframe;
+    });
+  }
+
+  overlayMode() {
+    return {
+      inIframe: false,
+      name: 'iframe',
+      token(stream, state) {
+        // 检测@@起始标记
+        if (!this.inIframe && stream.match(/^@@/)) {
+          this.inIframe = true;
+          return 'iframe-marker';
+        }
+
+        // 处理iframe内容区域
+        if (this.inIframe) {
+          // 匹配到空白字符结束
+          if (stream.eatWhile(/.*?$/)) {
+            this.inIframe = false;
+            return 'iframe-url';
+          }
+
+          // 遇到换行符自动结束
+          if (this.sol()) {
+            state.inIframe = false;
+          }
+        }
+
+        // 非iframe内容继续默认处理
+        stream.next();
+        return null;
+      },
+    };
+  }
+}