@diplodoc/transform 4.29.0 → 4.30.1

package/src/transform/plugins/includes/collect.ts

@@ -1,24 +1,39 @@
 import {relative} from 'path';
 import {bold} from 'chalk';
 
-import {isFileExists, resolveRelativePath} from '../../utilsFS';
+import {getRelativePath, isFileExists, resolveRelativePath} from '../../utilsFS';
 import {MarkdownItPluginOpts} from '../typings';
 
 const includesPaths: string[] = [];
 
 type Opts = MarkdownItPluginOpts & {
     destPath: string;
-    copyFile(path: string, dest: string, opts: Opts): void;
+    copyFile(path: string, dest: string, opts: Opts): string | null | undefined;
     singlePage: Boolean;
+    included: Boolean;
+    includedParentPath?: string;
 };
 
 const collect = (input: string, options: Opts) => {
-    const {root, path, destPath = '', log, copyFile, singlePage} = options;
+    const {
+        root,
+        path,
+        destPath = '',
+        log,
+        copyFile,
+        singlePage,
+        includedParentPath: includedParentPathNullable,
+        included,
+    } = options;
+    const includedParentPath = includedParentPathNullable || path;
+
     const INCLUDE_REGEXP = /{%\s*include\s*(notitle)?\s*\[(.+?)]\((.+?)\)\s*%}/g;
 
     let match,
         result = input;
 
+    const appendix: Map<string, string> = new Map();
+
     while ((match = INCLUDE_REGEXP.exec(result)) !== null) {
         let [, , , relativePath] = match;
         const [matchedInclude] = match;
@@ -55,7 +70,27 @@ const collect = (input: string, options: Opts) => {
         };
 
         try {
-            copyFile(includePath, targetDestPath, includeOptions);
+            const content = copyFile(includePath, targetDestPath, includeOptions);
+
+            // To reduce file reading we can include the file content into the generated content
+            if (included && content) {
+                const includedRelativePath = getRelativePath(includedParentPath, includePath);
+
+                // The appendix is the map that protects from multiple include files
+                if (!appendix.has(includedRelativePath)) {
+                    // Recursive function to include the depth structure
+                    const includeContent = collect(content, {
+                        ...options,
+                        path: includePath,
+                        includedParentPath,
+                    });
+                    // Add to appendix set structure
+                    appendix.set(
+                        includedRelativePath,
+                        `{% included (${includedRelativePath}) %}\n${includeContent}\n{% endincluded %}`,
+                    );
+                }
+            }
         } catch (e) {
             log.error(`No such file or has no access to ${bold(includePath)} in ${bold(path)}`);
         } finally {
@@ -63,11 +98,16 @@ const collect = (input: string, options: Opts) => {
         }
     }
 
+    // Appendix should be appended to the end of the file (it supports depth structure, so the included files will have included as well)
+    if (appendix.size > 0) {
+        result += '\n' + [...appendix.values()].join('\n');
+    }
+
     if (singlePage) {
         return result;
     }
 
-    return null;
+    return result;
 };
 
 export = collect;

package/src/transform/plugins/includes/index.ts

@@ -1,10 +1,18 @@
 import {bold} from 'chalk';
+import Token from 'markdown-it/lib/token';
 
-import {GetFileTokensOpts, getFileTokens, getFullIncludePath, isFileExists} from '../../utilsFS';
+import {StateCore} from '../../typings';
+import {
+    GetFileTokensOpts,
+    getFileTokens,
+    getFullIncludePath,
+    isFileExists,
+    resolveRelativePath,
+} from '../../utilsFS';
 import {findBlockTokens} from '../../utils';
-import Token from 'markdown-it/lib/token';
 import {MarkdownItPluginCb, MarkdownItPluginOpts} from '../typings';
-import {StateCore} from 'src/transform/typings';
+
+import {MarkdownItIncluded} from './types';
 
 const INCLUDE_REGEXP = /^{%\s*include\s*(notitle)?\s*\[(.+?)]\((.+?)\)\s*%}$/;
 
@@ -20,7 +28,7 @@ type Options = MarkdownItPluginOpts &
         noReplaceInclude: boolean;
     };
 
-function unfoldIncludes(state: StateCore, path: string, options: Options) {
+function unfoldIncludes(md: MarkdownItIncluded, state: StateCore, path: string, options: Options) {
     const {root, notFoundCb, log, noReplaceInclude = false} = options;
     const {tokens} = state;
     let i = 0;
@@ -41,6 +49,11 @@ function unfoldIncludes(state: StateCore, path: string, options: Options) {
                 const [, keyword /* description */, , includePath] = match;
 
                 const fullIncludePath = getFullIncludePath(includePath, root, path);
+                const relativeIncludePath = resolveRelativePath(path, includePath);
+
+                // Check the existed included store and extract it
+                const included = md.included?.[relativeIncludePath];
+
                 let pathname = fullIncludePath;
                 let hash = '';
                 const hashIndex = fullIncludePath.lastIndexOf('#');
@@ -55,7 +68,10 @@ function unfoldIncludes(state: StateCore, path: string, options: Options) {
                     continue;
                 }
 
-                const fileTokens = getFileTokens(pathname, state, options);
+                const fileTokens = getFileTokens(pathname, state, {
+                    ...options,
+                    content: included, // The content forces the function to use it instead of reading from the disk
+                });
 
                 let includedTokens;
                 if (hash) {
@@ -92,7 +108,7 @@ function unfoldIncludes(state: StateCore, path: string, options: Options) {
     }
 }
 
-const index: MarkdownItPluginCb<Options> = (md, options) => {
+const index: MarkdownItPluginCb<Options> = (md: MarkdownItIncluded, options) => {
     const {path: optPath, log} = options;
 
     const plugin = (state: StateCore) => {
@@ -113,7 +129,7 @@ const index: MarkdownItPluginCb<Options> = (md, options) => {
         }
 
         env.includes.push(path);
-        unfoldIncludes(state, path, options);
+        unfoldIncludes(md, state, path, options);
         env.includes.pop();
     };
 

package/src/transform/plugins/includes/types.ts

@@ -0,0 +1,7 @@
+import {MarkdownIt} from '../../typings';
+
+export interface MarkdownItIncluded extends MarkdownIt {
+    included?: {
+        [key: string]: string;
+    };
+}

package/src/transform/plugins/links/collect.ts

@@ -1,8 +1,10 @@
 import MarkdownIt from 'markdown-it';
 import {sep} from 'path';
 import url from 'url';
+
 import {PAGE_LINK_REGEXP, getHrefTokenAttr, isLocalUrl} from '../../utils';
 import {getSinglePageAnchorId, resolveRelativePath} from '../../utilsFS';
+
 import index from './index';
 
 const replaceLinkHref = (input: string, href: string, newHref: string) => {

package/src/transform/plugins/links/index.ts

@@ -1,5 +1,10 @@
 import url from 'url';
 import {bold} from 'chalk';
+import Token from 'markdown-it/lib/token';
+import path, {isAbsolute, parse, relative, resolve} from 'path';
+
+import {Logger} from 'src/transform/log';
+
 import {
     PAGE_LINK_REGEXP,
     defaultTransformLink,
@@ -9,11 +14,8 @@ import {
     isLocalUrl,
 } from '../../utils';
 import {getFileTokens, isFileExists} from '../../utilsFS';
-import Token from 'markdown-it/lib/token';
-import {Logger} from 'src/transform/log';
 import {CacheContext, StateCore} from '../../typings';
 import {MarkdownItPluginCb, MarkdownItPluginOpts} from '../typings';
-import path, {isAbsolute, parse, relative, resolve} from 'path';
 
 function getTitleFromTokens(tokens: Token[]) {
     let title = '';

package/src/transform/plugins/meta.ts

@@ -1,5 +1,6 @@
 // @ts-expect-error
 import meta from 'markdown-it-meta';
+
 import {MarkdownItPluginCb} from './typings';
 
 export = meta as MarkdownItPluginCb;

package/src/transform/plugins/monospace.ts

@@ -1,4 +1,5 @@
 import StateInline, {Delimiter} from 'markdown-it/lib/rules_inline/state_inline';
+
 import {MarkdownItPluginCb} from './typings';
 
 const monospace: MarkdownItPluginCb = (md) => {
@@ -108,7 +109,6 @@ const monospace: MarkdownItPluginCb = (md) => {
             state.delimiters.push({
                 marker: marker,
                 length: 0, // disable "rule of 3" length checks meant for emphasis
-                jump: i / 2, // 1 delimiter = 2 characters
                 token: state.tokens.length - 1,
                 end: -1,
                 open: scanned.can_open,

package/src/transform/plugins/notes/index.ts

@@ -1,9 +1,10 @@
 import {bold} from 'chalk';
 import StateCore from 'markdown-it/lib/rules_core/state_core';
 import Token from 'markdown-it/lib/token';
-import {MarkdownItPluginCb} from '../typings';
 
+import {MarkdownItPluginCb} from '../typings';
 import {MatchTokenFunction, nestedCloseTokenIdxFactory as closeTokenFactory} from '../utils';
+
 import {TITLES} from './constants';
 
 const ALERT_RE = /^{% note (alert|info|tip|warning)\s*(?:"(.*?)")? %}$/;
@@ -84,7 +85,6 @@ const index: MarkdownItPluginCb = (md, {lang, notesAutotitle, path: optPath, log
                 const titleInline = new state.Token('inline', '', 0);
                 const titleClose = new state.Token('yfm_note_title_close', 'p', -1);
 
-                if (match[2]) titleOpen.attrSet('yfm2xliff-explicit', 'true');
                 titleOpen.block = true;
                 titleClose.block = true;
 

package/src/transform/plugins/sup.ts

@@ -1,5 +1,6 @@
 // @ts-expect-error
 import sup from 'markdown-it-sup';
+
 import {MarkdownItPluginCb} from './typings';
 
 export = sup as MarkdownItPluginCb;

package/src/transform/plugins/table/index.ts

@@ -1,6 +1,8 @@
 import StateBlock from 'markdown-it/lib/rules_block/state_block';
-import {MarkdownItPluginCb} from '../typings';
 import Token from 'markdown-it/lib/token';
+
+import {MarkdownItPluginCb} from '../typings';
+
 import {AttrsParser} from './attrs';
 
 const pluginName = 'yfm_table';

package/src/transform/plugins/term/index.ts

@@ -3,6 +3,7 @@ import Token from 'markdown-it/lib/token';
 
 import {MarkdownItPluginCb} from '../typings';
 import {generateID} from '../utils';
+
 import {termDefinitions} from './termDefinitions';
 import {BASIC_TERM_REGEXP} from './constants';
 

package/src/transform/plugins/term/termDefinitions.ts

@@ -1,6 +1,8 @@
 import StateBlock from 'markdown-it/lib/rules_block/state_block';
+
 import {MarkdownIt} from '../../typings';
 import {MarkdownItPluginOpts} from '../typings';
+
 import {BASIC_TERM_REGEXP} from './constants';
 
 export function termDefinitions(md: MarkdownIt, options: MarkdownItPluginOpts) {

package/src/transform/plugins/typings.ts

@@ -1,16 +1 @@
-import type {Logger} from '../log';
-import type {CacheContext, MarkdownIt} from '../typings';
-
-export interface MarkdownItPluginOpts {
-    path: string;
-    log: Logger;
-    lang: 'ru' | 'en' | 'es' | 'fr' | 'cs' | 'ar' | 'he';
-    root: string;
-    rootPublicPath: string;
-    isLintRun: boolean;
-    cache?: CacheContext;
-}
-
-export type MarkdownItPluginCb<T extends {} = {}> = {
-    (md: MarkdownIt, opts: T & MarkdownItPluginOpts): void;
-};
+export * from '../typings'; // TODO: Remove in major release

package/src/transform/plugins/utils.ts

@@ -1,8 +1,9 @@
 import {bold} from 'chalk';
-import {Logger} from '../log';
 import {platform} from 'process';
 import Token from 'markdown-it/lib/token';
 
+import {Logger} from '../log';
+
 export type MatchTokenFunction = (
     tokens: Token[],
     idx: number,

package/src/transform/plugins/video/const.ts

@@ -1,4 +1,5 @@
 import type {VideoFullOptions} from './types';
+
 import {videoUrl} from './utils';
 
 /** Supported services */
@@ -8,6 +9,8 @@ export enum VideoService {
     Vine = 'vine',
     Prezi = 'prezi',
     Osf = 'osf',
+    Yandex = 'yandex',
+    Vk = 'vk',
 }
 
 export const defaults: VideoFullOptions = {
@@ -17,4 +20,6 @@ export const defaults: VideoFullOptions = {
     vine: {width: 600, height: 600, embed: 'simple'},
     prezi: {width: 550, height: 400},
     osf: {width: '100%', height: '100%'},
+    yandex: {width: 640, height: 390},
+    vk: {width: 640, height: 390},
 };

package/src/transform/plugins/video/index.ts

@@ -3,14 +3,16 @@
 // Process @[vine](vineVideoID)
 // Process @[prezi](preziID)
 // Process @[osf](guid)
+// Process @[yandex](videoID)
+// Process @[vk](videoID)
 
 import type MarkdownIt from 'markdown-it';
 // eslint-disable-next-line no-duplicate-imports
 import type {PluginWithOptions} from 'markdown-it';
 import type ParserInline from 'markdown-it/lib/parser_inline';
 import type Renderer from 'markdown-it/lib/renderer';
-
 import type {VideoFullOptions, VideoPluginOptions, VideoToken} from './types';
+
 import {parseVideoUrl} from './parsers';
 import {VideoService, defaults} from './const';
 
@@ -69,15 +71,11 @@ function tokenizeVideo(md: MarkdownIt, options: VideoFullOptions): Renderer.Rend
 
         return videoID === ''
             ? ''
-            : '<div class="embed-responsive embed-responsive-16by9"><iframe class="embed-responsive-item ' +
-                  service +
-                  '-player" type="text/html" width="' +
-                  width +
-                  '" height="' +
-                  height +
-                  '" src="' +
-                  options.url(service, videoID, options) +
-                  '" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe></div>';
+            : `<div class="embed-responsive embed-responsive-16by9"><iframe` +
+                  ` class="embed-responsive-item ${service}-player"` +
+                  ` type="text/html" width="${width}" height="${height}"` +
+                  ` src="${options.url(service, videoID, options)}"` +
+                  ` frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe></div>`;
     };
 }
 

package/src/transform/plugins/video/parsers.ts

@@ -33,6 +33,18 @@ export function mfrParser(url: string) {
     return match ? match[1] : url;
 }
 
+const yandexRegex = /^https:\/\/runtime.video.cloud.yandex.net\/player\/video\/([a-zA-Z0-9]+)/;
+export function yandexParser(url: string) {
+    const match = url.match(yandexRegex);
+    return match ? match[1] : url;
+}
+
+const vkRegex = /^https:\/\/vk.com\/video_ext\.php?(oid=[-\d]+&id=[-\d]+)/;
+export function vkParser(url: string) {
+    const match = url.match(vkRegex);
+    return match ? match[1] : url;
+}
+
 export function parseVideoUrl(service: string, url: string): string | false {
     let videoID = '';
 
@@ -52,6 +64,12 @@ export function parseVideoUrl(service: string, url: string): string | false {
         case VideoService.Osf:
             videoID = mfrParser(url);
             break;
+        case VideoService.Yandex:
+            videoID = yandexParser(url);
+            break;
+        case VideoService.Vk:
+            videoID = vkParser(url);
+            break;
         default:
             return false;
     }

package/src/transform/plugins/video/types.ts

@@ -12,6 +12,8 @@ export type VideoServicesOptions = {
     [VideoService.Vine]: {width: number; height: number; embed: 'simple' | string};
     [VideoService.Prezi]: {width: number; height: number};
     [VideoService.Osf]: {width: string; height: string};
+    [VideoService.Yandex]: {width: number; height: number};
+    [VideoService.Vk]: {width: number; height: number};
 };
 
 export type VideoFullOptions = VideoServicesOptions & {

package/src/transform/plugins/video/utils.ts

@@ -3,21 +3,24 @@ import type {VideoUrlFn} from './types';
 export const videoUrl: VideoUrlFn = (service, videoID, options) => {
     switch (service) {
         case 'youtube':
-            return 'https://www.youtube.com/embed/' + videoID;
+            return `https://www.youtube.com/embed/${videoID}`;
         case 'vimeo':
-            return 'https://player.vimeo.com/video/' + videoID;
+            return `https://player.vimeo.com/video/${videoID}`;
         case 'vine':
-            return 'https://vine.co/v/' + videoID + '/embed/' + options.vine.embed;
+            return `https://vine.co/v/${videoID}/embed/${options.vine.embed}`;
         case 'prezi':
             return (
-                'https://prezi.com/embed/' +
-                videoID +
+                `https://prezi.com/embed/${videoID}` +
                 '/?bgcolor=ffffff&lock_to_path=0&autoplay=0&autohide_ctrls=0&' +
                 'landing_data=bHVZZmNaNDBIWnNjdEVENDRhZDFNZGNIUE43MHdLNWpsdFJLb2ZHanI5N1lQVHkxSHFxazZ0UUNCRHloSXZROHh3PT0&' +
                 'landing_sign=1kD6c0N6aYpMUS0wxnQjxzSqZlEB8qNFdxtdjYhwSuI'
             );
         case 'osf':
-            return 'https://mfr.osf.io/render?url=https://osf.io/' + videoID + '/?action=download';
+            return `https://mfr.osf.io/render?url=https://osf.io/${videoID}/?action=download`;
+        case 'yandex':
+            return `https://runtime.video.cloud.yandex.net/player/${videoID}`;
+        case 'vk':
+            return `https://vk.com/video_ext.php?${videoID}`;
         default:
             return service;
     }

package/src/transform/preprocessors/included/index.ts

@@ -0,0 +1,101 @@
+import {getFullIncludePath} from '../../utilsFS';
+import {MarkdownItPreprocessorCb} from '../../typings';
+import {MarkdownItIncluded} from '../../plugins/includes/types';
+
+const INCLUDE_REGEXP = /^\s*{%\s*included\s*\((.+?)\)\s*%}\s*$/;
+const INCLUDE_END_REGEXP = /^\s*{% endincluded %}\s*$/;
+
+const preprocessLine = (
+    lines: string[],
+    start: number,
+    {
+        root,
+        path,
+    }: {
+        root?: string;
+        path?: string;
+    },
+    md?: MarkdownItIncluded,
+) => {
+    const hasIncludedCache = md && root && path;
+    const str = lines[start];
+    const match = str?.match(INCLUDE_REGEXP);
+
+    // Protect from unmatched results
+    if (!match) {
+        return false;
+    }
+
+    const includePathRelative = match[1];
+
+    // Protect from empty path
+    if (!includePathRelative) {
+        return false;
+    }
+
+    // Read all content from top to bottom(!) char of the included block
+    const data = [];
+    let line = start;
+    while (line < lines.length) {
+        line++;
+        const str = lines[line];
+        if (str === null) {
+            break;
+        }
+        if (str?.match(INCLUDE_END_REGEXP)) {
+            break;
+        }
+        data.push(str);
+    }
+
+    // No included cache for lint mode
+    if (hasIncludedCache) {
+        if (!md.included) {
+            md.included = {};
+        }
+
+        // Normalize the path to absolute
+        const includePath = getFullIncludePath(includePathRelative, root, path);
+
+        // Store the included content
+        md.included[includePath] = data.join('\n');
+    }
+
+    // Remove the content of the included file
+    lines.splice(start, data.length + 2);
+
+    return true;
+};
+
+const index: MarkdownItPreprocessorCb<{
+    included?: boolean;
+}> = (input, options, md?: MarkdownItIncluded) => {
+    const {included, path, root} = options;
+
+    // To reduce file reading we can include the file content into the generated content
+    if (included) {
+        const lines = input.split('\n') || [];
+
+        // The finction reads the files from bottom to top(!). It stops the loop if it does not have anything to swap.
+        // If the function finds something to process then it restarts the loop because the position of the last element has been moved.
+        // eslint-disable-next-line no-unmodified-loop-condition
+        while (input?.length) {
+            let hasChars = false;
+            for (let line = lines.length - 1; line >= 0; line--) {
+                hasChars = preprocessLine(lines, line, {path, root}, md);
+                if (hasChars) {
+                    break;
+                }
+            }
+            if (!hasChars) {
+                break;
+            }
+        }
+
+        input = lines.join('\n');
+    }
+
+    return input;
+};
+
+export = index;

package/src/transform/preprocessors.ts

@@ -0,0 +1,7 @@
+import type {MarkdownItPreprocessorCb} from './plugins/typings';
+
+import included from './preprocessors/included';
+
+const defaultPreprocessors = [included] as MarkdownItPreprocessorCb[];
+
+export = defaultPreprocessors;

package/src/transform/sanitize.ts

@@ -485,7 +485,7 @@ const defaultCssWhitelist = {
     '--method': true,
 };
 
-const yfmHtmlAttrs = ['note-type', 'yfm2xliff-explicit', 'term-key'];
+const yfmHtmlAttrs = ['note-type', 'term-key'];
 
 const allowedTags = Array.from(
     new Set([...htmlTags, ...svgTags, ...sanitizeHtml.defaults.allowedTags]),
@@ -518,7 +518,7 @@ export const defaultOptions: SanitizeOptions = {
 function sanitizeStyleTags(dom: cheerio.CheerioAPI, cssWhiteList: CssWhiteList) {
     const styleTags = dom('style');
 
-    styleTags.each((_index: number, element: cheerio.Element) => {
+    styleTags.each((_index, element) => {
         const styleText = dom(element).text();
 
         try {

package/src/transform/typings.ts

@@ -1,9 +1,9 @@
 import {LanguageFn} from 'highlight.js';
 import DefaultMarkdownIt from 'markdown-it';
 import DefaultStateCore from 'markdown-it/lib/rules_core/state_core';
+
 import {SanitizeOptions} from './sanitize';
-import {MarkdownItPluginCb} from './plugins/typings';
-import {LogLevels} from './log';
+import {LogLevels, Logger} from './log';
 import {ChangelogItem} from './plugins/changelog/types';
 
 export interface MarkdownIt extends DefaultMarkdownIt {
@@ -48,7 +48,9 @@ export interface OptionsType {
     needFlatListHeadings?: boolean;
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     plugins?: MarkdownItPluginCb<any>[];
+    preprocessors?: MarkdownItPreprocessorCb[]; // Preprocessors should modify the input before passing it to MD
     highlightLangs?: HighlightLangMap;
+    disableRules?: string[];
     extractChangelogs?: boolean;
     root?: string;
     rootPublicPath?: string;
@@ -73,3 +75,26 @@ export type EnvType<Extras extends {} = {}> = {
     meta?: object;
     changelogs?: ChangelogItem[];
 } & Extras;
+
+export interface MarkdownItPluginOpts {
+    path: string;
+    log: Logger;
+    lang: 'ru' | 'en' | 'es' | 'fr' | 'cs' | 'ar' | 'he';
+    root: string;
+    rootPublicPath: string;
+    isLintRun: boolean;
+    cache?: CacheContext;
+    conditionsInCode?: boolean;
+    vars?: Record<string, string>;
+    extractTitle?: boolean;
+    disableLiquid?: boolean;
+}
+
+export type MarkdownItPluginCb<T extends {} = {}> = {
+    // TODO: use "T extends unknown = {}"
+    (md: MarkdownIt, opts: T & MarkdownItPluginOpts): void;
+};
+
+export type MarkdownItPreprocessorCb<T extends unknown = {}> = {
+    (input: string, opts: T & Partial<MarkdownItPluginOpts>, md?: MarkdownIt): string;
+};