@diplodoc/transform 4.0.0

package/src/transform/plugins/table/index.ts

@@ -0,0 +1,290 @@
+import StateBlock from 'markdown-it/lib/rules_block/state_block';
+import {MarkdownItPluginCb} from '../typings';
+
+const pluginName = 'yfm_table';
+const pipeChar = 0x7c; // |
+const apostropheChar = 0x60; // `
+const hashChar = 0x23; // #
+const backSlashChar = 0x5c; // \
+
+const checkCharsOrder = (order: number[], src: string, pos: number) => {
+    const currentOrder = [...order];
+    const currentSrc = src.slice(pos);
+    for (let i = 0; i < currentOrder.length; i++) {
+        const rowSymbol = currentSrc.charCodeAt(i);
+        const orderSymbol = currentOrder[i];
+        if (rowSymbol !== orderSymbol) {
+            return false;
+        }
+    }
+    return true;
+};
+
+type CheckFn = (src: string, pos: number) => boolean;
+
+const codeBlockOrder = [apostropheChar, apostropheChar, apostropheChar];
+const isCodeBlockOrder: CheckFn = (src, pos) => checkCharsOrder(codeBlockOrder, src, pos);
+
+const openTableOrder = [hashChar, pipeChar];
+const isOpenTableOrder: CheckFn = (src, pos) => checkCharsOrder(openTableOrder, src, pos);
+
+const notEscaped: CheckFn = (src, pos) => src.charCodeAt(pos - 1) !== backSlashChar;
+
+const rowStartOrder = [pipeChar, pipeChar];
+const isRowOrder: CheckFn = (src, pos) =>
+    checkCharsOrder(rowStartOrder, src, pos) && notEscaped(src, pos);
+
+const cellStartOrder = [pipeChar];
+const isCellOrder: CheckFn = (src, pos) =>
+    checkCharsOrder(cellStartOrder, src, pos) && notEscaped(src, pos) && !isRowOrder(src, pos);
+
+const closeTableOrder = [pipeChar, hashChar];
+const isCloseTableOrder: CheckFn = (src, pos) => checkCharsOrder(closeTableOrder, src, pos);
+
+type Stats = {line: number; pos: number};
+
+class StateIterator {
+    pos: number;
+    line: number;
+
+    private state: StateBlock;
+    private lineEnds: number;
+
+    constructor(state: StateBlock, pos: number, line: number) {
+        this.state = state;
+        this.line = line;
+        this.pos = pos;
+        this.lineEnds = this.state.eMarks[this.line];
+    }
+
+    stats(): Stats {
+        return {
+            line: this.line,
+            pos: this.pos,
+        };
+    }
+
+    get symbol() {
+        return this.state.src[this.pos];
+    }
+
+    next(steps = 1) {
+        for (let i = 0; i < steps; i++) {
+            this.pos++;
+
+            if (this.pos > this.lineEnds) {
+                this.line++;
+                this.pos = this.state.bMarks[this.line] + this.state.tShift[this.line];
+                this.lineEnds = this.state.eMarks[this.line];
+            }
+        }
+    }
+}
+
+function getTableRows(
+    state: StateBlock,
+    startPosition: number,
+    endPosition: number,
+    startLine: number,
+) {
+    let endOfTable = null;
+    let tableLevel = 0;
+    let currentRow: [Stats, Stats][] = [];
+    let colStart: Stats | null = null;
+    let rowStart: number | null = null;
+
+    const iter = new StateIterator(state, startPosition + openTableOrder.length, startLine);
+
+    const rows: [number, number, typeof currentRow][] = [];
+
+    let isInsideCode = false;
+    let isInsideTable = false;
+    const rowMap = new Map();
+
+    const addRow = () => {
+        if (colStart) {
+            currentRow.push([colStart, iter.stats()]);
+        }
+        if (currentRow.length && rowStart) {
+            rows.push([rowStart, iter.line, currentRow]);
+        }
+        currentRow = [];
+        colStart = null;
+        rowStart = null;
+    };
+
+    while (iter.pos <= endPosition) {
+        if (iter.symbol === undefined) {
+            break;
+        }
+
+        if (isCodeBlockOrder(state.src, iter.pos)) {
+            isInsideCode = !isInsideCode;
+            iter.next(codeBlockOrder.length);
+        }
+
+        if (isInsideCode) {
+            iter.next();
+            continue;
+        }
+
+        if (isOpenTableOrder(state.src, iter.pos)) {
+            isInsideTable = true;
+            tableLevel++;
+            iter.next(openTableOrder.length);
+            continue;
+        }
+
+        if (isCloseTableOrder(state.src, iter.pos)) {
+            if (tableLevel === 0) {
+                addRow();
+                iter.next(closeTableOrder.length);
+                endOfTable = iter.line + 2;
+                break;
+            } else {
+                isInsideTable = false;
+                tableLevel--;
+                iter.next(closeTableOrder.length);
+                continue;
+            }
+        }
+
+        if (isInsideTable) {
+            iter.next();
+            continue;
+        }
+
+        if (isRowOrder(state.src, iter.pos)) {
+            const insideRow = rowMap.get(tableLevel);
+            if (insideRow) {
+                addRow();
+                iter.next(rowStartOrder.length);
+            } else {
+                iter.next(rowStartOrder.length);
+                rowStart = iter.line;
+                colStart = iter.stats();
+            }
+
+            rowMap.set(tableLevel, !insideRow);
+
+            continue;
+        }
+
+        if (isCellOrder(state.src, iter.pos)) {
+            if (colStart) {
+                currentRow.push([colStart, iter.stats()]);
+            }
+            iter.next(cellStartOrder.length);
+            colStart = iter.stats();
+            continue;
+        }
+
+        iter.next();
+    }
+    return {rows, endOfTable};
+}
+
+const yfmTable: MarkdownItPluginCb = (md) => {
+    md.block.ruler.before(
+        'code',
+        pluginName,
+        (state: StateBlock, startLine: number, endLine: number, silent: boolean) => {
+            let token;
+            const startPosition = state.bMarks[startLine] + state.tShift[startLine];
+            const endPosition = state.eMarks[endLine];
+
+            // #| minimum 2 symbols
+            if (endPosition - startPosition < 2) {
+                return false;
+            }
+
+            if (!isOpenTableOrder(state.src, startPosition)) {
+                return false;
+            }
+
+            if (silent) {
+                return true;
+            }
+
+            const {rows, endOfTable} = getTableRows(state, startPosition, endPosition, startLine);
+
+            if (!endOfTable) {
+                token = state.push('__yfm_lint', '', 0);
+                token.hidden = true;
+                token.map = [startLine, endLine];
+                token.attrSet('YFM004', 'true');
+                return false;
+            }
+
+            const oldParentLineMax = state.lineMax;
+
+            state.lineMax = endOfTable;
+            state.line = startLine;
+
+            token = state.push('yfm_table_open', 'table', 1);
+            token.map = [startLine, endOfTable];
+
+            token = state.push('yfm_tbody_open', 'tbody', 1);
+            token.map = [startLine + 1, endOfTable - 1];
+
+            const maxRowLength = Math.max(...rows.map(([, , cols]) => cols.length));
+
+            for (let i = 0; i < rows.length; i++) {
+                const [rowLineStarts, rowLineEnds, cols] = rows[i];
+
+                const rowLength = cols.length;
+
+                token = state.push('yfm_tr_open', 'tr', 1);
+                token.map = [rowLineStarts, rowLineEnds];
+
+                for (let j = 0; j < cols.length; j++) {
+                    const [begin, end] = cols[j];
+                    token = state.push('yfm_td_open', 'td', 1);
+                    token.map = [begin.line, end.line];
+
+                    const oldTshift = state.tShift[begin.line];
+                    const oldEMark = state.eMarks[end.line];
+                    const oldBMark = state.bMarks[begin.line];
+                    const oldLineMax = state.lineMax;
+
+                    state.tShift[begin.line] = 0;
+                    state.bMarks[begin.line] = begin.pos;
+                    state.eMarks[end.line] = end.pos;
+                    state.lineMax = end.line + 1;
+
+                    state.md.block.tokenize(state, begin.line, end.line + 1);
+
+                    state.lineMax = oldLineMax;
+                    state.tShift[begin.line] = oldTshift;
+                    state.bMarks[begin.line] = oldBMark;
+                    state.eMarks[end.line] = oldEMark;
+
+                    token = state.push('yfm_td_close', 'td', -1);
+                    state.tokens[state.tokens.length - 1].map = [end.line, end.line + 1];
+                }
+
+                if (rowLength < maxRowLength) {
+                    const emptyCellsCount = maxRowLength - rowLength;
+                    for (let k = 0; k < emptyCellsCount; k++) {
+                        token = state.push('yfm_td_open', 'td', 1);
+                        token = state.push('yfm_td_close', 'td', -1);
+                    }
+                }
+
+                token = state.push('yfm_tr_close', 'tr', -1);
+            }
+
+            token = state.push('yfm_tbody_close', 'tbody', -1);
+
+            token = state.push('yfm_table_close', 'table', -1);
+            state.tokens[state.tokens.length - 1].map = [endOfTable, endOfTable + 1];
+
+            state.lineMax = oldParentLineMax;
+            state.line = endOfTable;
+
+            return true;
+        },
+    );
+};
+
+export = yfmTable;

package/src/transform/plugins/tabs.ts

@@ -0,0 +1,3 @@
+import {transform} from '@diplodoc/tabs-extension';
+
+export = transform({bundle: false});

package/src/transform/plugins/term/constants.ts

@@ -0,0 +1 @@
+export const BASIC_TERM_REGEXP = '\\[([^\\[]+)\\](\\(\\*(\\w+)\\))';

package/src/transform/plugins/term/index.ts

@@ -0,0 +1,139 @@
+import StateCore from 'markdown-it/lib/rules_core/state_core';
+import Token from 'markdown-it/lib/token';
+
+import {MarkdownItPluginCb} from '../typings';
+import {generateID} from '../utils';
+import {termDefinitions} from './termDefinitions';
+import {BASIC_TERM_REGEXP} from './constants';
+
+const term: MarkdownItPluginCb = (md, options) => {
+    const escapeRE = md.utils.escapeRE;
+    const arrayReplaceAt = md.utils.arrayReplaceAt;
+
+    const {isLintRun} = options;
+    // Don't parse urls that starts with *
+    const defaultLinkValidation = md.validateLink;
+    md.validateLink = function (url) {
+        if (url.startsWith('*')) {
+            return false;
+        }
+
+        return defaultLinkValidation(url);
+    };
+
+    function termReplace(state: StateCore) {
+        let i, j, l, tokens, token, text, nodes, pos, term, currentToken;
+
+        const blockTokens = state.tokens;
+
+        if (!state.env.terms) {
+            return;
+        }
+
+        const regTerms = Object.keys(state.env.terms)
+            .map((el) => el.substr(1))
+            .map(escapeRE)
+            .join('|');
+        const regText = '\\[([^\\[]+)\\](\\(\\*(' + regTerms + ')\\))';
+        const reg = new RegExp(regText, 'g');
+
+        for (j = 0, l = blockTokens.length; j < l; j++) {
+            if (blockTokens[j].type === 'heading_open') {
+                while (blockTokens[j].type !== 'heading_close') {
+                    j++;
+                }
+                continue;
+            }
+
+            if (blockTokens[j].type !== 'inline') {
+                continue;
+            }
+
+            tokens = blockTokens[j].children as Token[];
+
+            for (i = tokens.length - 1; i >= 0; i--) {
+                currentToken = tokens[i];
+                if (currentToken.type === 'link_close') {
+                    while (tokens[i].type !== 'link_open') {
+                        i--;
+                    }
+                    continue;
+                }
+
+                if (!(currentToken.type === 'text')) {
+                    continue;
+                }
+
+                pos = 0;
+                text = currentToken.content;
+                reg.lastIndex = 0;
+                nodes = [];
+
+                // Find terms without definitions
+                const regexAllTerms = new RegExp(BASIC_TERM_REGEXP, 'gm');
+                const uniqueTerms = [
+                    ...new Set([...text.matchAll(regexAllTerms)].map((el) => `:${el[3]}`)),
+                ];
+                const notDefinedTerms = uniqueTerms.filter(
+                    (el) => !Object.keys(state.env.terms).includes(el),
+                );
+
+                if (notDefinedTerms.length && isLintRun) {
+                    token = new state.Token('__yfm_lint', '', 0);
+                    token.hidden = true;
+                    token.map = blockTokens[j].map;
+                    token.attrSet('YFM007', 'true');
+                    nodes.push(token);
+                }
+
+                while ((term = reg.exec(text))) {
+                    const termTitle = term[1];
+                    const termKey = term[3];
+
+                    if (term.index > 0 || term[1].length > 0) {
+                        token = new state.Token('text', '', 0);
+                        token.content = text.slice(pos, term.index);
+                        nodes.push(token);
+                    }
+
+                    token = new state.Token('term_open', 'i', 1);
+                    token.attrSet('class', 'yfm yfm-term_title');
+                    token.attrSet('term-key', ':' + termKey);
+                    token.attrSet('aria-describedby', ':' + termKey + '_element');
+                    token.attrSet('id', generateID());
+                    nodes.push(token);
+
+                    token = new state.Token('text', '', 0);
+                    token.content = termTitle;
+                    nodes.push(token);
+
+                    token = new state.Token('term_close', 'i', -1);
+                    nodes.push(token);
+
+                    pos = reg.lastIndex;
+                }
+
+                if (!nodes.length) {
+                    continue;
+                }
+
+                if (pos < text.length) {
+                    token = new state.Token('text', '', 0);
+                    token.content = text.slice(pos);
+                    nodes.push(token);
+                }
+
+                // replace current node
+                blockTokens[j].children = tokens = arrayReplaceAt(tokens, i, nodes);
+            }
+        }
+    }
+
+    md.block.ruler.before('reference', 'termDefinitions', termDefinitions(md, options), {
+        alt: ['paragraph', 'reference'],
+    });
+
+    md.core.ruler.after('linkify', 'termReplace', termReplace);
+};
+
+export = term;

package/src/transform/plugins/term/termDefinitions.ts

@@ -0,0 +1,177 @@
+import StateBlock from 'markdown-it/lib/rules_block/state_block';
+import {MarkdownIt} from '../../typings';
+import {MarkdownItPluginOpts} from '../typings';
+import {BASIC_TERM_REGEXP} from './constants';
+
+export function termDefinitions(md: MarkdownIt, options: MarkdownItPluginOpts) {
+    return (state: StateBlock, startLine: number, endLine: number, silent: boolean) => {
+        let ch;
+        let labelEnd;
+        let pos = state.bMarks[startLine] + state.tShift[startLine];
+        let max = state.eMarks[startLine];
+
+        if (pos + 2 >= max) {
+            return false;
+        }
+
+        if (state.src.charCodeAt(pos++) !== 0x5b /* [ */) {
+            return false;
+        }
+        if (state.src.charCodeAt(pos++) !== 0x2a /* * */) {
+            return false;
+        }
+
+        const labelStart = pos;
+
+        for (; pos < max; pos++) {
+            ch = state.src.charCodeAt(pos);
+            if (ch === 0x5b /* [ */) {
+                return false;
+            } else if (ch === 0x5d /* ] */) {
+                labelEnd = pos;
+                break;
+            } else if (ch === 0x5c /* \ */) {
+                pos++;
+            }
+        }
+
+        const newLineReg = new RegExp(/^(\r\n|\r|\n)/);
+        const termReg = new RegExp(/^\[\*(\w+)\]:/);
+        let currentLine = startLine;
+
+        // Allow multiline term definition
+        for (; currentLine < endLine; currentLine++) {
+            const nextLineStart = state.bMarks[currentLine + 1];
+            const nextLineEnd = state.eMarks[currentLine + 1];
+
+            const nextLine =
+                nextLineStart === nextLineEnd
+                    ? state.src[nextLineStart]
+                    : state.src.slice(nextLineStart, nextLineEnd);
+
+            if (newLineReg.test(nextLine) || termReg.test(nextLine)) {
+                break;
+            }
+
+            state.line = currentLine + 1;
+        }
+
+        max = state.eMarks[currentLine];
+
+        if (!labelEnd || labelEnd < 0 || state.src.charCodeAt(labelEnd + 1) !== 0x3a /* : */) {
+            return false;
+        }
+
+        if (silent) {
+            return true;
+        }
+
+        const label = state.src.slice(labelStart, labelEnd).replace(/\\(.)/g, '$1');
+        const title = state.src.slice(labelEnd + 2, max).trim();
+
+        if (label.length === 0 || title.length === 0) {
+            return false;
+        }
+
+        return processTermDefinition(
+            md,
+            options,
+            state,
+            currentLine,
+            startLine,
+            endLine,
+            label,
+            title,
+        );
+    };
+}
+
+function processTermDefinition(
+    md: MarkdownIt,
+    options: MarkdownItPluginOpts,
+    state: StateBlock,
+    currentLine: number,
+    startLine: number,
+    endLine: number,
+    label: string,
+    title: string,
+) {
+    let token;
+
+    if (!state.env.terms) {
+        state.env.terms = {};
+    }
+
+    const basicTermDefinitionRegexp = new RegExp(BASIC_TERM_REGEXP, 'gm');
+    // If term inside definition
+
+    const {isLintRun} = options;
+
+    if (basicTermDefinitionRegexp.test(title) && isLintRun) {
+        token = new state.Token('__yfm_lint', '', 0);
+        token.hidden = true;
+        token.map = [currentLine, endLine];
+        token.attrSet('YFM008', 'true');
+        state.tokens.push(token);
+    }
+
+    // If term definition duplicated
+    if (state.env.terms[':' + label] && isLintRun) {
+        token = new state.Token('__yfm_lint', '', 0);
+        token.hidden = true;
+        token.map = [currentLine, endLine];
+        token.attrSet('YFM006', 'true');
+        state.tokens.push(token);
+        state.line = currentLine + 1;
+        return true;
+    }
+
+    if (typeof state.env.terms[':' + label] === 'undefined') {
+        state.env.terms[':' + label] = title;
+    }
+
+    token = new state.Token('template_open', 'template', 1);
+    token.map = [startLine, currentLine + 1];
+    token.attrSet('id', ':' + label + '_template');
+    token.attrSet('label', label);
+
+    state.tokens.push(token);
+
+    token = new state.Token('dfn_open', 'dfn', 1);
+    token.attrSet('class', 'yfm yfm-term_dfn');
+    token.attrSet('id', ':' + label + '_element');
+    token.attrSet('role', 'tooltip');
+
+    state.tokens.push(token);
+
+    const titleTokens = md.parse(title, state.env);
+
+    for (const titleToken of titleTokens) {
+        if (titleToken.children?.length) {
+            titleToken.content = '';
+        }
+
+        if (!titleToken.map) {
+            state.tokens.push(titleToken);
+            continue;
+        }
+
+        const [start, end] = titleToken.map;
+
+        titleToken.map = [start + startLine, end + startLine];
+        state.tokens.push(titleToken);
+    }
+
+    token = new state.Token('dfn_close', 'dfn', -1);
+
+    state.tokens.push(token);
+
+    token = new state.Token('template_close', 'template', -1);
+
+    state.tokens.push(token);
+
+    /** current line links to end of term definition */
+    state.line = currentLine + 1;
+
+    return true;
+}

package/src/transform/plugins/typings.ts

@@ -0,0 +1,14 @@
+import type {Logger} from '../log';
+import type {MarkdownIt} from '../typings';
+
+export interface MarkdownItPluginOpts {
+    path: string;
+    log: Logger;
+    lang: 'ru' | 'en';
+    root: string;
+    isLintRun: boolean;
+}
+
+export type MarkdownItPluginCb<T extends {} = {}> = {
+    (md: MarkdownIt, opts: T & MarkdownItPluginOpts): void;
+};

package/src/transform/plugins/utils.ts

@@ -0,0 +1,38 @@
+import {bold} from 'chalk';
+import {Logger} from '../log';
+import {platform} from 'process';
+import Token from 'markdown-it/lib/token';
+
+export type MatchTokenFunction = (
+    tokens: Token[],
+    idx: number,
+) => boolean | null | RegExpMatchArray;
+
+export const nestedCloseTokenIdxFactory =
+    (tokenName: string, matchOpenToken: MatchTokenFunction, matchCloseToken: MatchTokenFunction) =>
+    (tokens: Token[], idx: number, path: string, log: Logger) => {
+        let level = 0;
+        let i = idx;
+        while (i < tokens.length) {
+            if (matchOpenToken(tokens, i)) {
+                level++;
+            } else if (matchCloseToken(tokens, i)) {
+                if (level === 0) {
+                    return i;
+                }
+                level--;
+            }
+
+            i++;
+        }
+
+        log.error(`${tokenName} must be closed${path ? ` in ${bold(path)}` : ''}`);
+
+        return null;
+    };
+
+export const сarriage = platform === 'win32' ? '\r\n' : '\n';
+
+export function generateID() {
+    return Math.random().toString(36).substr(2, 8);
+}

package/src/transform/plugins/video/const.ts

@@ -0,0 +1,20 @@
+import type {VideoFullOptions} from './types';
+import {videoUrl} from './utils';
+
+/** Supported services */
+export enum VideoService {
+    YouTube = 'youtube',
+    Vimeo = 'vimeo',
+    Vine = 'vine',
+    Prezi = 'prezi',
+    Osf = 'osf',
+}
+
+export const defaults: VideoFullOptions = {
+    url: videoUrl,
+    youtube: {width: 640, height: 390},
+    vimeo: {width: 500, height: 281},
+    vine: {width: 600, height: 600, embed: 'simple'},
+    prezi: {width: 550, height: 400},
+    osf: {width: '100%', height: '100%'},
+};