@diplodoc/transform 4.0.0

package/src/transform/liquid/index.ts

@@ -0,0 +1,141 @@
+import applySubstitutions from './substitutions';
+import {prepareSourceMap} from './sourceMap';
+
+import applyCycles from './cycles';
+import applyConditions from './conditions';
+
+import ArgvService, {ArgvSettings} from './services/argv';
+import {Dictionary} from 'lodash';
+
+const fence = '```';
+
+const find = (open: string, close: string, string: string, index: number) => {
+    const start = string.indexOf(open, index);
+    const end = start > -1 ? string.indexOf(close, start + open.length) : -1;
+
+    return [start, end];
+};
+
+const replace = (
+    open: string,
+    close: string,
+    value: (string: string) => string,
+    string: string,
+) => {
+    let result = '';
+    let carriage = 0;
+    let [start, end] = find(open, close, string, carriage);
+
+    while (start > -1 && end > -1) {
+        const fragment = string.slice(start + open.length, end);
+
+        result += string.slice(carriage, start) + open + value(fragment) + close;
+        carriage = end + close.length;
+        [start, end] = find(open, close, string, carriage);
+    }
+
+    result += string.slice(carriage);
+
+    return result;
+};
+
+function saveCode(
+    str: string,
+    vars: Record<string, unknown>,
+    codes: string[],
+    path?: string,
+    substitutions?: boolean,
+) {
+    return replace(
+        fence,
+        fence,
+        (code) => {
+            const codeWithVars = substitutions ? applySubstitutions(code, vars, path) : code;
+            const index = codes.push(codeWithVars) - 1;
+
+            /* Keep the same count of lines to avoid transformation of the source map */
+            const codeLines = codeWithVars.split('\n');
+            const emptyLines = codeLines.length > 1 ? '\n'.repeat(codeLines.length) : '';
+
+            return `${index}${emptyLines}`;
+        },
+        str,
+    );
+}
+
+function repairCode(str: string, codes: string[]) {
+    return replace(fence, fence, (code) => codes[Number(code)], str);
+}
+
+function liquid<
+    B extends boolean = false,
+    C = B extends false ? string : {output: string; sourceMap: Dictionary<string>},
+>(
+    originInput: string,
+    vars: Record<string, unknown>,
+    path?: string,
+    settings?: ArgvSettings & {withSourceMap?: B},
+): C {
+    const {
+        cycles = true,
+        conditions = true,
+        substitutions = true,
+        conditionsInCode = false,
+        keepNotVar = false,
+        withSourceMap,
+    } = settings || {};
+
+    ArgvService.init({
+        cycles,
+        conditions,
+        substitutions,
+        conditionsInCode,
+        keepNotVar,
+        withSourceMap,
+    });
+
+    const codes: string[] = [];
+
+    let output = conditionsInCode
+        ? originInput
+        : saveCode(originInput, vars, codes, path, substitutions);
+
+    let sourceMap: Record<number, number> = {};
+
+    if (withSourceMap) {
+        const lines = output.split('\n');
+        sourceMap = lines.reduce((acc: Record<number, number>, _cur, index) => {
+            acc[index + 1] = index + 1;
+
+            return acc;
+        }, {});
+    }
+
+    if (cycles) {
+        output = applyCycles(output, vars, path, {sourceMap});
+    }
+
+    if (conditions) {
+        output = applyConditions(output, vars, path, {sourceMap});
+    }
+
+    if (substitutions) {
+        output = applySubstitutions(output, vars, path);
+    }
+
+    output = conditionsInCode ? output : repairCode(output, codes);
+    codes.length = 0;
+
+    if (withSourceMap) {
+        return {
+            output,
+            sourceMap: prepareSourceMap(sourceMap),
+        } as unknown as C;
+    }
+
+    return output as unknown as C;
+}
+
+// 'export default' instead of 'export = ' because of circular dependency with './cycles.ts'.
+// somehow it breaks import in './cycles.ts' and imports nothing
+export default liquid;

package/src/transform/liquid/lexical.ts

@@ -0,0 +1,86 @@
+// quote related
+const singleQuoted = /'[^']*'/;
+const doubleQuoted = /"[^"]*"/;
+
+const quoted = new RegExp(`${singleQuoted.source}|${doubleQuoted.source}`);
+export const quoteBalanced = new RegExp(`(?:${quoted.source}|[^'"])*`);
+
+export const vars = /((not_var)?({{2}([. \w-|(),]+)}{2}))/gm;
+
+// basic types
+const number = /-?\d+\.?\d*|\.?\d+/;
+const bool = /true|false/;
+
+// property access
+const identifier = /[\w-|]+[?]?/;
+const subscript = new RegExp(`\\[(?:${quoted.source}|[\\w-\\.]+)\\]`);
+const literal = new RegExp(`(?:${quoted.source}|${bool.source}|${number.source})`);
+export const variable = new RegExp(
+    `${identifier.source}(?:\\.${identifier.source}|${subscript.source})*`,
+);
+
+// range related
+const rangeLimit = new RegExp(`(?:${variable.source}|${number.source})`);
+const rangeCapture = new RegExp(`\\((${rangeLimit.source})\\.\\.(${rangeLimit.source})\\)`);
+
+// full match
+export const tagLine = new RegExp(`^\\s*(${identifier.source})\\s*([\\s\\S]*)\\s*$`);
+const literalLine = new RegExp(`^${literal.source}$`, 'i');
+const variableLine = new RegExp(`^${variable.source}$`);
+const numberLine = new RegExp(`^${number.source}$`);
+const boolLine = new RegExp(`^${bool.source}$`, 'i');
+const quotedLine = new RegExp(`^${quoted.source}$`);
+export const rangeLine = new RegExp(`^${rangeCapture.source}$`);
+
+export const operators = [
+    /\s+or\s+/,
+    /\s+and\s+/,
+    /[=]=|!=|<=|>=|<|>|\s+contains\s+/,
+    /\s+\|\s+/, // myVar | filterName
+    /\s+\|/, // myVar |filterName
+    /\|\s+/, // myVar| filterName
+    /\./,
+];
+
+const prepareArgsForMethods = {
+    slice: (args: string[]) => args.map((arg) => Number(arg)),
+};
+const supportedMethods = Object.keys(prepareArgsForMethods);
+const supportedMethodsRE = new RegExp(`^(${supportedMethods.join('\\|')})\\(([^)]*)\\)$`);
+export const isSupportedMethod = (exp: string) => {
+    return supportedMethodsRE.test(exp);
+};
+export const getParsedMethod = (exp: String) => {
+    const match = exp.match(supportedMethodsRE);
+
+    if (!match) return null;
+
+    const name = match[1];
+    const args = match[2].split(/[\s,]+/);
+
+    return {
+        name,
+        args,
+    };
+};
+
+export const isLiteral = (str: string) => literalLine.test(str);
+export const isVariable = (str: string) => variableLine.test(str);
+
+export function parseLiteral(str: string) {
+    let res = str.match(numberLine);
+
+    if (res) {
+        return Number(str);
+    }
+    res = str.match(boolLine);
+    if (res) {
+        return str.toLowerCase() === 'true';
+    }
+    res = str.match(quotedLine);
+    if (res) {
+        return str.slice(1, -1);
+    }
+
+    throw new TypeError(`cannot parse '${str}' as literal`);
+}

package/src/transform/liquid/services/argv.ts

@@ -0,0 +1,23 @@
+export type ArgvSettings = {
+    conditions?: boolean;
+    conditionsInCode?: boolean;
+    cycles?: boolean;
+    substitutions?: boolean;
+    keepNotVar?: boolean;
+    withSourceMap?: boolean;
+};
+
+let _argv: ArgvSettings;
+
+export function getConfig() {
+    return _argv || {};
+}
+
+export function init(argv: ArgvSettings) {
+    _argv = {...argv};
+}
+
+export default {
+    init,
+    getConfig,
+};

package/src/transform/liquid/sourceMap.ts

@@ -0,0 +1,72 @@
+import invert from 'lodash/invert';
+
+const DELETED_LINE = Number.MIN_SAFE_INTEGER;
+
+export function getLineNumber(input: string, index: number) {
+    return input.substring(0, index).split('\n').length;
+}
+
+export function prepareSourceMap(sourceMap: object) {
+    const newToOldMap = invert(sourceMap);
+    delete newToOldMap[DELETED_LINE];
+
+    return newToOldMap;
+}
+
+type Options = {
+    firstLineNumber: number;
+    lastLineNumber: number;
+    sourceMap: Record<number, number>;
+};
+
+type MoveLinesOptions = {
+    start: number;
+    end: number;
+    offset: number;
+    withReplace?: boolean;
+};
+
+export function createSourceMapApi({firstLineNumber, lastLineNumber, sourceMap}: Options) {
+    const isInlineTag = firstLineNumber === lastLineNumber;
+    const newToOldIndexes = invert(sourceMap);
+
+    const getOriginIndex = (i: number) => Number(newToOldIndexes[i]);
+
+    const setSourceMapValue = (i: number, value: number) => {
+        sourceMap[getOriginIndex(i)] = value;
+    };
+    const getSourceMapValue = (i: number) => sourceMap[getOriginIndex(i)];
+
+    const moveLines = ({start, end, offset, withReplace = false}: MoveLinesOptions) => {
+        if (isInlineTag) {
+            return;
+        }
+
+        for (let i = start; i <= end; i++) {
+            const newLineNumber = i + offset;
+            setSourceMapValue(i, newLineNumber);
+
+            if (withReplace) {
+                setSourceMapValue(newLineNumber, DELETED_LINE);
+            }
+        }
+    };
+
+    const removeLine = (i: number) => {
+        setSourceMapValue(i, DELETED_LINE);
+    };
+
+    const removeLines = ({start, end}: {start: number; end: number}) => {
+        for (let i = start; i <= end; i++) {
+            removeLine(i);
+        }
+    };
+
+    return {
+        isInlineTag,
+        getSourceMapValue,
+        moveLines,
+        removeLines,
+        removeLine,
+    };
+}

package/src/transform/liquid/substitutions.ts

@@ -0,0 +1,40 @@
+import {bold} from 'chalk';
+
+import ArgvService from './services/argv';
+import getObject from '../getObject';
+import {evalExp} from './evaluation';
+import {log} from '../log';
+import {vars as varsRe, isVariable} from './lexical';
+
+const substitutions = (str: string, builtVars: Record<string, unknown>, path?: string) => {
+    const {keepNotVar} = ArgvService.getConfig();
+
+    return str.replace(varsRe, (match, _groupNotVar, flag, groupVar, groupVarValue) => {
+        if (flag) {
+            return keepNotVar ? _groupNotVar : groupVar;
+        }
+
+        const trimVarPath = groupVarValue.trim();
+
+        if (trimVarPath.startsWith('.')) {
+            return groupVar;
+        }
+
+        let value;
+        if (isVariable(trimVarPath)) {
+            value = getObject(trimVarPath, builtVars);
+        } else {
+            value = evalExp(trimVarPath, builtVars);
+        }
+
+        if (value === undefined) {
+            value = match;
+
+            log.warn(`Variable ${bold(trimVarPath)} not found${path ? ` in ${bold(path)}` : ''}`);
+        }
+
+        return value;
+    });
+};
+
+export = substitutions;

package/src/transform/liquid/utils.ts

@@ -0,0 +1,30 @@
+export function removeIndentBlock(str: string) {
+    let i = str.length - 1;
+    let curChar = str[i];
+
+    while (curChar === ' ') {
+        curChar = str[--i];
+    }
+
+    if (curChar === '\n') {
+        return str.substring(0, i + 1);
+    }
+
+    return str;
+}
+
+type Opts = {
+    content: string;
+    tagStartPos: number;
+    tagContent: string;
+};
+
+export function getPreparedLeftContent({content, tagStartPos, tagContent}: Opts) {
+    const preparedLeftContent = content.substring(0, tagStartPos);
+
+    if (tagContent === '') {
+        return removeIndentBlock(preparedLeftContent);
+    }
+
+    return preparedLeftContent;
+}

package/src/transform/log.ts

@@ -0,0 +1,60 @@
+import {green, yellow, red} from 'chalk';
+
+export enum LogLevels {
+    INFO = 'info',
+    WARN = 'warn',
+    ERROR = 'error',
+    DISABLED = 'disabled',
+}
+
+const problems: Record<LogLevels, string[]> = {
+    [LogLevels.INFO]: [],
+    [LogLevels.WARN]: [],
+    [LogLevels.ERROR]: [],
+    [LogLevels.DISABLED]: [],
+};
+
+function createLogger(type: LogLevels) {
+    const formatter: Record<string, (v: string) => string> = {
+        [LogLevels.INFO]: (msg) => `${green('INFO')} ${msg}`,
+        [LogLevels.WARN]: (msg) => `${yellow('WARN')} ${msg}`,
+        [LogLevels.ERROR]: (msg) => `${red('ERR')} ${msg}`,
+    };
+
+    return function log(msg: string) {
+        const problem = formatter[type](msg);
+
+        if (!problems[type].includes(problem)) {
+            problems[type].push(problem);
+        }
+    };
+}
+
+export const log = {
+    [LogLevels.INFO]: createLogger(LogLevels.INFO),
+    [LogLevels.WARN]: createLogger(LogLevels.WARN),
+    [LogLevels.ERROR]: createLogger(LogLevels.ERROR),
+    LogLevels,
+    get: () => problems,
+    add: (logs: Record<LogLevels, string[]>) => {
+        problems[LogLevels.INFO] = problems[LogLevels.INFO].concat(logs[LogLevels.INFO]);
+        problems[LogLevels.WARN] = problems[LogLevels.WARN].concat(logs[LogLevels.WARN]);
+        problems[LogLevels.ERROR] = problems[LogLevels.ERROR].concat(logs[LogLevels.ERROR]);
+    },
+    clear: () => {
+        problems[LogLevels.INFO] = [];
+        problems[LogLevels.WARN] = [];
+        problems[LogLevels.ERROR] = [];
+    },
+    isEmpty: () => {
+        return !(
+            problems[LogLevels.INFO].length ||
+            problems[LogLevels.WARN].length ||
+            problems[LogLevels.ERROR].length
+        );
+    },
+};
+
+export type Logger = typeof log;
+
+export default log;

package/src/transform/md.ts

@@ -0,0 +1,123 @@
+import type {MarkdownIt, OptionsType, EnvType} from './typings';
+import type Token from 'markdown-it/lib/token';
+
+import DefaultMarkdownIt from 'markdown-it';
+import DefaultPlugins from './plugins';
+import {log} from './log';
+import makeHighlight from './highlight';
+import attrs from 'markdown-it-attrs';
+import extractTitle from './title';
+import getHeadings from './headings';
+import sanitizeHtml from './sanitize';
+
+function initMarkdownit(options: OptionsType) {
+    const {allowHTML = false, linkify = false, breaks = true, highlightLangs = {}} = options;
+
+    const highlight = makeHighlight(highlightLangs);
+    const md = new DefaultMarkdownIt({html: allowHTML, linkify, highlight, breaks}) as MarkdownIt;
+    const env = {
+        // TODO: move md.meta directly to env
+        get meta() {
+            return md.meta;
+        },
+
+        set meta(value) {
+            md.meta = value;
+        },
+
+        // TODO: move md.assets directly to env
+        get assets() {
+            return md.assets;
+        },
+
+        set assets(value) {
+            md.assets = value;
+        },
+
+        headings: [],
+        title: '',
+    } as EnvType;
+
+    initPlugins(md, options);
+
+    const parse = initParser(md, options, env);
+    const compile = initCompiler(md, options, env);
+
+    return {parse, compile, env};
+}
+
+function initPlugins(md: MarkdownIt, options: OptionsType) {
+    const {
+        vars = {},
+        path,
+        extractTitle,
+        conditionsInCode = false,
+        disableLiquid = false,
+        linkify = false,
+        linkifyTlds,
+        leftDelimiter = '{',
+        rightDelimiter = '}',
+        plugins = DefaultPlugins,
+        ...customOptions
+    } = options;
+
+    const pluginOptions = {
+        ...customOptions,
+        conditionsInCode,
+        vars,
+        path,
+        extractTitle,
+        disableLiquid,
+        log,
+    };
+
+    // Need for ids of headers
+    md.use(attrs, {leftDelimiter, rightDelimiter});
+
+    plugins.forEach((plugin) => md.use(plugin, pluginOptions));
+
+    if (linkify && linkifyTlds) {
+        md.linkify.tlds(linkifyTlds, true);
+    }
+}
+
+function initParser(md: MarkdownIt, options: OptionsType, env: EnvType) {
+    return (input: string) => {
+        const {extractTitle: extractTitleOption, needTitle, needFlatListHeadings = false} = options;
+
+        let tokens = md.parse(input, env);
+
+        if (extractTitleOption) {
+            const {title, tokens: slicedTokens, titleTokens} = extractTitle(tokens);
+
+            tokens = slicedTokens;
+
+            // title tokens include other tokens that need to be transformed
+            if (titleTokens.length > 1) {
+                env.title = md.renderer.render(titleTokens, md.options, env);
+            } else {
+                env.title = title;
+            }
+        }
+
+        if (needTitle) {
+            env.title = extractTitle(tokens).title;
+        }
+
+        env.headings = getHeadings(tokens, needFlatListHeadings);
+
+        return tokens;
+    };
+}
+
+function initCompiler(md: MarkdownIt, options: OptionsType, env: EnvType) {
+    const {needToSanitizeHtml = true, sanitizeOptions} = options;
+
+    return (tokens: Token[]) => {
+        const html = md.renderer.render(tokens, md.options, env);
+
+        return needToSanitizeHtml ? sanitizeHtml(html, sanitizeOptions) : html;
+    };
+}
+
+export = initMarkdownit;

package/src/transform/plugins/anchors/collect.ts

@@ -0,0 +1,101 @@
+import _ from 'lodash';
+import {sep} from 'path';
+
+import {getSinglePageAnchorId, resolveRelativePath} from '../../utilsFS';
+import {transformLinkToOriginalArticle} from '../../utils';
+import {CUSTOM_ID_REGEXP, CUSTOM_ID_EXCEPTION} from './constants';
+import {сarriage} from '../utils';
+import {MarkdownItPluginOpts} from '../typings';
+
+const slugify: (str: string, opts: {}) => string = require('slugify');
+
+/* eslint-disable max-len */
+/* If the singlePage option is passed in the options:
+ *  - replace the default anchor and custom anchors with anchors with file at the beginning
+ *  - increase the level of headers,
+ *  - add page anchor to the first matched header
+ *  Example for file index.md:
+ *    Before: # Title {#CustomAnchor} {#custom-anchor}
+ *            ## Subtitle
+ *    After: ## Title {data-original-article=/index} {#_index} {#_index_title} {#_index_CustomAnchor} {#_index_custom-anchor}
+ *           ## Subtitle {#_index_subtitle}
+ * */
+const collect = (input: string, options: MarkdownItPluginOpts & {singlePage: boolean}) => {
+    const {root, path, singlePage} = options;
+
+    if (!singlePage || path.includes(`_includes${sep}`)) {
+        return;
+    }
+
+    const currentPath = resolveRelativePath(root, path);
+    const pageId = getSinglePageAnchorId({root, currentPath});
+    let needToSetPageId = true;
+    let needToSetOriginalPathAttr = true;
+    const lines = input.split(сarriage);
+    let i = 0;
+    const HEADER_LINK_REGEXP = /^(?<headerLevel>#+)\s(?<headerContent>.+)$/i;
+
+    while (i < lines.length) {
+        const line = lines[i];
+        const headerMatch = line.match(HEADER_LINK_REGEXP);
+
+        if (!headerMatch) {
+            i++;
+            continue;
+        }
+
+        const {headerLevel, headerContent} = headerMatch.groups as {
+            headerLevel: string;
+            headerContent: string;
+        };
+
+        let newHeaderContent = headerContent;
+        const newCustomHeaders = [];
+
+        while (CUSTOM_ID_REGEXP.test(newHeaderContent)) {
+            newHeaderContent = newHeaderContent.replace(CUSTOM_ID_REGEXP, (match, customId) => {
+                if (match === CUSTOM_ID_EXCEPTION) {
+                    return match;
+                }
+
+                newCustomHeaders.push(`${pageId}_${customId}`);
+                return '';
+            });
+        }
+        newHeaderContent = newHeaderContent.trim();
+
+        const slugifiedTitle = slugify(newHeaderContent, {lower: true});
+        newCustomHeaders.push(`${pageId}_${slugifiedTitle}`);
+        if (needToSetPageId) {
+            newCustomHeaders.push(pageId);
+            needToSetPageId = false;
+        }
+
+        const newCustomHeadersStr = _.chain(newCustomHeaders)
+            .uniq()
+            .map((id) => {
+                return `{${id}}`;
+            })
+            .value()
+            .join(' ');
+
+        const baseHeaderSyntax = `${headerLevel}# ${newHeaderContent}`;
+        lines[i] = `${baseHeaderSyntax} ${newCustomHeadersStr}`;
+
+        if (needToSetOriginalPathAttr) {
+            const originalArticleHref = transformLinkToOriginalArticle({root, currentPath});
+            lines[
+                i
+            ] = `${baseHeaderSyntax} {data-original-article=${originalArticleHref}} ${newCustomHeadersStr}`;
+
+            needToSetOriginalPathAttr = false;
+        }
+
+        i++;
+    }
+
+    // eslint-disable-next-line consistent-return
+    return lines.join(сarriage);
+};
+
+export = collect;

package/src/transform/plugins/anchors/constants.ts

@@ -0,0 +1,2 @@
+export const CUSTOM_ID_REGEXP = /\[?{ ?#(\S+) ?}]?/g;
+export const CUSTOM_ID_EXCEPTION = '[{#T}]';