@diplodoc/cli 4.0.0

package/src/services/preset.ts

@@ -0,0 +1,55 @@
+import {dirname, normalize} from 'path';
+
+import {DocPreset, YfmPreset} from '../models';
+
+export type PresetStorage = Map<string, YfmPreset>;
+
+let presetStorage: PresetStorage = new Map();
+
+function add(parsedPreset: DocPreset, path: string, varsPreset: string) {
+    const combinedValues: YfmPreset = {
+        ...parsedPreset.default || {},
+        ...parsedPreset[varsPreset] || {},
+    };
+
+    const key = dirname(normalize(path));
+    presetStorage.set(key, combinedValues);
+}
+
+function get(path: string): YfmPreset {
+    let combinedValues: YfmPreset = {};
+    let localPath = normalize(path);
+
+    while (localPath !== '.') {
+        const presetValues: YfmPreset = presetStorage.get(localPath) || {};
+        localPath = dirname(localPath);
+
+        combinedValues = {
+            ...presetValues,
+            ...combinedValues,
+        };
+    }
+
+    // Add root' presets
+    combinedValues = {
+        ...presetStorage.get('.'),
+        ...combinedValues,
+    };
+
+    return combinedValues;
+}
+
+function getPresetStorage(): Map<string, YfmPreset> {
+    return presetStorage;
+}
+
+function setPresetStorage(preset: Map<string, YfmPreset>): void {
+    presetStorage = preset;
+}
+
+export default {
+    add,
+    get,
+    getPresetStorage,
+    setPresetStorage,
+};

package/src/services/tocs.ts

@@ -0,0 +1,401 @@
+import {dirname, extname, join, parse, resolve, relative, normalize, sep} from 'path';
+import {copyFileSync, readFileSync, writeFileSync, existsSync} from 'fs';
+import {load, dump} from 'js-yaml';
+import shell from 'shelljs';
+import walkSync from 'walk-sync';
+import liquid from '@doc-tools/transform/lib/liquid';
+import log from '@doc-tools/transform/lib/log';
+import {bold} from 'chalk';
+
+import {ArgvService, PresetService} from './index';
+import {getContentWithUpdatedStaticMetadata} from './metadata';
+import {YfmToc} from '../models';
+import {Stage, IncludeMode} from '../constants';
+import {isExternalHref, logger} from '../utils';
+import {filterFiles, firstFilterTextItems, liquidField} from './utils';
+import {applyIncluders, IncludersError} from './includers';
+
+export interface TocServiceData {
+    storage: Map<string, YfmToc>;
+    navigationPaths: string[];
+    includedTocPaths: Set<string>;
+}
+
+const storage: TocServiceData['storage'] = new Map();
+let navigationPaths: TocServiceData['navigationPaths'] = [];
+const includedTocPaths: TocServiceData['includedTocPaths'] = new Set();
+
+async function add(path: string) {
+    const {
+        input: inputFolderPath,
+        output: outputFolderPath,
+        outputFormat,
+        ignoreStage,
+        vars,
+    } = ArgvService.getConfig();
+
+    const pathToDir = dirname(path);
+    const content = readFileSync(resolve(inputFolderPath, path), 'utf8');
+    const parsedToc = load(content) as YfmToc;
+
+    // Should ignore toc with specified stage.
+    if (parsedToc.stage === ignoreStage) {
+        return;
+    }
+
+    const combinedVars = {
+        ...PresetService.get(pathToDir),
+        ...vars,
+    };
+
+    if (parsedToc.title) {
+        parsedToc.title = firstFilterTextItems(
+            parsedToc.title,
+            combinedVars,
+            {resolveConditions: true},
+        );
+    }
+
+    if (typeof parsedToc.title === 'string') {
+        parsedToc.title = liquidField(parsedToc.title, combinedVars, path);
+    }
+
+    parsedToc.items = await processTocItems(
+        path,
+        parsedToc.items,
+        join(inputFolderPath, pathToDir),
+        resolve(inputFolderPath),
+        combinedVars,
+    );
+
+    /* Store parsed toc for .md output format */
+    storage.set(path, parsedToc);
+
+    /* Store path to toc file to handle relative paths in navigation */
+    parsedToc.base = pathToDir;
+
+    if (outputFormat === 'md') {
+        /* Should copy resolved and filtered toc to output folder */
+        const outputPath = resolve(outputFolderPath, path);
+        const outputToc = dump(parsedToc);
+        shell.mkdir('-p', dirname(outputPath));
+        writeFileSync(outputPath, outputToc);
+    }
+
+    prepareNavigationPaths(parsedToc, pathToDir);
+}
+
+async function processTocItems(
+    path: string,
+    items: YfmToc[],
+    tocDir: string,
+    sourcesDir: string,
+    vars: Record<string, string>,
+) {
+    const {
+        resolveConditions,
+        removeHiddenTocItems,
+    } = ArgvService.getConfig();
+
+    let preparedItems = items;
+
+    /* Should remove all links with false expressions */
+    if (resolveConditions || removeHiddenTocItems) {
+        try {
+            preparedItems = filterFiles(items, 'items', vars, {
+                resolveConditions,
+                removeHiddenTocItems,
+            });
+        } catch (error) {
+            log.error(`Error while filtering toc file: ${path}. Error message: ${error}`);
+        }
+    }
+
+    /* Should resolve all includes */
+    return _replaceIncludes(path, preparedItems, tocDir, sourcesDir, vars);
+}
+
+function getForPath(path: string): YfmToc|undefined {
+    return storage.get(path);
+}
+
+function getNavigationPaths(): string[] {
+    return [...navigationPaths];
+}
+
+function getIncludedTocPaths(): string[] {
+    return [...includedTocPaths];
+}
+
+function prepareNavigationPaths(parsedToc: YfmToc, dirPath: string) {
+    function processItems(items: YfmToc[], pathToDir: string) {
+        items.forEach((item) => {
+            if (!parsedToc.singlePage && item.items) {
+                const preparedSubItems = item.items.map(((yfmToc: YfmToc, index: number) => {
+                    // Generate personal id for each navigation item
+                    yfmToc.id = `${yfmToc.name}-${index}-${Math.random()}`;
+                    return yfmToc;
+                }));
+                processItems(preparedSubItems, pathToDir);
+            }
+
+            if (item.href && !isExternalHref(item.href)) {
+                const href = join(pathToDir, item.href);
+                storage.set(href, parsedToc);
+
+                const navigationPath = _normalizeHref(href);
+                navigationPaths.push(navigationPath);
+            }
+        });
+    }
+
+    processItems([parsedToc], dirPath);
+}
+
+/**
+ * Should normalize hrefs. MD and YAML files will be ignored.
+ * @param href
+ * @return {string}
+ * @example instance-groups/create-with-coi/ -> instance-groups/create-with-coi/index.yaml
+ * @example instance-groups/create-with-coi -> instance-groups/create-with-coi.md
+ * @private
+ */
+function _normalizeHref(href: string): string {
+    const preparedHref = normalize(href);
+
+    if (preparedHref.endsWith('.md') || preparedHref.endsWith('.yaml')) {
+        return preparedHref;
+    }
+
+    if (preparedHref.endsWith(sep)) {
+        return `${preparedHref}index.yaml`;
+    }
+
+    return `${preparedHref}.md`;
+}
+
+/**
+ * Copies all files of include toc to original dir.
+ * @param tocPath
+ * @param destDir
+ * @return
+ * @private
+ */
+function _copyTocDir(tocPath: string, destDir: string) {
+    const {input: inputFolderPath} = ArgvService.getConfig();
+
+    const {dir: tocDir} = parse(tocPath);
+    const files: string[] = walkSync(tocDir, {
+        globs: ['**/*.*'],
+        ignore: ['**/toc.yaml'],
+        directories: false,
+    });
+
+    files.forEach((relPath) => {
+        const from = resolve(tocDir, relPath);
+        const to = resolve(destDir, relPath);
+        const fileExtension = extname(relPath);
+        const isMdFile = fileExtension === '.md';
+
+        shell.mkdir('-p', parse(to).dir);
+
+        if (isMdFile) {
+            const fileContent = readFileSync(from, 'utf8');
+            const sourcePath = relative(inputFolderPath, from);
+            const updatedFileContent = getContentWithUpdatedStaticMetadata({
+                fileContent,
+                sourcePath,
+                addSourcePath: true,
+            });
+
+            writeFileSync(to, updatedFileContent);
+        } else {
+            copyFileSync(from, to);
+        }
+    });
+}
+
+/**
+ * Make hrefs relative to the main toc in the included toc.
+ * @param items
+ * @param includeTocDir
+ * @param tocDir
+ * @return
+ * @private
+ */
+function _replaceIncludesHrefs(items: YfmToc[], includeTocDir: string, tocDir: string): YfmToc[] {
+    return items.reduce((acc, tocItem) => {
+        if (tocItem.href) {
+            tocItem.href = relative(tocDir, resolve(includeTocDir, tocItem.href));
+        }
+
+        if (tocItem.items) {
+            tocItem.items = _replaceIncludesHrefs(tocItem.items, includeTocDir, tocDir);
+        }
+
+        if (tocItem.include) {
+            const {path} = tocItem.include;
+            tocItem.include.path = relative(tocDir, resolve(includeTocDir, path));
+        }
+
+        return acc.concat(tocItem);
+    }, [] as YfmToc[]);
+}
+
+/**
+ * Liquid substitutions in toc file.
+ * @param input
+ * @param vars
+ * @param path
+ * @return {string}
+ * @private
+ */
+function _liquidSubstitutions(input: string, vars: Record<string, string>, path: string) {
+    const {outputFormat, applyPresets} = ArgvService.getConfig();
+    if (outputFormat === 'md' && !applyPresets) {
+        return input;
+    }
+
+    return liquid(input, vars, path, {
+        conditions: false,
+        substitutions: true,
+    });
+}
+
+function addIncludeTocPath(includeTocPath: string) {
+    includedTocPaths.add(includeTocPath);
+}
+
+/**
+ * Replaces include fields in toc file by resolved toc.
+ * @param path
+ * @param items
+ * @param tocDir
+ * @param sourcesDir
+ * @param vars
+ * @return
+ * @private
+ */
+async function _replaceIncludes(
+    path: string,
+    items: YfmToc[],
+    tocDir: string,
+    sourcesDir: string,
+    vars: Record<string, string>,
+): Promise<YfmToc[]> {
+    const result: YfmToc[] = [];
+
+    for (const item of items) {
+        let includedInlineItems: YfmToc[] | null = null;
+
+        if (item.name) {
+            const tocPath = join(tocDir, 'toc.yaml');
+
+            item.name = _liquidSubstitutions(item.name, vars, tocPath);
+        }
+
+        try {
+            await applyIncluders(path, item, vars);
+        } catch (err) {
+            if (err instanceof Error || err instanceof IncludersError) {
+                const message = err.toString();
+
+                const file = err instanceof IncludersError ? err.path : path;
+
+                logger.error(file, message);
+            }
+        }
+
+        if (item.include) {
+            const {mode = IncludeMode.ROOT_MERGE} = item.include;
+            const includeTocPath = mode === IncludeMode.ROOT_MERGE
+                ? resolve(sourcesDir, item.include.path)
+                : resolve(tocDir, item.include.path);
+            const includeTocDir = dirname(includeTocPath);
+
+            try {
+                const includeToc = load(readFileSync(includeTocPath, 'utf8')) as YfmToc;
+
+                // Should ignore included toc with tech-preview stage.
+                if (includeToc.stage === Stage.TECH_PREVIEW) {
+                    continue;
+                }
+
+                if (mode === IncludeMode.MERGE || mode === IncludeMode.ROOT_MERGE) {
+                    _copyTocDir(includeTocPath, tocDir);
+                }
+
+                /* Save the path to exclude toc from the output directory in the next step */
+                addIncludeTocPath(includeTocPath);
+
+                let includedTocItems = (item.items || []).concat(includeToc.items);
+
+                /* Resolve nested toc inclusions */
+                const baseTocDir = mode === IncludeMode.LINK ? includeTocDir : tocDir;
+                includedTocItems = await processTocItems(path, includedTocItems, baseTocDir, sourcesDir, vars);
+
+                /* Make hrefs relative to the main toc */
+                if (mode === IncludeMode.LINK) {
+                    includedTocItems = _replaceIncludesHrefs(includedTocItems, includeTocDir, tocDir);
+                }
+
+                if (item.name) {
+                    item.items = includedTocItems;
+                } else {
+                    includedInlineItems = includedTocItems;
+                }
+            } catch (err) {
+                const message = (
+                    `Error while including toc: ${bold(includeTocPath)} to ${bold(join(tocDir, 'toc.yaml'))}`
+                );
+
+                log.error(message);
+
+                continue;
+            } finally {
+                delete item.include;
+            }
+        } else if (item.items) {
+            item.items = await processTocItems(path, item.items, tocDir, sourcesDir, vars);
+        }
+
+        if (includedInlineItems) {
+            result.push(...includedInlineItems);
+        } else {
+            result.push(item);
+        }
+    }
+
+    return result;
+}
+
+function getTocDir(pagePath: string): string {
+    const {input: inputFolderPath} = ArgvService.getConfig();
+
+    const tocDir = dirname(pagePath);
+    const tocPath = resolve(tocDir, 'toc.yaml');
+
+
+    if (!tocDir.includes(inputFolderPath)) {
+        throw new Error('Error while finding toc dir');
+    }
+
+    if (existsSync(tocPath)) {
+        return tocDir;
+    }
+
+    return getTocDir(tocDir);
+}
+
+function setNavigationPaths(paths: TocServiceData['navigationPaths']) {
+    navigationPaths = paths;
+}
+
+export default {
+    add,
+    getForPath,
+    getNavigationPaths,
+    getTocDir,
+    getIncludedTocPaths,
+    setNavigationPaths,
+};

package/src/services/utils.ts

@@ -0,0 +1,151 @@
+import evalExp from '@doc-tools/transform/lib/liquid/evaluation';
+import {Filter, TextItems} from '../models';
+import liquid from '@doc-tools/transform/lib/liquid';
+import {ArgvService} from './index';
+
+export interface FilterFilesOptions {
+    resolveConditions?: boolean;
+    removeHiddenTocItems?: boolean;
+}
+
+/**
+ * Filters file by expression and removes empty file's items.
+ * @param items
+ * @param itemsKey
+ * @param vars
+ * @param options
+ * @return {T[]}
+ */
+export function filterFiles<T extends Filter>(
+    items: T[],
+    itemsKey: string,
+    vars: Record<string, string>,
+    options?: FilterFilesOptions,
+): T[] {
+    if (!Array.isArray(items)) {
+        return [];
+    }
+
+    const reducer = (results: T[], item: T) => {
+        if (shouldProcessItem(item, vars, options)) {
+            const prop = item[itemsKey] as T[];
+
+            if (prop) {
+                const filteredProperty = filterFiles(prop, itemsKey, vars, options);
+
+                if (filteredProperty.length) {
+                    results.push({
+                        ...item,
+                        [itemsKey]: filteredProperty,
+                    });
+                }
+            } else {
+                results.push(item);
+            }
+        }
+
+        return results;
+    };
+
+    return items.reduce(reducer, []);
+}
+
+export function filterTextItems(
+    items: undefined | TextItems,
+    vars: Record<string, string>,
+    options?: FilterFilesOptions,
+) {
+    if (!Array.isArray(items)) {
+        return items;
+    }
+
+    return items.reduce((result: string[], item) => {
+        if (!isObject(item)) {
+            result.push(item);
+            return result;
+        }
+
+        const useItem = shouldProcessItem(item, vars, options);
+
+        if (useItem) {
+            if (Array.isArray(item.text)) {
+                result.push(...item.text);
+            } else {
+                result.push(item.text);
+            }
+        }
+
+        return result;
+    }, []);
+}
+
+export function firstFilterTextItems(
+    items: TextItems,
+    vars: Record<string, string>,
+    options?: FilterFilesOptions,
+) {
+    const filteredItems = filterTextItems(items, vars, options);
+
+    if (!Array.isArray(filteredItems)) {
+        return filteredItems || '';
+    }
+
+    return filteredItems[0] || '';
+}
+
+function shouldProcessItem<T extends Filter>(item: T, vars: Record<string, string>, options?: FilterFilesOptions) {
+    const {resolveConditions, removeHiddenTocItems} = options || {};
+    let useItem = true;
+
+    if (resolveConditions) {
+        const {when} = item;
+        useItem =
+            when === true || when === undefined || (typeof when === 'string' && evalExp(when, vars));
+
+        delete item.when;
+    }
+
+    if (useItem && removeHiddenTocItems) {
+        useItem = !item.hidden;
+
+        delete item.hidden;
+    }
+
+    return useItem;
+}
+
+export function liquidFields(fields: undefined | string | string[], vars: Record<string, unknown>, path: string) {
+    if (typeof fields === 'string') {
+        return liquidField(fields, vars, path);
+    }
+
+    if (!Array.isArray(fields)) {
+        return fields;
+    }
+
+    return fields.map((item) => {
+        if (typeof item === 'string') {
+            return liquidField(item, vars, path);
+        }
+        return item;
+    });
+}
+
+export function liquidField(input: string, vars: Record<string, unknown>, path: string) {
+    const {applyPresets, resolveConditions} = ArgvService.getConfig();
+
+    if (!applyPresets && !resolveConditions) {
+        return input;
+    }
+
+    return liquid(input, vars, path, {
+        substitutions: applyPresets,
+        conditions: resolveConditions,
+        keepNotVar: true,
+        withSourceMap: false,
+    });
+}
+
+export function isObject(o: unknown): o is object {
+    return typeof o === 'object' && o !== null;
+}

package/src/steps/index.ts

@@ -0,0 +1,6 @@
+export * from './processAssets';
+export * from './processExcludedFiles';
+export * from './processLogs';
+export * from './processPages';
+export * from './processLinter';
+export * from './processServiceFiles';

package/src/steps/processAssets.ts

@@ -0,0 +1,36 @@
+import walkSync from 'walk-sync';
+import shell from 'shelljs';
+
+import client from '../../scripts/client';
+import {ArgvService} from '../services';
+import {copyFiles} from '../utils';
+
+/**
+ * Processes assets files (everything except .yaml and .md files)
+ * @param {string} outputBundlePath
+ * @return {void}
+ */
+export function processAssets(outputBundlePath: string) {
+    const {
+        input: inputFolderPath,
+        output: outputFolderPath,
+    } = ArgvService.getConfig();
+
+    const assetFilePath: string[] = walkSync(inputFolderPath, {
+        directories: false,
+        includeBasePath: false,
+        ignore: [
+            '**/*.yaml',
+            '**/*.md',
+        ],
+    });
+
+    copyFiles(inputFolderPath, outputFolderPath, assetFilePath);
+
+    /* Copy js bundle to user' output folder */
+    shell.mkdir('-p', outputBundlePath);
+
+    for (const path of Object.values(client.dst)) {
+        shell.cp(path, outputBundlePath);
+    }
+}

package/src/steps/processExcludedFiles.ts

@@ -0,0 +1,47 @@
+import {resolve, relative} from 'path';
+import walkSync from 'walk-sync';
+import shell from 'shelljs';
+
+import {ArgvService, TocService} from '../services';
+import {convertBackSlashToSlash} from '../utils';
+
+/**
+ * Removes all content files that unspecified in toc files or ignored.
+ * @return {void}
+ */
+export function processExcludedFiles() {
+    const {
+        input: inputFolderPath,
+        output: outputFolderPath,
+        ignore,
+    } = ArgvService.getConfig();
+
+    const allContentFiles: string[] = walkSync(inputFolderPath, {
+        directories: false,
+        includeBasePath: true,
+        globs: [
+            '**/*.md',
+            '**/index.yaml',
+            ...ignore,
+        ],
+        // Ignores service directories like "_includes", "_templates" and etc.
+        ignore: ['**/_*/**/*'],
+    });
+    const navigationPaths = TocService.getNavigationPaths()
+        .map((filePath) => convertBackSlashToSlash(resolve(inputFolderPath, filePath)));
+    const tocSpecifiedFiles = new Set(navigationPaths);
+    const excludedFiles = allContentFiles
+        .filter((filePath) => !tocSpecifiedFiles.has(filePath));
+
+    shell.rm('-f', excludedFiles);
+
+    const includedTocPaths = TocService.getIncludedTocPaths()
+        .map((filePath) => {
+            const relativeTocPath = relative(inputFolderPath, filePath);
+            const destTocPath = resolve(outputFolderPath, relativeTocPath);
+
+            return convertBackSlashToSlash(destTocPath);
+        });
+
+    shell.rm('-rf', includedTocPaths);
+}