@diplodoc/cli 4.0.0

package/src/resolvers/md2html.ts

@@ -0,0 +1,142 @@
+import {basename, dirname, join, relative, resolve, sep} from 'path';
+import {readFileSync, writeFileSync} from 'fs';
+import yaml from 'js-yaml';
+
+import transform, {Output} from '@doc-tools/transform';
+import log from '@doc-tools/transform/lib/log';
+import liquid from '@doc-tools/transform/lib/liquid';
+
+import {ResolverOptions, YfmToc, ResolveMd2HTMLResult, LeadingPage} from '../models';
+import {ArgvService, TocService, PluginService} from '../services';
+import {generateStaticMarkup, logger, transformToc, getVarsPerFile, getVarsPerRelativeFile} from '../utils';
+import {PROCESSING_FINISHED, Lang} from '../constants';
+import {getAssetsPublicPath, getUpdatedMetadata} from '../services/metadata';
+import {MarkdownItPluginCb} from '@doc-tools/transform/lib/plugins/typings';
+
+export interface FileTransformOptions {
+    path: string;
+    root?: string;
+}
+
+const FileTransformer: Record<string, Function> = {
+    '.yaml': YamlFileTransformer,
+    '.md': MdFileTransformer,
+};
+
+const fixRelativePath = (relativeTo: string) => (path: string) => {
+    return join(getAssetsPublicPath(relativeTo), path);
+};
+
+export async function resolveMd2HTML(options: ResolverOptions): Promise<ResolveMd2HTMLResult> {
+    const {inputPath, fileExtension, outputPath, outputBundlePath, metadata} = options;
+
+    const pathToDir: string = dirname(inputPath);
+    const toc: YfmToc|null = TocService.getForPath(inputPath) || null;
+    const tocBase: string = toc && toc.base ? toc.base : '';
+    const pathToFileDir: string = pathToDir === tocBase ? '' : pathToDir.replace(`${tocBase}${sep}`, '');
+    const relativePathToIndex = relative(pathToDir, `${tocBase}${sep}`);
+
+    const {input, lang, allowCustomResources} = ArgvService.getConfig();
+    const resolvedPath: string = resolve(input, inputPath);
+    const content: string = readFileSync(resolvedPath, 'utf8');
+
+    const transformFn: Function = FileTransformer[fileExtension];
+    const {result} = transformFn(content, {path: inputPath});
+
+    const updatedMetadata = metadata && metadata.isContributorsEnabled
+        ? await getUpdatedMetadata(metadata, content, result?.meta)
+        : result.meta;
+
+    const fileMeta = fileExtension === '.yaml' ? (result.data.meta ?? {}) : updatedMetadata;
+
+    if (allowCustomResources) {
+        const {script, style} = metadata?.resources || {};
+        fileMeta.style = (fileMeta.style || []).concat(style || [])
+            .map(fixRelativePath(inputPath));
+        fileMeta.script = (fileMeta.script || []).concat(script || [])
+            .map(fixRelativePath(inputPath));
+    } else {
+        fileMeta.style = [];
+        fileMeta.script = [];
+    }
+
+    const props = {
+        data: {
+            leading: inputPath.endsWith('.yaml'),
+            toc: transformToc(toc, pathToDir) || {},
+            ...result,
+            meta: fileMeta,
+        },
+        router: {
+            pathname: join(relativePathToIndex, pathToFileDir, basename(outputPath)),
+        },
+        lang: lang || Lang.RU,
+    };
+
+    const outputDir = dirname(outputPath);
+    const relativePathToBundle: string = relative(resolve(outputDir), resolve(outputBundlePath));
+
+    const outputFileContent = generateStaticMarkup(props, relativePathToBundle);
+    writeFileSync(outputPath, outputFileContent);
+    logger.info(inputPath, PROCESSING_FINISHED);
+
+    return props;
+}
+
+function YamlFileTransformer(content: string): Object {
+    let data: LeadingPage | null = null;
+
+    try {
+        data = yaml.load(content) as LeadingPage;
+    } catch (error) {
+        log.error(`Yaml transform has been failed. Error: ${error}`);
+    }
+
+    if (!data) {
+        return {
+            result: {data: {}},
+        };
+    }
+
+    const links = data?.links?.map(
+        (link) =>
+            link.href ? ({...link, href: link.href.replace(/.md$/gmu, '.html')}) : link,
+    );
+
+    if (links) { data.links = links; }
+
+    return {
+        result: {data},
+    };
+}
+
+export function liquidMd2Html(input: string, vars: Record<string, unknown>, path: string) {
+    const {conditionsInCode} = ArgvService.getConfig();
+
+    return liquid(input, vars, path, {
+        conditionsInCode,
+        withSourceMap: true,
+    });
+}
+
+function MdFileTransformer(content: string, transformOptions: FileTransformOptions): Output {
+    const {input, ...options} = ArgvService.getConfig();
+    const {path: filePath} = transformOptions;
+
+    const plugins = PluginService.getPlugins();
+    const vars = getVarsPerFile(filePath);
+    const root = resolve(input);
+    const path: string = resolve(input, filePath);
+
+
+    return transform(content, {
+        ...options,
+        plugins: plugins as MarkdownItPluginCb<unknown>[],
+        vars,
+        root,
+        path,
+        assetsPublicPath: getAssetsPublicPath(filePath),
+        getVarsPerFile: getVarsPerRelativeFile,
+        extractTitle: true,
+    });
+}

package/src/resolvers/md2md.ts

@@ -0,0 +1,147 @@
+import {existsSync, readFileSync, writeFileSync} from 'fs';
+import {dirname, resolve, join, basename, extname} from 'path';
+import shell from 'shelljs';
+import log from '@doc-tools/transform/lib/log';
+import liquid from '@doc-tools/transform/lib/liquid';
+
+import {ArgvService, PluginService} from '../services';
+import {logger, getVarsPerFile} from '../utils';
+import {PluginOptions, ResolveMd2MdOptions} from '../models';
+import {PROCESSING_FINISHED} from '../constants';
+import {getContentWithUpdatedMetadata} from '../services/metadata';
+import {ChangelogItem} from '@doc-tools/transform/lib/plugins/changelog/types';
+
+export async function resolveMd2Md(options: ResolveMd2MdOptions): Promise<void> {
+    const {inputPath, outputPath, metadata} = options;
+    const {input, output} = ArgvService.getConfig();
+    const resolvedInputPath = resolve(input, inputPath);
+    const vars = getVarsPerFile(inputPath);
+
+    const content = await getContentWithUpdatedMetadata(
+        readFileSync(resolvedInputPath, 'utf8'),
+        metadata,
+        vars.__system,
+    );
+
+    const {result, changelogs} = transformMd2Md(content, {
+        path: resolvedInputPath,
+        destPath: outputPath,
+        root: resolve(input),
+        destRoot: resolve(output),
+        collectOfPlugins: PluginService.getCollectOfPlugins(),
+        vars,
+        log,
+        copyFile,
+    });
+
+    writeFileSync(outputPath, result);
+
+    if (changelogs?.length) {
+        const mdFilename = basename(outputPath, extname(outputPath));
+        const outputDir = dirname(outputPath);
+        changelogs.forEach((changes, index) => {
+            let changesName;
+            const changesDate = changes.date as string | undefined;
+            const changesIdx = changes.index as number | undefined;
+            if (typeof changesIdx === 'number') {
+                changesName = String(changesIdx);
+            }
+            if (!changesName && changesDate && /^\d{4}/.test(changesDate)) {
+                changesName = Math.trunc(new Date(changesDate).getTime() / 1000);
+            }
+            if (!changesName) {
+                changesName = `name-${mdFilename}-${String(changelogs.length - index).padStart(3, '0')}`;
+            }
+
+            const changesPath = join(outputDir, `changes-${changesName}.json`);
+
+            if (existsSync(changesPath)) {
+                throw new Error(`Changelog ${changesPath} already exists!`);
+            }
+
+            writeFileSync(changesPath, JSON.stringify({
+                ...changes,
+                source: mdFilename,
+            }));
+        });
+    }
+
+    logger.info(inputPath, PROCESSING_FINISHED);
+
+    return undefined;
+}
+
+function copyFile(targetPath: string, targetDestPath: string, options?: PluginOptions) {
+    shell.mkdir('-p', dirname(targetDestPath));
+
+    if (options) {
+        const sourceIncludeContent = readFileSync(targetPath, 'utf8');
+        const {result} = transformMd2Md(sourceIncludeContent, options);
+
+        writeFileSync(targetDestPath, result);
+    } else {
+        shell.cp(targetPath, targetDestPath);
+    }
+}
+
+export function liquidMd2Md(input: string, vars: Record<string, unknown>, path: string) {
+    const {
+        applyPresets,
+        resolveConditions,
+        conditionsInCode,
+    } = ArgvService.getConfig();
+
+    return liquid(input, vars, path, {
+        conditions: resolveConditions,
+        substitutions: applyPresets,
+        conditionsInCode,
+        withSourceMap: true,
+        keepNotVar: true,
+    });
+}
+
+function transformMd2Md(input: string, options: PluginOptions) {
+    const {
+        disableLiquid,
+    } = ArgvService.getConfig();
+    const {
+        vars = {},
+        path,
+        root,
+        destPath,
+        destRoot,
+        collectOfPlugins,
+        log: pluginLog,
+        copyFile: pluginCopyFile,
+    } = options;
+
+    let output = input;
+    const changelogs: ChangelogItem[] = [];
+
+    if (!disableLiquid) {
+        const liquidResult = liquidMd2Md(input, vars, path);
+
+        output = liquidResult.output;
+    }
+
+    if (collectOfPlugins) {
+        output = collectOfPlugins(output, {
+            vars,
+            path,
+            root,
+            destPath,
+            destRoot,
+            log: pluginLog,
+            copyFile: pluginCopyFile,
+            collectOfPlugins,
+            changelogs,
+            extractChangelogs: true,
+        });
+    }
+
+    return {
+        result: output,
+        changelogs,
+        logs: pluginLog.get(),
+    };
+}

package/src/services/argv.ts

@@ -0,0 +1,38 @@
+import {YfmArgv} from '../models';
+import {join} from 'path';
+import {readFileSync} from 'fs';
+
+let _argv!: YfmArgv;
+
+function getConfig() {
+    return _argv;
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function init(argv: any) {
+    _argv = {
+        ...argv,
+        ignore: Array.isArray(argv.ignore) ? argv.ignore : [],
+    } as YfmArgv;
+
+    if (argv.vars) {
+        _argv.vars = JSON.parse(argv.vars);
+    }
+
+    try {
+        const ignorefile = readFileSync(join(_argv.rootInput, '.yfmignore'), 'utf8');
+        const ignore = ignorefile.split('\n');
+
+        _argv.ignore = _argv.ignore.concat(ignore);
+    } catch {}
+}
+
+function set(argv: YfmArgv) {
+    _argv = argv;
+}
+
+export default {
+    getConfig,
+    init,
+    set,
+};

package/src/services/authors.ts

@@ -0,0 +1,64 @@
+import {replaceDoubleToSingleQuotes} from '../utils';
+import {VCSConnector} from '../vcs-connector/connector-models';
+
+async function updateAuthorMetadataStringByAuthorLogin(
+    authorLogin: string,
+    vcsConnector?: VCSConnector,
+): Promise<string> {
+    if (!vcsConnector) {
+        return '';
+    }
+
+    const user = await getAuthorDetails(vcsConnector, authorLogin);
+
+    if (user) {
+        return user;
+    }
+
+    return '';
+}
+
+
+async function updateAuthorMetadataStringByFilePath(
+    filePath: string,
+    vcsConnector?: VCSConnector,
+): Promise<string> {
+    if (!vcsConnector) {
+        return '';
+    }
+
+    const user = vcsConnector.getExternalAuthorByPath(filePath);
+
+    if (user) {
+        const author = replaceDoubleToSingleQuotes(JSON.stringify(user));
+        return author;
+    }
+
+    return '';
+}
+
+async function getAuthorDetails(vcsConnector: VCSConnector, author: string | object): Promise<string | null> {
+    if (typeof author === 'object') {
+        // Avoiding problems when adding to html markup
+        return replaceDoubleToSingleQuotes(JSON.stringify(author));
+    }
+
+    try {
+        JSON.parse(author);
+        return replaceDoubleToSingleQuotes(author);
+    } catch {
+        const user = await vcsConnector.getUserByLogin(author);
+
+        if (user) {
+            return replaceDoubleToSingleQuotes(JSON.stringify(user));
+        }
+
+        return null;
+    }
+}
+
+export {
+    updateAuthorMetadataStringByAuthorLogin,
+    updateAuthorMetadataStringByFilePath,
+    getAuthorDetails,
+};

package/src/services/contributors.ts

@@ -0,0 +1,104 @@
+import {readFile} from 'fs/promises';
+import {dirname, join} from 'path';
+
+import {replaceDoubleToSingleQuotes} from '../utils';
+import {REGEXP_INCLUDE_CONTENTS, REGEXP_INCLUDE_FILE_PATH} from '../constants';
+import {Contributor, Contributors, FileData} from '../models';
+import {FileContributors, VCSConnector} from '../vcs-connector/connector-models';
+
+async function getFileContributorsMetadata(fileData: FileData, vcsConnector: VCSConnector): Promise<string> {
+    const contributors = await getFileContributorsString(fileData, vcsConnector);
+
+    return `contributors: ${contributors}`;
+}
+
+async function getFileContributorsString(fileData: FileData, vcsConnector: VCSConnector): Promise<string> {
+    const {tmpInputFilePath, inputFolderPathLength} = fileData;
+
+    const relativeFilePath = tmpInputFilePath.substring(inputFolderPathLength);
+    const fileContributors: FileContributors = await vcsConnector.getContributorsByPath(relativeFilePath);
+    let nestedContributors: Contributors = {};
+
+    if (!fileContributors.hasIncludes) {
+        nestedContributors = await getContributorsForNestedFiles(fileData, vcsConnector);
+        vcsConnector.addNestedContributorsForPath(relativeFilePath, nestedContributors);
+    }
+
+    const fileContributorsWithContributorsIncludedFiles: Contributors = {
+        ...fileContributors.contributors,
+        ...nestedContributors,
+    };
+
+    const contributorsArray: Contributor[] =
+        Object.entries(fileContributorsWithContributorsIncludedFiles).map(([, contributor]) => contributor);
+
+    return replaceDoubleToSingleQuotes(JSON.stringify(contributorsArray));
+}
+
+async function getContributorsForNestedFiles(fileData: FileData, vcsConnector: VCSConnector): Promise<Contributors> {
+    const {fileContent, inputFolderPathLength} = fileData;
+
+    const includeContents = fileContent.match(REGEXP_INCLUDE_CONTENTS);
+    if (!includeContents || includeContents.length === 0) {
+        return {};
+    }
+
+    const includesContributors: Contributors[] = [];
+    const relativeIncludeFilePaths: Set<string> = getRelativeIncludeFilePaths(fileData, includeContents);
+
+    for (const relativeIncludeFilePath of relativeIncludeFilePaths.values()) {
+        const relativeFilePath = relativeIncludeFilePath.substring(inputFolderPathLength);
+        const includeContributors = await vcsConnector.getContributorsByPath(relativeFilePath);
+
+        let nestedContributors: Contributors = {};
+
+        if (!includeContributors.hasIncludes) {
+            let contentIncludeFile: string;
+            try {
+                contentIncludeFile = await readFile(relativeIncludeFilePath, 'utf8');
+            } catch (err) {
+                if (err.code === 'ENOENT') {
+                    continue;
+                }
+                throw err;
+            }
+
+            const newFileData: FileData = {
+                ...fileData,
+                fileContent: contentIncludeFile,
+                tmpInputFilePath: relativeIncludeFilePath,
+            };
+
+            nestedContributors = await getContributorsForNestedFiles(newFileData, vcsConnector);
+            vcsConnector.addNestedContributorsForPath(relativeFilePath, nestedContributors);
+        }
+
+        includesContributors.push(includeContributors.contributors);
+        includesContributors.push(nestedContributors);
+    }
+
+    return Object.assign({}, ...includesContributors);
+}
+
+function getRelativeIncludeFilePaths(fileData: FileData, includeContents: string[]): Set<string> {
+    const {tmpInputFilePath} = fileData;
+    const relativeIncludeFilePaths: Set<string> = new Set();
+
+    includeContents.forEach((includeContent: string) => {
+        const relativeIncludeFilePath = includeContent.match(REGEXP_INCLUDE_FILE_PATH);
+
+        if (relativeIncludeFilePath && relativeIncludeFilePath.length !== 0) {
+            const relativeIncludeFilePathWithoutHash = relativeIncludeFilePath[0].split('#');
+            const includeFilePath = join(dirname(tmpInputFilePath), relativeIncludeFilePathWithoutHash[0]);
+
+            relativeIncludeFilePaths.add(includeFilePath);
+        }
+    });
+
+    return relativeIncludeFilePaths;
+}
+
+export {
+    getFileContributorsMetadata,
+    getFileContributorsString,
+};

package/src/services/includers/batteries/common.ts

@@ -0,0 +1,34 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+const {promises: {readdir}} = require('fs');
+
+async function getDirs(path: string) {
+    const isDir = (i: any) => i.isDirectory();
+
+    return readdir(path, {withFileTypes: true}).then((list: any) => list.filter(isDir));
+}
+
+async function getFiles(path: string) {
+    const isFile = (i: any) => i.isFile();
+
+    return readdir(path, {withFileTypes: true}).then((list: any) => list.filter(isFile));
+}
+
+const complement = (fn: Function) => (x: any) => !fn(x);
+
+const isMdExtension = (str: string): boolean => /.md$/gmu.test(str);
+
+const isHidden = (str: string) => /^\./gmu.test(str);
+
+const allPass = (predicates: Function[]) => (arg: any) =>
+    predicates.map((fn) => fn(arg)).reduce((p, c) => p && c, true);
+
+const compose = <R>(fn1: (a: R) => R, ...fns: Array<(a: R) => R>) =>
+    fns.reduce((prevFn, nextFn) => (value) => prevFn(nextFn(value)), fn1);
+
+const prop = (string: string) => (object: Object) => object[string as keyof typeof object];
+
+function concatNewLine(prefix: string, suffix: string) {
+    return prefix.trim().length ? `${prefix}<br>${suffix}` : suffix;
+}
+
+export {complement, isMdExtension, isHidden, allPass, compose, prop, getDirs, getFiles, concatNewLine};

package/src/services/includers/batteries/generic.ts

@@ -0,0 +1,130 @@
+import {readFile, writeFile, mkdir} from 'fs/promises';
+import {parse, join, dirname} from 'path';
+
+import {updateWith} from 'lodash';
+import {dump} from 'js-yaml';
+
+import {glob} from '../../../utils/glob';
+
+import {IncluderFunctionParams} from '../../../models';
+
+class GenericIncluderError extends Error {
+    path: string;
+
+    constructor(message: string, path: string) {
+        super(message);
+
+        this.name = 'GenericIncluderError';
+        this.path = path;
+    }
+}
+
+const name = 'generic';
+
+const MD_GLOB = '**/*.md';
+
+type Params = {
+    input: string;
+    leadingPage: {
+        name?: string;
+    };
+};
+
+async function includerFunction(params: IncluderFunctionParams<Params>) {
+    const {readBasePath, writeBasePath, tocPath, item, passedParams: {input, leadingPage}, index} = params;
+
+    if (!input?.length || !item.include?.path) {
+        throw new GenericIncluderError('provide includer with input parameter', tocPath);
+    }
+
+    try {
+        const leadingPageName = leadingPage?.name ?? 'Overview';
+
+        const tocDirPath = dirname(tocPath);
+
+        const contentPath = index === 0
+            ? join(writeBasePath, tocDirPath, input)
+            : join(readBasePath, tocDirPath, input);
+
+        let cache = {};
+        let found = [];
+
+        ({state: {found, cache}} = await glob(MD_GLOB, {
+            cwd: contentPath,
+            nosort: true,
+            nocase: true,
+            cache,
+        }));
+
+        const writePath = join(writeBasePath, tocDirPath, item.include.path);
+
+        await mkdir(writePath, {recursive: true});
+
+        for (const filePath of found) {
+            const file = await readFile(join(contentPath, filePath));
+
+            await mkdir(dirname(join(writePath, filePath)), {recursive: true});
+            await writeFile(join(writePath, filePath), file);
+        }
+
+        const graph = createGraphFromPaths(found);
+
+        const toc = createToc(leadingPageName, item.include.path)(graph, []);
+
+        await writeFile(join(writePath, 'toc.yaml'), dump(toc));
+    } catch (err) {
+        throw new GenericIncluderError(err.toString(), tocPath);
+    }
+}
+
+function createGraphFromPaths(paths: string[]) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const graph: Record<string, any> = {};
+
+    for (const path of paths) {
+        const chunks = path.split('/').filter(Boolean);
+        if (chunks.length < 2) {
+            if (chunks.length === 1) {
+                graph.files = chunks;
+            }
+
+            continue;
+        }
+
+        const file = chunks.pop();
+
+        updateWith(graph, chunks, (old) => {
+            return old ? {files: [...old.files, file]} : {files: [file]};
+        }, Object);
+    }
+
+    return graph;
+}
+
+function createToc(leadingPageName: string, tocName: string) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    return function createTocRec(graph: Record<string, any>, cursor: string[]): Record<string, any> {
+        const handler = (file: string) => ({
+            name: parse(file).name === 'index' ? leadingPageName : file,
+            href: join(...cursor, file),
+        });
+
+        const recurse = (key: string) => createTocRec(graph[key], [...cursor, key]);
+
+        const current = {
+            name: cursor[cursor.length - 1] ?? tocName,
+            items: [
+                ...(graph.files ?? []).map(handler),
+                ...Object.keys(graph)
+                    .filter((key) => key !== 'files')
+                    .map(recurse),
+            ],
+        };
+
+        return current;
+    };
+}
+
+export {name, includerFunction};
+
+export default {name, includerFunction};

package/src/services/includers/batteries/index.ts

@@ -0,0 +1,3 @@
+export * as generic from './generic';
+export * as sourcedocs from './sourcedocs';
+export * as unarchive from './unarchive';

package/src/services/includers/batteries/sourcedocs.ts

@@ -0,0 +1,33 @@
+import {logger} from '../../../utils/logger';
+
+import generic from './generic';
+
+import {IncluderFunctionParams} from '../../../models';
+
+const name = 'sourcedocs';
+
+const usage = `include:
+  path: <path-where-to-include>
+  includers:
+    - name: generic
+      input: <path-to-directory-with-markdown>
+      leadingPage:
+        name: <leading-page-name>
+`;
+
+type Params = {
+    input: string;
+    leadingPage: {
+        name?: string;
+    };
+};
+
+async function includerFunction(params: IncluderFunctionParams<Params>) {
+    logger.warn(params.tocPath, `sourcedocs inlcuder is getting depricated in favor of generic includer\n${usage}`);
+
+    await generic.includerFunction(params);
+}
+
+export {name, includerFunction};
+
+export default {name, includerFunction};