@diplodoc/cli 4.0.0

package/src/steps/processLinter.ts

@@ -0,0 +1,100 @@
+import log from '@doc-tools/transform/lib/log';
+import {spawn, Worker, Thread} from 'threads';
+import {extname} from 'path';
+
+import {ArgvService, TocService, PresetService, PluginService} from '../services';
+import {ProcessLinterWorker} from '../workers/linter';
+import {logger} from '../utils';
+import {LINTING_FINISHED, WORKERS_COUNT, MIN_CHUNK_SIZE} from '../constants';
+import {lintPage} from '../resolvers';
+import {splitOnChunks} from '../utils/worker';
+
+let processLinterWorkers: (ProcessLinterWorker & Thread)[];
+let navigationPathsChunks: (string[])[];
+
+export async function processLinter(): Promise<void> {
+    const argvConfig = ArgvService.getConfig();
+
+    const navigationPaths = TocService.getNavigationPaths();
+
+    if (!processLinterWorkers) {
+        lintPagesFallback(navigationPaths);
+
+        return;
+    }
+
+    const presetStorage = PresetService.getPresetStorage();
+
+    /* Subscribe on the linted page event */
+    processLinterWorkers.forEach((worker) => {
+        worker.getProcessedPages().subscribe((pathToFile) => {
+            logger.info(pathToFile as string, LINTING_FINISHED);
+        });
+    });
+
+    /* Run processing the linter */
+    await Promise.all(
+        processLinterWorkers.map((worker, i) => {
+            const navigationPathsChunk = navigationPathsChunks[i];
+
+            return worker.run({
+                argvConfig,
+                presetStorage,
+                navigationPaths: navigationPathsChunk,
+            });
+        }),
+    );
+
+    /* Unsubscribe from workers */
+    await Promise.all(
+        processLinterWorkers.map((worker) => {
+            return worker.finish().then((logs) => {
+                log.add(logs);
+            });
+        }),
+    );
+
+    /* Terminate workers */
+    await Promise.all(
+        processLinterWorkers.map((worker) => {
+            return Thread.terminate(worker);
+        }),
+    );
+}
+
+export async function initLinterWorkers() {
+    const navigationPaths = TocService.getNavigationPaths();
+    const chunkSize = getChunkSize(navigationPaths);
+
+    if (process.env.DISABLE_PARALLEL_BUILD || chunkSize < MIN_CHUNK_SIZE || WORKERS_COUNT <= 0) {
+        return;
+    }
+
+    navigationPathsChunks = splitOnChunks(navigationPaths, chunkSize)
+        .filter((arr) => arr.length);
+
+    const workersCount = navigationPathsChunks.length;
+
+    processLinterWorkers = await Promise.all((new Array(workersCount)).fill(null).map(() => {
+        // TODO: get linter path from env
+        return spawn<ProcessLinterWorker>(new Worker('./linter'), {timeout: 60000});
+    }));
+}
+
+function getChunkSize(arr: string[]) {
+    return Math.ceil(arr.length / WORKERS_COUNT);
+}
+
+function lintPagesFallback(navigationPaths: string[]) {
+    PluginService.setPlugins();
+
+    navigationPaths.forEach((pathToFile) => {
+        lintPage({
+            inputPath: pathToFile,
+            fileExtension: extname(pathToFile),
+            onFinish: () => {
+                logger.info(pathToFile, LINTING_FINISHED);
+            },
+        });
+    });
+}

package/src/steps/processLogs.ts

@@ -0,0 +1,18 @@
+import log from '@doc-tools/transform/lib/log';
+import _uniq from 'lodash/uniq';
+
+export function processLogs(inputFolder: string) {
+    const replacementRegExp = new RegExp(inputFolder, 'ig');
+    const {info, warn, error} = log.get();
+    const outputLogs = _uniq([
+        '', ...info,
+        '', ...warn,
+        '', ...error,
+        '',
+    ]);
+
+    for (const outputLog of outputLogs) {
+        const preparedLog = outputLog.replace(replacementRegExp, '');
+        console.log(preparedLog);
+    }
+}

package/src/steps/processMapFile.ts

@@ -0,0 +1,35 @@
+import {writeFileSync} from 'fs';
+import {extname, join} from 'path';
+
+import {ArgvService, TocService} from '../services';
+import {convertBackSlashToSlash} from '../utils';
+
+type TocItem = {
+    name: string;
+    items?: TocItems;
+    href?: string;
+};
+
+type TocItems = TocItem[];
+
+export function prepareMapFile(): void {
+    const {
+        output: outputFolderPath,
+    } = ArgvService.getConfig();
+
+    const navigationPathsWithoutExtensions =
+        TocService.getNavigationPaths().map((path) => {
+            let preparedPath = convertBackSlashToSlash(path.replace(extname(path), ''));
+
+            if (preparedPath.endsWith('/index')) {
+                preparedPath = preparedPath.substring(0, preparedPath.length - 5);
+            }
+
+            return preparedPath;
+        });
+    const navigationPaths = {files: [...new Set(navigationPathsWithoutExtensions)]};
+    const filesMapBuffer = Buffer.from(JSON.stringify(navigationPaths, null, '\t'), 'utf8');
+    const mapFile = join(outputFolderPath, 'files.json');
+
+    writeFileSync(mapFile, filesMapBuffer);
+}

package/src/steps/processPages.ts

@@ -0,0 +1,312 @@
+import type {DocInnerProps} from '@diplodoc/client';
+import {basename, dirname, extname, resolve, join, relative} from 'path';
+import shell from 'shelljs';
+import {copyFileSync, readFileSync, writeFileSync} from 'fs';
+import {bold} from 'chalk';
+import {dump, load} from 'js-yaml';
+import {mapLimit, asyncify} from 'async';
+
+import log from '@doc-tools/transform/lib/log';
+
+import {ArgvService, LeadingService, TocService, PluginService} from '../services';
+import {resolveMd2HTML, resolveMd2Md} from '../resolvers';
+import {generateStaticMarkup, joinSinglePageResults, logger, transformTocForSinglePage} from '../utils';
+import {
+    MetaDataOptions,
+    SinglePageResult,
+    PathData,
+    YfmToc,
+    Resources,
+    LeadingPage,
+} from '../models';
+import {VCSConnector} from '../vcs-connector/connector-models';
+import {getVCSConnector} from '../vcs-connector';
+import {
+    SINGLE_PAGE_FILENAME,
+    SINGLE_PAGE_DATA_FILENAME,
+    Lang, ResourceType,
+} from '../constants';
+
+const singlePageResults: Record<string, SinglePageResult[]> = {};
+const singlePagePaths: Record<string, Set<string>> = {};
+
+// Processes files of documentation (like index.yaml, *.md)
+export async function processPages(outputBundlePath: string): Promise<void> {
+    const {
+        input: inputFolderPath,
+        output: outputFolderPath,
+        outputFormat,
+        singlePage,
+        resolveConditions,
+    } = ArgvService.getConfig();
+
+    const vcsConnector = await getVCSConnector();
+
+    PluginService.setPlugins();
+
+    const navigationPaths = TocService.getNavigationPaths();
+    const concurrency = 500;
+
+    await mapLimit(navigationPaths, concurrency, asyncify(async (pathToFile: string) => {
+        const pathData = getPathData(pathToFile, inputFolderPath, outputFolderPath, outputFormat, outputBundlePath);
+
+        logger.proc(pathToFile);
+
+        const metaDataOptions = getMetaDataOptions(pathData, inputFolderPath.length, vcsConnector);
+
+        await preparingPagesByOutputFormat(pathData, metaDataOptions, resolveConditions, singlePage);
+    }));
+
+    if (singlePage) {
+        await saveSinglePages(outputBundlePath);
+    }
+}
+
+function getPathData(
+    pathToFile: string,
+    inputFolderPath: string,
+    outputFolderPath: string,
+    outputFormat: string,
+    outputBundlePath: string,
+): PathData {
+    const pathToDir: string = dirname(pathToFile);
+    const filename: string = basename(pathToFile);
+    const fileExtension: string = extname(pathToFile);
+    const fileBaseName: string = basename(filename, fileExtension);
+    const outputDir = resolve(outputFolderPath, pathToDir);
+    const outputFileName = `${fileBaseName}.${outputFormat}`;
+    const outputPath = resolve(outputDir, outputFileName);
+    const resolvedPathToFile = resolve(inputFolderPath, pathToFile);
+    const outputTocDir = TocService.getTocDir(resolvedPathToFile);
+
+    const pathData: PathData = {
+        pathToFile,
+        resolvedPathToFile,
+        filename,
+        fileBaseName,
+        fileExtension,
+        outputDir,
+        outputPath,
+        outputFormat,
+        outputBundlePath,
+        outputTocDir,
+        inputFolderPath,
+        outputFolderPath,
+    };
+
+    return pathData;
+}
+
+async function saveSinglePages(outputBundlePath: string) {
+    const {
+        input: inputFolderPath,
+        output: outputFolderPath,
+        lang,
+        resources,
+    } = ArgvService.getConfig();
+
+    try {
+        await Promise.all(Object.keys(singlePageResults).map(async (tocDir) => {
+            if (!singlePageResults[tocDir].length) {
+                return;
+            }
+
+            const singlePageBody = joinSinglePageResults(singlePageResults[tocDir], inputFolderPath, tocDir);
+            const tocPath = join(relative(inputFolderPath, tocDir), 'toc.yaml');
+            const toc: YfmToc|null = TocService.getForPath(tocPath) || null;
+            const preparedToc = transformTocForSinglePage(toc, {
+                root: inputFolderPath,
+                currentPath: join(tocDir, SINGLE_PAGE_FILENAME),
+            }) as YfmToc;
+
+            const pageData = {
+                data: {
+                    leading: false as const,
+                    html: singlePageBody,
+                    headings: [],
+                    meta: resources || {},
+                    toc: preparedToc,
+                },
+                router: {
+                    pathname: SINGLE_PAGE_FILENAME,
+                },
+                lang: lang || Lang.RU,
+            };
+
+            const outputTocDir = resolve(outputFolderPath, relative(inputFolderPath, tocDir));
+            const relativeOutputBundlePath = relative(outputTocDir, outputBundlePath);
+
+            // Save the full single page for viewing locally
+            const singlePageFn = join(tocDir, SINGLE_PAGE_FILENAME);
+            const singlePageDataFn = join(tocDir, SINGLE_PAGE_DATA_FILENAME);
+            const singlePageContent = generateStaticMarkup(pageData, relativeOutputBundlePath);
+
+            writeFileSync(singlePageFn, singlePageContent);
+            writeFileSync(singlePageDataFn, JSON.stringify(pageData));
+        }));
+    } catch (error) {
+        console.log(error);
+    }
+}
+
+function savePageResultForSinglePage(pageProps: DocInnerProps, pathData: PathData): void {
+    const {pathToFile, outputTocDir} = pathData;
+
+    if (pageProps.data.leading) {
+        return;
+    }
+
+    singlePagePaths[outputTocDir] = singlePagePaths[outputTocDir] || new Set();
+
+    if (singlePagePaths[outputTocDir].has(pathToFile)) {
+        return;
+    }
+
+    singlePagePaths[outputTocDir].add(pathToFile);
+
+    singlePageResults[outputTocDir] = singlePageResults[outputTocDir] || [];
+    singlePageResults[outputTocDir].push({
+        path: pathToFile,
+        content: pageProps.data.html,
+        title: pageProps.data.title,
+    });
+}
+
+function getMetaDataOptions(pathData: PathData, inputFolderPathLength: number, vcsConnector?: VCSConnector,
+): MetaDataOptions {
+    const {contributors, addSystemMeta, resources, allowCustomResources} = ArgvService.getConfig();
+
+    const metaDataOptions: MetaDataOptions = {
+        vcsConnector,
+        fileData: {
+            tmpInputFilePath: pathData.resolvedPathToFile,
+            inputFolderPathLength,
+            fileContent: '',
+        },
+        isContributorsEnabled: Boolean(contributors && vcsConnector),
+        addSystemMeta,
+    };
+
+
+    if (allowCustomResources && resources) {
+        const allowedResources = Object.entries(resources).reduce((acc: Resources, [key, val]) => {
+            if (Object.keys(ResourceType).includes(key)) {
+                acc[key as keyof typeof ResourceType] = val;
+            }
+            return acc;
+        }, {});
+
+        metaDataOptions.resources = allowedResources;
+    }
+
+
+    return metaDataOptions;
+}
+
+async function preparingPagesByOutputFormat(
+    path: PathData,
+    metaDataOptions: MetaDataOptions,
+    resolveConditions: boolean,
+    singlePage: boolean,
+): Promise<void> {
+    const {
+        filename,
+        fileExtension,
+        fileBaseName,
+        outputDir,
+        resolvedPathToFile,
+        outputFormat,
+        pathToFile,
+    } = path;
+    const {allowCustomResources} = ArgvService.getConfig();
+
+    try {
+        shell.mkdir('-p', outputDir);
+
+        const isYamlFileExtension = fileExtension === '.yaml';
+
+        if (resolveConditions && fileBaseName === 'index' && isYamlFileExtension) {
+            LeadingService.filterFile(pathToFile);
+        }
+
+        if (outputFormat === 'md' && isYamlFileExtension && allowCustomResources) {
+            processingYamlFile(path, metaDataOptions);
+            return;
+        }
+
+        if (outputFormat === 'md' && isYamlFileExtension ||
+            outputFormat === 'html' && !isYamlFileExtension && fileExtension !== '.md') {
+            copyFileWithoutChanges(resolvedPathToFile, outputDir, filename);
+            return;
+        }
+
+        switch (outputFormat) {
+            case 'md':
+                await processingFileToMd(path, metaDataOptions);
+                return;
+            case 'html': {
+                const resolvedFileProps = await processingFileToHtml(path, metaDataOptions);
+
+                if (singlePage) {
+                    savePageResultForSinglePage(resolvedFileProps, path);
+                }
+
+                return;
+            }
+        }
+    } catch (e) {
+        const message = `No such file or has no access to ${bold(resolvedPathToFile)}`;
+        console.log(message, e);
+        log.error(message);
+    }
+}
+//@ts-ignore
+function processingYamlFile(path: PathData, metaDataOptions: MetaDataOptions) {
+    const {pathToFile, outputFolderPath, inputFolderPath} = path;
+
+    const filePath = resolve(inputFolderPath, pathToFile);
+    const content = readFileSync(filePath, 'utf8');
+    const parsedContent = load(content) as LeadingPage;
+
+    if (metaDataOptions.resources) {
+        parsedContent.meta = {...parsedContent.meta, ...metaDataOptions.resources};
+    }
+
+    writeFileSync(resolve(outputFolderPath, pathToFile), dump(parsedContent));
+}
+
+function copyFileWithoutChanges(resolvedPathToFile: string, outputDir: string, filename: string): void {
+    const from = resolvedPathToFile;
+    const to = resolve(outputDir, filename);
+
+    copyFileSync(from, to);
+}
+
+async function processingFileToMd(path: PathData, metaDataOptions: MetaDataOptions): Promise<void> {
+    const {outputPath, pathToFile} = path;
+
+    await resolveMd2Md({
+        inputPath: pathToFile,
+        outputPath,
+        metadata: metaDataOptions,
+    });
+}
+
+async function processingFileToHtml(path: PathData, metaDataOptions: MetaDataOptions): Promise<DocInnerProps> {
+    const {
+        outputBundlePath,
+        filename,
+        fileExtension,
+        outputPath,
+        pathToFile,
+    } = path;
+
+    return resolveMd2HTML({
+        inputPath: pathToFile,
+        outputBundlePath,
+        fileExtension,
+        outputPath,
+        filename,
+        metadata: metaDataOptions,
+    });
+}

package/src/steps/processServiceFiles.ts

@@ -0,0 +1,95 @@
+import {dirname, resolve} from 'path';
+import walkSync from 'walk-sync';
+import {readFileSync, writeFileSync} from 'fs';
+import {load, dump} from 'js-yaml';
+import log from '@doc-tools/transform/lib/log';
+
+import {ArgvService, PresetService, TocService} from '../services';
+import {logger} from '../utils';
+import {DocPreset} from '../models';
+import shell from 'shelljs';
+
+type GetFilePathsByGlobalsFunction = (globs: string[]) => string[];
+
+export async function processServiceFiles(): Promise<void> {
+    const {input: inputFolderPath, ignore = []} = ArgvService.getConfig();
+
+    const getFilePathsByGlobals = (globs: string[]): string[] => {
+        return walkSync(inputFolderPath, {
+            directories: false,
+            includeBasePath: false,
+            globs,
+            ignore,
+        });
+    };
+
+    preparingPresetFiles(getFilePathsByGlobals);
+    await preparingTocFiles(getFilePathsByGlobals);
+}
+
+function preparingPresetFiles(getFilePathsByGlobals: GetFilePathsByGlobalsFunction): void {
+    const {
+        input: inputFolderPath,
+        varsPreset = '',
+        outputFormat,
+        applyPresets,
+        resolveConditions,
+    } = ArgvService.getConfig();
+
+    try {
+        const presetsFilePaths = getFilePathsByGlobals(['**/presets.yaml']);
+
+        for (const path of presetsFilePaths) {
+            logger.proc(path);
+
+            const pathToPresetFile = resolve(inputFolderPath, path);
+            const content = readFileSync(pathToPresetFile, 'utf8');
+            const parsedPreset = load(content) as DocPreset;
+
+            PresetService.add(parsedPreset, path, varsPreset);
+
+            if (outputFormat === 'md' && (!applyPresets || !resolveConditions)) {
+                // Should save filtered presets.yaml only when --apply-presets=false or --resolve-conditions=false
+                saveFilteredPresets(path, parsedPreset);
+            }
+        }
+    } catch (error) {
+        log.error(`Preparing presets.yaml files failed. Error: ${error}`);
+        throw error;
+    }
+}
+
+function saveFilteredPresets(path: string, parsedPreset: DocPreset): void {
+    const {output: outputFolderPath, varsPreset = ''} = ArgvService.getConfig();
+
+    const outputPath = resolve(outputFolderPath, path);
+    const filteredPreset: Record<string, Object> = {
+        default: parsedPreset.default,
+    };
+
+    if (parsedPreset[varsPreset]) {
+        filteredPreset[varsPreset] = parsedPreset[varsPreset];
+    }
+
+    const outputPreset = dump(filteredPreset, {
+        lineWidth: 120,
+    });
+
+    shell.mkdir('-p', dirname(outputPath));
+    writeFileSync(outputPath, outputPreset);
+}
+
+async function preparingTocFiles(getFilePathsByGlobals: GetFilePathsByGlobalsFunction): Promise<void> {
+    try {
+        const tocFilePaths = getFilePathsByGlobals(['**/toc.yaml']);
+
+        for (const path of tocFilePaths) {
+            logger.proc(path);
+
+            await TocService.add(path);
+        }
+    } catch (error) {
+        log.error(`Preparing toc.yaml files failed. Error: ${error}`);
+        throw error;
+    }
+}

package/src/steps/publishFilesToS3.ts

@@ -0,0 +1,47 @@
+import {readFileSync} from 'fs';
+import walkSync from 'walk-sync';
+import {resolve, join} from 'path';
+import S3 from 'aws-sdk/clients/s3';
+import mime from 'mime-types';
+
+import {ArgvService} from '../services';
+import {convertBackSlashToSlash, logger} from '../utils';
+
+const DEFAULT_PREFIX = process.env.YFM_STORAGE_PREFIX ?? '';
+
+export async function publishFilesToS3(): Promise<void> {
+    const {
+        output: outputFolderPath,
+        ignore = [],
+        storageEndpoint: endpoint,
+        storageBucket: bucket,
+        storagePrefix: prefix = DEFAULT_PREFIX,
+        storageKeyId: accessKeyId,
+        storageSecretKey: secretAccessKey,
+    } = ArgvService.getConfig();
+
+    const s3Client = new S3({
+        endpoint, accessKeyId, secretAccessKey,
+    });
+
+    const filesToPublish: string[] = walkSync(resolve(outputFolderPath), {
+        directories: false,
+        includeBasePath: false,
+        ignore,
+    });
+
+    await Promise.all(filesToPublish.map(async (pathToFile) => {
+        const mimeType = mime.lookup(pathToFile);
+
+        const params: S3.Types.PutObjectRequest = {
+            ContentType: mimeType ? mimeType : undefined,
+            Bucket: bucket,
+            Key: convertBackSlashToSlash(join(prefix, pathToFile)),
+            Body: readFileSync(resolve(outputFolderPath, pathToFile)),
+        };
+
+        logger.upload(pathToFile);
+
+        return s3Client.upload(params).promise();
+    }));
+}

package/src/utils/file.ts

@@ -0,0 +1,17 @@
+import {dirname, resolve} from 'path';
+import shell from 'shelljs';
+import {copyFileSync} from 'fs';
+import {logger} from './logger';
+
+export function copyFiles(inputFolderPath: string, outputFolderPath: string, files: string[]): void {
+    for (const pathToAsset of files) {
+        const outputDir: string = resolve(outputFolderPath, dirname(pathToAsset));
+        const from = resolve(inputFolderPath, pathToAsset);
+        const to = resolve(outputFolderPath, pathToAsset);
+
+        shell.mkdir('-p', outputDir);
+        copyFileSync(from, to);
+
+        logger.copy(pathToAsset);
+    }
+}

package/src/utils/glob.ts

@@ -0,0 +1,14 @@
+import libglob, {IGlob, IOptions} from 'glob';
+
+export type Glob = {state: IGlob};
+
+const glob = async (pattern: string, options: IOptions): Promise<Glob> =>
+    new Promise((res, rej) => {
+        const state: IGlob = libglob(pattern, options, (err) =>
+            err ? rej(err) : res({state}),
+        );
+    });
+
+export {glob};
+
+export default {glob};

package/src/utils/index.ts

@@ -0,0 +1,8 @@
+export * from './logger';
+export * from './markup';
+export * from './url';
+export * from './path';
+export * from './toc';
+export * from './presets';
+export * from './glob';
+export * from './file';

package/src/utils/logger.ts

@@ -0,0 +1,42 @@
+import log from '@doc-tools/transform/lib/log';
+import {blue, green, grey, red, yellow} from 'chalk';
+import {ArgvService} from '../services';
+
+function writeLog(msg: string, fatal = false) {
+    const {quiet} = ArgvService.getConfig();
+
+    if (quiet && !fatal) {
+        return;
+    }
+
+    console.log(msg);
+}
+
+export const logger = {
+    info: function (pathToFile: string, extraMessage?: string) {
+        writeLog(`${grey('INFO')} ${extraMessage} ${pathToFile}`);
+    },
+    proc: function (pathToFile: string) {
+        writeLog(`${blue('PROC')} Processing file ${pathToFile}`);
+    },
+    copy: function (pathToFile: string) {
+        writeLog(`${green('COPY')} Copying file ${pathToFile}`);
+    },
+    upload: function (pathToFile: string) {
+        writeLog(`${green('UPLOAD')} Uploading file ${pathToFile}`);
+    },
+    warn: function (pathToFile: string, extraMessage: string) {
+        const message = `${yellow('WARNING')} file: ${pathToFile} error: ${extraMessage}`;
+
+        writeLog(message);
+
+        log.warn(`file: ${pathToFile} ${extraMessage}`);
+    },
+    error: function (pathToFile: string, extraMessage: string) {
+        const message = `${red('ERROR')} file: ${pathToFile} error: ${extraMessage}`;
+
+        writeLog(message, true);
+
+        log.error(`file: ${pathToFile} ${extraMessage}`);
+    },
+};