@diplodoc/cli 4.0.0

package/src/cmd/build/index.ts

@@ -0,0 +1,304 @@
+import {Arguments, Argv} from 'yargs';
+import {
+    BUNDLE_FOLDER, LINT_CONFIG_FILENAME,
+    REDIRECTS_FILENAME,
+    Stage,
+    TMP_INPUT_FOLDER,
+    TMP_OUTPUT_FOLDER,
+    YFM_CONFIG_FILENAME,
+} from '../../constants';
+import {argvValidator} from '../../validator';
+import {join, resolve} from 'path';
+import {ArgvService, Includers} from '../../services';
+import OpenapiIncluder from '@diplodoc/openapi-extension/includer';
+import {
+    initLinterWorkers,
+    processAssets,
+    processExcludedFiles,
+    processLinter, processLogs,
+    processPages,
+    processServiceFiles,
+} from '../../steps';
+import {prepareMapFile} from '../../steps/processMapFile';
+import shell from 'shelljs';
+import {Resources} from '../../models';
+import {copyFiles, logger} from '../../utils';
+import {upload as publishFilesToS3} from '../publish/upload';
+import glob from 'glob';
+
+export const build = {
+    command: ['build', '$0'],
+    description: 'Build documentation in target directory',
+    handler,
+    builder,
+};
+
+function builder<T>(argv: Argv<T>) {
+    return argv
+        .option('input', {
+            alias: 'i',
+            describe: 'Path to input folder with .md files',
+            type: 'string',
+            group: 'Build options:',
+        })
+        .option('output', {
+            alias: 'o',
+            describe: 'Path to output folder',
+            type: 'string',
+            group: 'Build options:',
+        })
+        .option('varsPreset', {
+            default: 'default',
+            describe: 'Target vars preset of documentation <external|internal>',
+            group: 'Build options:',
+        })
+        .option('output-format', {
+            default: 'html',
+            describe: 'Format of output file <html|md>',
+            group: 'Build options:',
+        })
+        .option('vars', {
+            alias: 'v',
+            default: '{}',
+            describe: 'List of markdown variables',
+            group: 'Build options:',
+        })
+        .option('apply-presets', {
+            default: true,
+            describe: 'Should apply presets. Only for --output-format=md',
+            type: 'boolean',
+            group: 'Build options:',
+        })
+        .option('resolve-conditions', {
+            default: true,
+            describe: 'Should resolve conditions. Only for --output-format=md',
+            type: 'boolean',
+            group: 'Build options:',
+        })
+        .option('conditions-in-code', {
+            default: false,
+            describe: 'Meet conditions in code blocks',
+            type: 'boolean',
+            group: 'Build options:',
+        })
+        .option('disable-liquid', {
+            default: false,
+            describe: 'Disable template engine',
+            type: 'boolean',
+            group: 'Build options:',
+        })
+        .option('allowHTML', {
+            default: false,
+            describe: 'Allow to use HTML in Markdown files',
+            type: 'boolean',
+            group: 'Build options:',
+        })
+        .option('ignore-stage', {
+            default: Stage.SKIP,
+            describe: 'Ignore tocs with stage',
+            group: 'Build options:',
+        })
+        .option('ignore-author-patterns', {
+            default: [] as string[],
+            describe: 'Ignore authors if they contain passed string',
+            group: 'Build options:',
+            type: 'array',
+        })
+        .option('contributors', {
+            default: false,
+            describe: 'Should attach contributors into files',
+            type: 'boolean',
+            group: 'Build options:',
+        })
+        .option('add-system-meta', {
+            default: false,
+            describe: 'Should add system section variables form presets into files meta data',
+            type: 'boolean',
+            group: 'Build options:',
+        })
+        .option('add-map-file', {
+            default: false,
+            describe: 'Should add all paths of documentation into file.json',
+            type: 'boolean',
+            group: 'Build options:',
+        })
+        .option('single-page', {
+            default: false,
+            describe: 'Beta functionality: Build a single page in the output folder also',
+            type: 'boolean',
+            group: 'Build options:',
+        })
+        .option('publish', {
+            default: false,
+            describe: 'Should upload output files to S3 storage',
+            type: 'boolean',
+            group: 'Build options:',
+        })
+        .option('remove-hidden-toc-items', {
+            default: false,
+            describe: 'Remove hidden toc items',
+            type: 'boolean',
+            group: 'Build options:',
+        })
+        .option('lint-disabled', {
+            default: false,
+            describe: 'Disable linting',
+            type: 'boolean',
+            group: 'Build options:',
+        })
+        .option('build-disabled', {
+            default: false,
+            describe: 'Disable building',
+            type: 'boolean',
+            group: 'Build options:',
+        })
+        .option('allow-custom-resources', {
+            default: false,
+            describe: 'Allow loading custom resources',
+            type: 'boolean',
+            group: 'Build options:',
+        })
+        .option('static-content', {
+            default: false,
+            describe: 'Include static content in the page',
+            type: 'boolean',
+            group: 'Build options:',
+        })
+        .check(argvValidator)
+        .example('yfm -i ./input -o ./output', '')
+        .demandOption(['input', 'output'], 'Please provide input and output arguments to work with this tool');
+}
+
+async function handler(args: Arguments<any>) {
+    const userOutputFolder = resolve(args.output);
+    const tmpInputFolder = resolve(args.output, TMP_INPUT_FOLDER);
+    const tmpOutputFolder = resolve(args.output, TMP_OUTPUT_FOLDER);
+
+    try {
+        ArgvService.init({
+            ...args,
+            rootInput: args.input,
+            input: tmpInputFolder,
+            output: tmpOutputFolder,
+        });
+        Includers.init([OpenapiIncluder as any]);
+
+        const {
+            output: outputFolderPath,
+            outputFormat,
+            publish,
+            lintDisabled,
+            buildDisabled,
+            addMapFile,
+            allowCustomResources,
+            resources,
+        } = ArgvService.getConfig();
+
+        preparingTemporaryFolders(userOutputFolder);
+
+        await processServiceFiles();
+        processExcludedFiles();
+
+        if (addMapFile) {
+            prepareMapFile();
+        }
+
+        const outputBundlePath = join(outputFolderPath, BUNDLE_FOLDER);
+        const pathToConfig = args.config || join(args.input, YFM_CONFIG_FILENAME);
+        const pathToRedirects = join(args.input, REDIRECTS_FILENAME);
+        const pathToLintConfig = join(args.input, LINT_CONFIG_FILENAME);
+
+        if (!lintDisabled) {
+            /* Initialize workers in advance to avoid a timeout failure due to not receiving a message from them */
+            await initLinterWorkers();
+        }
+
+        const processes = [
+            !lintDisabled && processLinter(),
+            !buildDisabled && processPages(outputBundlePath),
+        ].filter(Boolean) as Promise<void>[];
+
+        await Promise.all(processes);
+
+        if (!buildDisabled) {
+            // process additional files
+            switch (outputFormat) {
+                case 'html':
+                    processAssets(outputBundlePath);
+                    break;
+                case 'md': {
+                    shell.cp(resolve(pathToConfig), tmpOutputFolder);
+                    shell.cp(resolve(pathToRedirects), tmpOutputFolder);
+                    shell.cp(resolve(pathToLintConfig), tmpOutputFolder);
+
+                    if (resources && allowCustomResources) {
+                        const resourcePaths: string[] = [];
+
+                        // collect paths of all resources
+                        Object.keys(resources).forEach((type) =>
+                            resources[type as keyof Resources]?.forEach((path: string) => resourcePaths.push(path)));
+
+                        //copy resources
+                        copyFiles(args.input, tmpOutputFolder, resourcePaths);
+                    }
+
+                    break;
+                }
+            }
+
+            // Copy all generated files to user' output folder
+            shell.cp('-r', [join(tmpOutputFolder, '*'), join(tmpOutputFolder, '.*')], userOutputFolder);
+
+            if (publish) {
+                const DEFAULT_PREFIX = process.env.YFM_STORAGE_PREFIX ?? '';
+                const {
+                    ignore = [],
+                    storageRegion,
+                    storageEndpoint: endpoint,
+                    storageBucket: bucket,
+                    storagePrefix: prefix = DEFAULT_PREFIX,
+                    storageKeyId: accessKeyId,
+                    storageSecretKey: secretAccessKey,
+                } = ArgvService.getConfig();
+
+                await publishFilesToS3({
+                    input: userOutputFolder,
+                    region: storageRegion,
+                    ignore,
+                    endpoint,
+                    bucket,
+                    prefix,
+                    accessKeyId,
+                    secretAccessKey,
+                });
+            }
+        }
+    } catch (err) {
+        logger.error('', err.message);
+    } finally {
+        processLogs(tmpInputFolder);
+
+        shell.rm('-rf', tmpInputFolder, tmpOutputFolder);
+    }
+}
+
+function preparingTemporaryFolders(userOutputFolder: string) {
+    const args = ArgvService.getConfig();
+
+    shell.mkdir('-p', userOutputFolder);
+
+    // Create temporary input/output folders
+    shell.rm('-rf', args.input, args.output);
+    shell.mkdir(args.input, args.output);
+    shell.chmod('-R', 'u+w', args.input);
+
+    copyFiles(args.rootInput, args.input, glob.sync('**', {
+        cwd: args.rootInput,
+        nodir: true,
+        follow: true,
+        ignore: [
+            'node_modules/**',
+            '*/node_modules/**',
+        ],
+    }));
+}

package/src/cmd/index.ts

@@ -0,0 +1,4 @@
+export {build} from './build';
+export {publish} from './publish';
+export {xliff} from './xliff';
+export {translate} from './translate';

package/src/cmd/publish/index.ts

@@ -0,0 +1,92 @@
+import {join} from 'path';
+import {ArgvService} from '../../services';
+import {logger} from '../../utils';
+
+import {Argv, Arguments} from 'yargs';
+
+import {upload} from './upload';
+
+const command = 'publish';
+
+const description = 'Upload builded documentation to target S3 bucket';
+
+const publish = {
+    command,
+    description,
+    handler,
+    builder,
+};
+
+function builder<T>(argv: Argv<T>) {
+    return argv
+        .option('strict', {
+            default: true,
+        })
+        .option('input', {
+            alias: 'i',
+            describe: 'Path to folder with builded files',
+            type: 'string',
+            required: true,
+            group: 'Upload options',
+        })
+        .option('endpoint', {
+            describe: 'S3 bucket endpoint',
+            default: 'https://s3.amazonaws.com',
+            type: 'string',
+            group: 'Upload options',
+        })
+        .option('region', {
+            describe: 'S3 bucket region',
+            default: 'eu-central-1',
+            type: 'string',
+            group: 'Upload options',
+        })
+        .option('bucket', {
+            describe: 'S3 bucket name',
+            type: 'string',
+            required: true,
+            group: 'Upload options',
+        })
+        .option('prefix', {
+            describe: 'S3 bucket ',
+            default: '',
+            type: 'string',
+            group: 'Upload options',
+        })
+        .option('access-key-id', {
+            describe: 'S3 bucket AccessKeyId',
+            type: 'string',
+            required: true,
+            group: 'Upload options',
+        })
+        .option('secret-access-key', {
+            describe: 'S3 bucket SecretAccessKey',
+            type: 'string',
+            required: true,
+            group: 'Upload options',
+        });
+}
+
+/* eslint-disable-next-line @typescript-eslint/no-explicit-any */
+async function handler(args: Arguments<any>) {
+    ArgvService.init({
+        ...args,
+    });
+
+    const {
+        input,
+        endpoint,
+        bucket,
+        prefix,
+    } = ArgvService.getConfig();
+
+    logger.info('', `Upload artifacts from ${input} to ${join(endpoint, bucket, prefix)}`);
+
+    try {
+        await upload(ArgvService.getConfig());
+    } catch (error) {
+        logger.error('', error.message);
+    }
+}
+
+export {publish};

package/src/cmd/publish/upload.ts

@@ -0,0 +1,61 @@
+import {createReadStream} from 'fs';
+import walkSync from 'walk-sync';
+import {resolve, join} from 'path';
+import {S3Client, PutObjectCommand} from '@aws-sdk/client-s3';
+import mime from 'mime-types';
+
+import {convertBackSlashToSlash, logger} from '../../utils';
+import {asyncify, mapLimit} from 'async';
+
+interface UploadProps {
+    input: string;
+    ignore: string[];
+    endpoint: string;
+    bucket: string;
+    prefix: string;
+    accessKeyId: string;
+    secretAccessKey: string;
+    region: string;
+}
+
+export async function upload(props: UploadProps): Promise<void> {
+    const {
+        input,
+        ignore = [],
+        endpoint,
+        region,
+        bucket,
+        prefix,
+        accessKeyId,
+        secretAccessKey,
+    } = props;
+
+    const s3Client = new S3Client({
+        endpoint,
+        region,
+        credentials: {accessKeyId, secretAccessKey},
+    });
+
+    const filesToPublish: string[] = walkSync(resolve(input), {
+        directories: false,
+        includeBasePath: false,
+        ignore,
+    });
+
+    await mapLimit(filesToPublish, 100, asyncify(async (pathToFile: string) => {
+        const mimeType = mime.lookup(pathToFile);
+
+        logger.upload(pathToFile);
+
+        try {
+            await s3Client.send(new PutObjectCommand({
+                ContentType: mimeType ? mimeType : undefined,
+                Bucket: bucket,
+                Key: convertBackSlashToSlash(join(prefix, pathToFile)),
+                Body: createReadStream(resolve(input, pathToFile)),
+            }));
+        } catch (error) {
+            logger.error(pathToFile, error.message);
+        }
+    }));
+}