@diplodoc/cli 4.0.0

package/src/services/includers/batteries/unarchive.ts

@@ -0,0 +1,97 @@
+import {mkdirSync, createReadStream, createWriteStream} from 'fs';
+import {join, dirname} from 'path';
+import {extract, Headers} from 'tar-stream';
+
+import type {PassThrough} from 'stream';
+
+import {IncluderFunctionParams} from '../../../models';
+
+const name = 'unarchive';
+
+class UnarchiveIncluderError extends Error {
+    path: string;
+
+    constructor(message: string, path: string) {
+        super(message);
+
+        this.name = 'UnarchiveIncluderError';
+        this.path = path;
+    }
+}
+
+function pipeline(readPath: string, writeBasePath: string): Promise<void> {
+    return new Promise((res, rej) => {
+        const reader = createReadStream(readPath);
+
+        reader.on('error', (err: Error) => {
+            rej(err);
+        });
+
+        const extractor = extract();
+
+        extractor.on('error', (err: Error) => {
+            rej(err);
+        });
+
+        mkdirSync(writeBasePath, {recursive: true});
+
+        extractor.on('entry', (header: Headers, stream: PassThrough, next: Function) => {
+            const {type, name} = header;
+
+            const writePath = join(writeBasePath, name);
+
+            const writeDirPath = type === 'directory' ? writePath : dirname(writePath);
+
+            mkdirSync(writeDirPath, {recursive: true});
+
+            if (type !== 'directory') {
+                const writer = createWriteStream(writePath, {flags: 'w'});
+
+                writer.on('error', (err) => {
+                    rej(err);
+                });
+
+                stream.pipe(writer);
+            }
+
+            stream.on('end', () => {
+                next();
+            });
+
+            stream.resume();
+        });
+
+        reader.pipe(extractor).on('finish', () => {
+            res();
+        });
+    });
+}
+
+type Params = {
+    input: string;
+    output: string;
+};
+
+async function includerFunction(params: IncluderFunctionParams<Params>) {
+    const {readBasePath, writeBasePath, tocPath, passedParams: {input, output}, index} = params;
+
+    if (!input?.length || !output?.length) {
+        throw new UnarchiveIncluderError('provide includer with input parameter', tocPath);
+    }
+
+    const contentPath = index === 0
+        ? join(writeBasePath, input)
+        : join(readBasePath, input);
+
+    const writePath = join(writeBasePath, output);
+
+    try {
+        await pipeline(contentPath, writePath);
+    } catch (err) {
+        throw new UnarchiveIncluderError(err.toString(), tocPath);
+    }
+}
+
+export {name, includerFunction};
+
+export default {name, includerFunction};

package/src/services/includers/index.ts

@@ -0,0 +1,157 @@
+import {join} from 'path';
+
+import {isObject} from 'lodash';
+
+import {ArgvService} from '../index';
+import {IncludeMode} from '../../constants';
+import {generic, sourcedocs, unarchive} from './batteries';
+
+import type {
+    YfmPreset,
+    Includer,
+    YfmToc,
+    YfmTocInclude,
+    YfmTocIncluder,
+    YfmTocIncluders,
+} from '../../models';
+
+const includersUsage = `include:
+  path: <path-where-to-include>
+  includers:
+    - name: <includer-name-0>
+      <includer-parameter>: <value-for-includer-parameter>
+    - name: <includer-name-1>
+      <includer-parameter>: <value-for-includer-parameter>
+`;
+
+type IncludersMap = Record<string, Includer>;
+
+let includersMap!: IncludersMap;
+
+class IncludersError extends Error {
+    path: string;
+
+    constructor(message: string, path: string) {
+        super(message);
+
+        this.name = 'IncludersError';
+        this.path = path;
+    }
+}
+
+function init(custom: Includer[] = []) {
+    if (includersMap) { return; }
+
+    includersMap = {generic, sourcedocs, unarchive};
+
+    for (const includer of custom) {
+        includersMap[includer.name] = includer;
+    }
+}
+
+async function applyIncluders(path: string, item: YfmToc, vars: YfmPreset) {
+    if (!item.include?.includers) {
+        return;
+    }
+
+    if (!includeValid(item.include)) {
+        throw new IncludersError('include doesn\'t comply with includers standard', path);
+    }
+
+    // normalize include mode (includers support link mode only)
+    item.include.mode = IncludeMode.LINK;
+
+    const {status, message} = includersValid(item.include.includers);
+    if (!status) {
+        throw new IncludersError(message ?? '', path);
+    }
+
+    let index = 0;
+    for (const {name, ...rest} of item.include.includers) {
+        const includer = getIncluder(name);
+        const passedParams = {...rest};
+
+        await applyIncluder({path, item, includer, passedParams, index, vars});
+    }
+
+    // contract to be fullfilled by the includer:
+    // provide builder generated toc.yaml
+    item.include.path = join(item.include.path, 'toc.yaml');
+    index++;
+}
+
+function includeValid(include: YfmTocInclude) {
+    return (include.mode === IncludeMode.LINK || !include.mode) && include.path?.length;
+}
+
+function includersValid(includers: YfmTocIncluders) {
+    for (const includer of includers) {
+        const {status, message} = includerValid(includer);
+
+        if (!status) {
+            return {status, message};
+        }
+    }
+
+    return {status: true};
+}
+
+function includerValid(includer: YfmTocIncluder) {
+    if (isObject(includer)) {
+        if (typeof includer.name !== 'string') {
+            return {
+                status: false,
+                message: 'use string in the includer.name to specify includers name',
+            };
+        }
+
+        if (includerExists(includer)) {
+            return {status: true};
+        }
+
+        return {status: false, message: `includer ${includer.name} not implemented`};
+    }
+
+    return {
+        status: false,
+        message: `use appropriate includers format:\n${includersUsage}`,
+    };
+}
+
+function getIncluder(includerName: string) {
+    return includersMap[includerName];
+}
+
+function includerExists(includer: YfmTocIncluder) {
+    return includersMap[includer.name as keyof typeof includersMap];
+}
+
+export type applyIncluderParams = {
+    path: string;
+    item: YfmToc;
+    includer: Includer;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    passedParams: Record<string, any>;
+    index: number;
+    vars: YfmPreset;
+};
+
+async function applyIncluder(args: applyIncluderParams) {
+    const {rootInput: readBasePath, input: writeBasePath} = ArgvService.getConfig();
+
+    const {path, item, includer, passedParams, index, vars} = args;
+
+    const params = {
+        tocPath: path,
+        passedParams,
+        index,
+        item,
+        readBasePath,
+        writeBasePath,
+        vars,
+    };
+
+    return await includer.includerFunction(params);
+}
+
+export {init, applyIncluders, IncludersError};

package/src/services/index.ts

@@ -0,0 +1,6 @@
+export {default as TocService} from './tocs';
+export {default as PresetService} from './preset';
+export {default as ArgvService} from './argv';
+export {default as LeadingService} from './leading';
+export * as PluginService from './plugins';
+export * as Includers from './includers';

package/src/services/leading.ts

@@ -0,0 +1,88 @@
+import {dirname, resolve} from 'path';
+import {readFileSync, writeFileSync} from 'fs';
+import {dump, load} from 'js-yaml';
+import log from '@doc-tools/transform/lib/log';
+
+import {ArgvService, PresetService} from './index';
+import {LeadingPage} from '../models';
+import {filterTextItems, filterFiles, firstFilterTextItems, liquidFields, liquidField} from './utils';
+
+function filterFile(path: string) {
+    const {
+        input: inputFolderPath,
+        vars,
+    } = ArgvService.getConfig();
+
+    const pathToDir = dirname(path);
+    const filePath = resolve(inputFolderPath, path);
+    const content = readFileSync(filePath, 'utf8');
+    const parsedIndex = load(content) as LeadingPage;
+
+    const combinedVars = {
+        ...PresetService.get(pathToDir),
+        ...vars,
+    };
+
+    /* Should remove all links with false expressions */
+    try {
+        const title = firstFilterTextItems(
+            parsedIndex.title,
+            combinedVars,
+            {resolveConditions: true},
+        );
+        parsedIndex.title = liquidField(title, combinedVars, path);
+
+        const description = filterTextItems(
+            parsedIndex.description,
+            combinedVars,
+            {resolveConditions: true},
+        );
+        parsedIndex.description = liquidFields(description, combinedVars, path);
+
+        if (parsedIndex.meta?.title) {
+            const metaTitle = firstFilterTextItems(
+                parsedIndex.meta.title,
+                combinedVars,
+                {resolveConditions: true},
+            );
+            parsedIndex.meta.title = liquidField(metaTitle, combinedVars, path);
+        }
+
+        if (parsedIndex.meta?.description) {
+            const metaDescription = firstFilterTextItems(
+                parsedIndex.meta.description,
+                combinedVars,
+                {resolveConditions: true},
+            );
+            parsedIndex.meta.description = liquidField(metaDescription, combinedVars, path);
+        }
+
+        if (parsedIndex.nav) {
+            const navTitle = firstFilterTextItems(
+                parsedIndex.nav.title,
+                combinedVars,
+                {resolveConditions: true},
+            );
+            parsedIndex.nav.title = liquidField(navTitle, combinedVars, path);
+        }
+
+        parsedIndex.links = filterFiles(parsedIndex.links, 'links', combinedVars, {resolveConditions: true});
+
+        parsedIndex.links.forEach((link) => {
+            if (link.title) {
+                link.title = liquidField(link.title, combinedVars, path);
+            }
+            if (link.description) {
+                link.description = liquidField(link.description, combinedVars, path);
+            }
+        });
+
+        writeFileSync(filePath, dump(parsedIndex));
+    } catch (error) {
+        log.error(`Error while filtering index file: ${path}. Error message: ${error}`);
+    }
+}
+
+export default {
+    filterFile,
+};

package/src/services/metadata.ts

@@ -0,0 +1,249 @@
+import {dump} from 'js-yaml';
+
+import {VCSConnector} from '../vcs-connector/connector-models';
+import {Metadata, MetaDataOptions, Resources} from '../models';
+import {
+    getAuthorDetails,
+    updateAuthorMetadataStringByAuthorLogin,
+    updateAuthorMetadataStringByFilePath,
+} from './authors';
+import {getFileContributorsMetadata, getFileContributorsString} from './contributors';
+import {isObject} from './utils';
+import {сarriage} from '../utils';
+import {metadataBorder, REGEXP_AUTHOR} from '../constants';
+import {dirname, relative, resolve} from 'path';
+import {ArgvService} from './index';
+
+async function getContentWithUpdatedMetadata(
+    fileContent: string,
+    options?: MetaDataOptions,
+    systemVars?: unknown,
+): Promise<string> {
+    let result;
+
+    result = getContentWithUpdatedStaticMetadata({
+        fileContent,
+        sourcePath: options?.fileData?.sourcePath,
+        addSystemMeta: options?.addSystemMeta,
+        addSourcePath: options?.addSourcePath,
+        resources: options?.resources,
+        systemVars,
+    });
+    result = await getContentWithUpdatedDynamicMetadata(result, options);
+
+    return result;
+}
+
+function getContentWithUpdatedStaticMetadata({
+    fileContent, sourcePath, addSystemMeta, addSourcePath, resources, systemVars,
+}: {
+    fileContent: string;
+    sourcePath?: string;
+    addSystemMeta?: boolean;
+    addSourcePath?: boolean;
+    resources?: Resources;
+    systemVars?: unknown;
+}): string {
+    const newMetadatas: string[] = [];
+
+    if ((!addSystemMeta || !systemVars) && !addSourcePath && !resources) {
+        return fileContent;
+    }
+
+    const matches = matchMetadata(fileContent);
+
+    if (addSystemMeta && systemVars && isObject(systemVars)) {
+        newMetadatas.push(getSystemVarsMetadataString(systemVars));
+    }
+
+    if (resources) {
+        newMetadatas.push(dump(resources));
+    }
+
+    if (addSourcePath && sourcePath) {
+        const sourcePathMetadataString = `sourcePath: ${sourcePath}`;
+        newMetadatas.push(sourcePathMetadataString);
+    }
+
+    if (matches && matches.length > 0) {
+        const [, fileMetadata, , fileMainContent] = matches;
+
+        return `${getUpdatedMetadataString(newMetadatas, fileMetadata)}${fileMainContent}`;
+    }
+
+    return `${getUpdatedMetadataString(newMetadatas)}${fileContent}`;
+}
+
+async function getContentWithUpdatedDynamicMetadata(
+    fileContent: string,
+    options?: MetaDataOptions,
+): Promise<string> {
+    if (!options || !options?.isContributorsEnabled) {
+        return fileContent;
+    }
+
+    let fileMetadata: string | undefined, fileMainContent: string | undefined;
+    const matches = matchMetadata(fileContent);
+    if (matches && matches.length > 0) {
+        const [, matchedFileMetadata, , matchedFileMainContent] = matches;
+        fileMetadata = matchedFileMetadata;
+        fileMainContent = matchedFileMainContent;
+    }
+
+    const newMetadatas: string[] = [];
+
+    const {isContributorsEnabled} = options;
+
+    if (isContributorsEnabled) {
+        const contributorsMetaData = await getContributorsMetadataString(options, fileContent);
+        if (contributorsMetaData) {
+            newMetadatas.push(contributorsMetaData);
+        }
+
+        let authorMetadata = '';
+        if (fileMetadata) {
+            const matchAuthor = fileMetadata.match(REGEXP_AUTHOR);
+            if (matchAuthor) {
+                const matchedAuthor = matchAuthor[0];
+                authorMetadata = await updateAuthorMetadataStringByAuthorLogin(matchedAuthor, options.vcsConnector);
+            }
+        }
+
+        if (!authorMetadata) {
+            const {fileData: {tmpInputFilePath, inputFolderPathLength}} = options;
+            const relativeFilePath = tmpInputFilePath.substring(inputFolderPathLength);
+            authorMetadata = await updateAuthorMetadataStringByFilePath(relativeFilePath, options.vcsConnector);
+        }
+
+        if (authorMetadata) {
+            newMetadatas.push(`author: ${authorMetadata}`);
+        }
+    }
+
+    if (fileMetadata && fileMainContent) {
+        let updatedFileMetadata = fileMetadata;
+        const matchAuthor = fileMetadata.match(REGEXP_AUTHOR);
+
+        const isNewMetadataIncludesAuthor = newMetadatas.some((item) => /^author: /.test(item));
+        if (matchAuthor && isNewMetadataIncludesAuthor) {
+            updatedFileMetadata = updatedFileMetadata.replace(`author: ${matchAuthor[0]}`, '');
+        }
+
+        return `${getUpdatedMetadataString(newMetadatas, updatedFileMetadata)}${fileMainContent}`;
+    }
+
+    return `${getUpdatedMetadataString(newMetadatas)}${fileContent}`;
+}
+
+function matchMetadata(fileContent: string) {
+    if (!fileContent.startsWith('---')) {
+        return null;
+    }
+
+    // Search by format:
+    // ---
+    // metaName1: metaValue1
+    // metaName2: meta value2
+    // incorrectMetadata
+    // ---
+    const regexpMetadata = '(?<=-{3}\\r?\\n)((.*\\r?\\n)*?)(?=-{3}\\r?\\n)';
+    // Search by format:
+    // ---
+    // main content 123
+    const regexpFileContent = '-{3}((.*[\r?\n]*)*)';
+
+    const regexpParseFileContent = new RegExp(`${regexpMetadata}${regexpFileContent}`, 'gm');
+
+    return regexpParseFileContent.exec(fileContent);
+}
+
+async function getContributorsMetadataString(
+    options: MetaDataOptions, fileContent: string,
+): Promise<string | undefined> {
+    const {isContributorsEnabled, vcsConnector, fileData} = options;
+
+    if (isContributorsEnabled && vcsConnector) {
+        const updatedFileData = {
+            ...fileData,
+            fileContent,
+        };
+
+        return getFileContributorsMetadata(updatedFileData, vcsConnector);
+    }
+
+    return undefined;
+}
+
+function getUpdatedMetadataString(newMetadatas: string[], defaultMetadata = ''): string {
+    const newMetadata = newMetadatas.join(сarriage) + (newMetadatas.length ? сarriage : '');
+    const preparedDefaultMetadata = defaultMetadata.trimRight();
+    const defaultMetadataСarriage = preparedDefaultMetadata ? сarriage : '';
+    const updatedMetadata = `${preparedDefaultMetadata}${defaultMetadataСarriage}${newMetadata}`;
+
+    return `${metadataBorder}${сarriage}${updatedMetadata}${metadataBorder}${defaultMetadata.length ? '' : сarriage}`;
+}
+
+async function getUpdatedMetadata(options: MetaDataOptions, fileContent: string, meta?: Metadata): Promise<Metadata> {
+    const {vcsConnector} = options;
+
+    const newMetadata: Metadata = {
+        contributors: await getContributorsMetadata(options, fileContent),
+    };
+
+    if (!meta) {
+        return newMetadata;
+    }
+
+    const updatedAuthor = await getAuthorMetadata(meta as Metadata, vcsConnector);
+
+    return {
+        ...meta,
+        ...newMetadata,
+        author: updatedAuthor,
+    };
+}
+
+async function getContributorsMetadata(options: MetaDataOptions, fileContent: string): Promise<string> {
+    const {isContributorsEnabled, vcsConnector, fileData} = options;
+
+    if (isContributorsEnabled && vcsConnector) {
+        const updatedFileData = {
+            ...fileData,
+            fileContent,
+        };
+
+        return getFileContributorsString(updatedFileData, vcsConnector);
+    }
+
+    return JSON.stringify([]);
+}
+
+async function getAuthorMetadata(meta: Metadata, vcsConnector?: VCSConnector): Promise<string | null> {
+    if (meta.author && vcsConnector) {
+        const updatedAuthor = await getAuthorDetails(vcsConnector, meta.author);
+
+        return updatedAuthor;
+    }
+
+    return null;
+}
+
+function getSystemVarsMetadataString(systemVars: object) {
+    return `__system: ${JSON.stringify(systemVars)}`;
+}
+
+
+function getAssetsPublicPath(filePath: string) {
+    const {input} = ArgvService.getConfig();
+    const path: string = resolve(input, filePath);
+
+    /* Relative path from folder of .md file to root of user' output folder */
+    return relative(dirname(path), resolve(input));
+}
+
+export {
+    getContentWithUpdatedMetadata,
+    getContentWithUpdatedStaticMetadata,
+    getUpdatedMetadata,
+    getAssetsPublicPath,
+};

package/src/services/plugins.ts

@@ -0,0 +1,76 @@
+import {LintRule, LintConfig} from '@doc-tools/transform/lib/yfmlint';
+
+import {PluginOptions, Plugin, CollectionOfPluginsFunction} from '../models';
+import {YFM_PLUGINS} from '../constants';
+
+let plugins: Function[] | Plugin[];
+let collectionOfPlugins: CollectionOfPluginsFunction;
+
+export function setPlugins(): void {
+    plugins = getAllPlugins();
+    collectionOfPlugins = makeCollectOfPlugins();
+}
+
+export function getPlugins() {
+    return plugins;
+}
+
+export function getCollectOfPlugins(): CollectionOfPluginsFunction {
+    return collectionOfPlugins;
+}
+
+function makeCollectOfPlugins(): CollectionOfPluginsFunction {
+    const pluginsWithCollect = (plugins as Plugin[]).filter((plugin: Plugin) => {
+        return typeof plugin.collect === 'function';
+    });
+
+    return (output: string, options: PluginOptions) => {
+        let collectsOutput = output;
+
+        pluginsWithCollect.forEach((plugin: Plugin) => {
+            const collectOutput = plugin.collect(collectsOutput, options);
+
+            collectsOutput = typeof collectOutput === 'string' ? collectOutput : collectsOutput;
+        });
+
+        return collectsOutput;
+    };
+}
+
+function getAllPlugins(): Function[] {
+    const customPlugins = getCustomPlugins();
+    return [...YFM_PLUGINS, ...customPlugins];
+}
+
+function getCustomPlugins(): Function[] {
+    try {
+        const customPlugins = require(require.resolve('./plugins'));
+        return Array.isArray(customPlugins) ? customPlugins : [];
+    } catch (e) {
+        return [];
+    }
+}
+
+export function getHeadContent(): string {
+    try {
+        return require(require.resolve('./head-content.js'));
+    } catch (e) {
+        return '';
+    }
+}
+
+export function getCustomLintRules(): LintRule[] {
+    try {
+        return require(require.resolve('./lint-rules'));
+    } catch (e) {
+        return [];
+    }
+}
+
+export function getDefaultLintConfig(): LintConfig | undefined {
+    try {
+        return require(require.resolve('./default-lint-config'));
+    } catch (e) {
+        return undefined;
+    }
+}