@diplodoc/cli 4.0.0

package/src/cmd/xliff/index.ts

@@ -0,0 +1,27 @@
+import {extract} from './extract';
+import {compose} from './compose';
+
+import {Argv} from 'yargs';
+
+const command = 'xliff';
+
+const description =
+    'extract xliff and skeleton from documentation files\ncompose xliff and skeleton into documentation';
+
+const xliff = {
+    command,
+    description,
+    handler: () => {},
+    builder,
+};
+
+function builder<T>(argv: Argv<T>) {
+    return argv
+        .command(extract)
+        .command(compose)
+        .demandCommand(1, `provide one of the folowing ${command} commands: ${extract.command}, ${compose.command}`);
+}
+
+export {xliff};
+
+export default {xliff};

package/src/constants.ts

@@ -0,0 +1,122 @@
+import {dirname} from 'path';
+const os = require('os');
+const notes = require('@doc-tools/transform/lib/plugins/notes');
+const anchors = require('@doc-tools/transform/lib/plugins/anchors');
+const code = require('@doc-tools/transform/lib/plugins/code');
+const cut = require('@doc-tools/transform/lib/plugins/cut');
+const deflist = require('@doc-tools/transform/lib/plugins/deflist');
+const imsize = require('@doc-tools/transform/lib/plugins/imsize');
+const meta = require('@doc-tools/transform/lib/plugins/meta');
+const sup = require('@doc-tools/transform/lib/plugins/sup');
+const tabs = require('@doc-tools/transform/lib/plugins/tabs');
+const video = require('@doc-tools/transform/lib/plugins/video');
+const includes = require('@doc-tools/transform/lib/plugins/includes');
+const links = require('@doc-tools/transform/lib/plugins/links');
+const images = require('@doc-tools/transform/lib/plugins/images');
+const monospace = require('@doc-tools/transform/lib/plugins/monospace');
+const table = require('@doc-tools/transform/lib/plugins/table');
+const term = require('@doc-tools/transform/lib/plugins/term');
+const changelog = require('@doc-tools/transform/lib/plugins/changelog');
+const mermaid = require('@diplodoc/mermaid-extension');
+const openapi = require('@diplodoc/openapi-extension');
+
+includes.collect = require('@doc-tools/transform/lib/plugins/includes/collect');
+images.collect = require('@doc-tools/transform/lib/plugins/images/collect');
+changelog.collect = require('@doc-tools/transform/lib/plugins/changelog/collect');
+
+export const BUILD_FOLDER = 'build';
+export const BUNDLE_FOLDER = '_bundle';
+export const BUNDLE_JS_FILENAME = 'app.client.js';
+export const BUNDLE_CSS_FILENAME = 'app.client.css';
+export const TMP_INPUT_FOLDER = '.tmp_input';
+export const TMP_OUTPUT_FOLDER = '.tmp_output';
+export const MAIN_TIMER_ID = 'Build time';
+export const YFM_CONFIG_FILENAME = '.yfm';
+export const REDIRECTS_FILENAME = 'redirects.yaml';
+export const LINT_CONFIG_FILENAME = '.yfmlint';
+export const SINGLE_PAGE_FILENAME = 'single-page.html';
+export const SINGLE_PAGE_DATA_FILENAME = 'single-page.json';
+export const CUSTOM_STYLE = 'custom-style';
+
+export enum Stage {
+    NEW = 'new',
+    PREVIEW = 'preview',
+    TECH_PREVIEW = 'tech-preview',
+    SKIP = 'skip',
+}
+
+export enum Lang {
+    RU = 'ru',
+    EN = 'en',
+}
+
+export enum Platforms {
+    WINDOWS = 'win32',
+    MAC = 'darwin',
+    LINUX = 'linux',
+}
+
+export enum IncludeMode {
+    ROOT_MERGE = 'root_merge',
+    MERGE = 'merge',
+    LINK = 'link'
+}
+
+export enum ResourceType {
+    style = 'style',
+    script = 'script',
+}
+
+export const BUILD_FOLDER_PATH = dirname(require.resolve('@diplodoc/client'));
+
+export const YFM_PLUGINS = [
+    meta,
+    deflist,
+    includes,
+    cut,
+    links,
+    images,
+    notes,
+    anchors,
+    tabs,
+    code,
+    imsize,
+    sup,
+    video,
+    monospace,
+    table,
+    term,
+    openapi.transform(),
+    mermaid.transform(),
+    changelog,
+];
+
+export const PROCESSING_FINISHED = 'Processing finished:';
+export const LINTING_FINISHED = 'Linting finished:';
+export const GETTING_ALL_CONTRIBUTORS = 'Getting all contributors.';
+export const ALL_CONTRIBUTORS_RECEIVED = 'All contributors received.';
+export const getMsgСonfigurationMustBeProvided =
+    (repo: string) => `Сonfiguration must be provided for ${repo} like env variables or in .yfm file`;
+
+export const FIRST_COMMIT_FROM_ROBOT_IN_GITHUB = '2dce14271359cd20d7e874956d604de087560cf4';
+
+// Include example: 'master\n' or 'nanov94/QUEUE-1234_some_branch_name.1.2.3\n'
+// Regexp result: 'master' or 'nanov94/QUEUE-1234_some_branch_name'
+export const REGEXP_BRANCH_NAME = /([\d\w\-_/.]+)(?=\r?\n)/g;
+
+// Include example: {% include [createfolder](create-folder.md) %}
+// Regexp result: [createfolder](create-folder.md)
+export const REGEXP_INCLUDE_CONTENTS = /(?<=[{%]\sinclude\s).+(?=\s[%}])/gm;
+
+// Include example: [createfolder](create-folder.md)
+// Regexp result: create-folder.md
+export const REGEXP_INCLUDE_FILE_PATH = /(?<=[(]).+(?=[)])/g;
+
+// Include example: author: authorLogin
+// Regexp result: authorLogin
+export const REGEXP_AUTHOR = /(?<=author:\s).+(?=\r?\n)/g;
+
+export const MIN_CHUNK_SIZE = Number(process.env.MIN_CHUNK_SIZE) || 1000;
+export const WORKERS_COUNT = Number(process.env.WORKERS_COUNT) || (os.cpus().length - 1);
+
+export const metadataBorder = '---';

package/src/globals.d.ts

@@ -0,0 +1 @@
+declare const VERSION: string;

package/src/index.ts

@@ -0,0 +1,54 @@
+import yargs from 'yargs';
+import {hideBin} from 'yargs/helpers';
+import log from '@doc-tools/transform/lib/log';
+import 'threads/register';
+
+import {MAIN_TIMER_ID} from './constants';
+
+import {build, xliff, translate, publish} from './cmd';
+
+console.time(MAIN_TIMER_ID);
+
+yargs
+    .command(build)
+    .command(publish)
+    .command(xliff)
+    .command(translate)
+    .option('config', {
+        alias: 'c',
+        describe: 'YFM configuration file',
+        type: 'string',
+    })
+    .option('strict', {
+        alias: 's',
+        default: false,
+        describe: 'Run in strict mode',
+        type: 'boolean',
+    })
+    .option('quiet', {
+        alias: 'q',
+        default: false,
+        describe: 'Run in quiet mode. Don\'t write logs to stdout',
+        type: 'boolean',
+    })
+    .group(['config', 'strict', 'quiet', 'help', 'version'], 'Common options:')
+    .version(typeof VERSION !== 'undefined' ? VERSION : '')
+    .help()
+    .parse(hideBin(process.argv), {}, (err, {strict}, output) => {
+        console.timeEnd(MAIN_TIMER_ID);
+
+        if (err) {
+            console.error(err);
+            process.exit(1);
+        }
+
+        const {warn, error} = log.get();
+
+        if (strict && warn.length || error.length) {
+            process.exit(1);
+        }
+
+        console.log(output);
+
+        process.exit(0);
+    });

package/src/models.ts

@@ -0,0 +1,249 @@
+import {Logger} from '@doc-tools/transform/lib/log';
+import {LintConfig} from '@doc-tools/transform/lib/yfmlint';
+
+import {FileContributors, VCSConnector, VCSConnectorConfig} from './vcs-connector/connector-models';
+import {Lang, Stage, IncludeMode, ResourceType} from './constants';
+import {ChangelogItem} from '@doc-tools/transform/lib/plugins/changelog/types';
+
+export type VarsPreset = 'internal'|'external';
+
+export type YfmPreset = Record<string, string>;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type Metadata = Record<string, any>;
+
+export type ExternalAuthorByPathFunction = (path: string) => Contributor | null;
+export type ContributorsByPathFunction = (path: string) => Promise<FileContributors>;
+export type NestedContributorsForPathFunction = (path: string, nestedContributors: Contributors) => void;
+export type UserByLoginFunction = (login: string) => Promise<Contributor | null>;
+export type CollectionOfPluginsFunction = (output: string, options: PluginOptions) => string;
+
+interface YfmConfig {
+    varsPreset: VarsPreset;
+    ignore: string[];
+    outputFormat: string;
+    allowHTML: boolean;
+    vars: Record<string, string>;
+    applyPresets: boolean;
+    resolveConditions: boolean;
+    conditionsInCode: boolean;
+    disableLiquid: boolean;
+    strict: boolean;
+    ignoreStage: string;
+    singlePage: boolean;
+    removeHiddenTocItems: boolean;
+    connector?: VCSConnectorConfig;
+    lang?: Lang;
+    lintDisabled: boolean;
+    buildDisabled: boolean;
+    lintConfig: LintConfig;
+    resources?: Resources;
+    yandexCloudTranslateFolderId: string;
+    yandexCloudTranslateGlossaryPairs: YandexCloudTranslateGlossaryPair[];
+}
+
+export interface YfmArgv extends YfmConfig {
+    rootInput: string;
+    input: string;
+    output: string;
+    quiet: string;
+    publish: boolean;
+    storageEndpoint: string;
+    storageBucket: string;
+    storagePrefix: string;
+    storageKeyId: string;
+    storageSecretKey: string;
+    storageRegion: string;
+    contributors: boolean;
+    ignoreAuthorPatterns: string;
+    addSystemMeta: boolean;
+    addMapFile: boolean;
+    allowCustomResources: boolean;
+    staticContent: boolean;
+}
+
+export interface DocPreset {
+    default: YfmPreset;
+    [varsPreset: string]: YfmPreset;
+}
+
+export interface YfmToc extends Filter {
+    name: string;
+    href: string;
+    items: YfmToc[];
+    stage?: Stage;
+    base?: string;
+    title?: TextItems;
+    include?: YfmTocInclude;
+    id?: string;
+    singlePage?: boolean;
+    hidden?: boolean;
+}
+
+export interface YfmTocInclude {
+    repo: string;
+    path: string;
+    mode?: IncludeMode;
+    includers?: YfmTocIncluders;
+}
+
+export type YfmTocIncluders = YfmTocIncluder[];
+
+export type YfmTocIncluder = {
+    name: YfmTocIncluderName;
+    // arbitrary includer parameters
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+} & Record<string, unknown>;
+
+export const includersNames = ['sourcedocs', 'openapi', 'generic', 'unarchive'] as const;
+
+export type YfmTocIncluderName = typeof includersNames[number];
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type Includer<FnParams = any> = {
+    name: YfmTocIncluderName;
+    includerFunction: IncluderFunction<FnParams>;
+};
+
+export type IncluderFunction<PassedParams> = (args: IncluderFunctionParams<PassedParams>) => Promise<void>;
+
+export type IncluderFunctionParams<PassedParams> = {
+    // item that contains include that uses includer
+    item: YfmToc;
+    // base read directory path
+    readBasePath: string;
+    // base write directory path
+    writeBasePath: string;
+    // toc with includer path
+    tocPath: string;
+    vars: YfmPreset;
+    // arbitrary includer parameters
+    passedParams: PassedParams;
+    index: number;
+};
+
+export interface LeadingPage {
+    title: TextItems;
+    description?: TextItems;
+    meta?: {
+        title?: TextItems;
+        description?: TextItems;
+        noIndex?: boolean;
+    };
+    nav?: {
+        title: TextItems;
+    };
+    links: LeadingPageLinks[];
+}
+
+export type TextItems = string | (TextItem | string)[];
+
+export interface TextItem extends Filter {
+    text: string | string[];
+}
+
+export interface LeadingPageLinks extends Filter {
+    title?: string;
+    description?: string;
+    href?: string;
+}
+
+export interface Filter {
+    when?: boolean|string;
+    [key: string]: unknown;
+}
+
+export interface SinglePageResult {
+    path: string;
+    content: string;
+    title?: string;
+}
+
+export interface Contributor {
+    avatar: string;
+    email: string;
+    login: string;
+    name: string;
+    url: string;
+}
+
+export interface Contributors {
+    [email: string]: Contributor;
+}
+
+export interface FileData {
+    tmpInputFilePath: string;
+    inputFolderPathLength: number;
+    fileContent: string;
+    sourcePath?: string;
+}
+
+export interface MetaDataOptions {
+    fileData: FileData;
+    isContributorsEnabled?: boolean;
+    vcsConnector?: VCSConnector;
+    addSystemMeta?: boolean;
+    addSourcePath?: boolean;
+    resources?: Resources;
+}
+
+export interface PluginOptions {
+    vars: YfmPreset;
+    path: string;
+    log: Logger;
+    copyFile: (targetPath: string, targetDestPath: string, options?: PluginOptions) => void;
+    singlePage?: boolean;
+    root?: string;
+    destPath?: string;
+    destRoot?: string;
+    collectOfPlugins?: (input: string, options: PluginOptions) => string;
+    changelogs?: ChangelogItem[];
+    extractChangelogs?: boolean;
+}
+
+export interface Plugin {
+    collect: (input: string, options: PluginOptions) => string | void;
+}
+
+export interface ResolveMd2MdOptions {
+    inputPath: string;
+    outputPath: string;
+    metadata?: MetaDataOptions;
+}
+
+export interface ResolverOptions {
+    inputPath: string;
+    filename: string;
+    fileExtension: string;
+    outputPath: string;
+    outputBundlePath: string;
+    metadata?: MetaDataOptions;
+}
+
+export interface PathData {
+    pathToFile: string;
+    resolvedPathToFile: string;
+    filename: string;
+    fileBaseName: string;
+    fileExtension: string;
+    outputDir: string;
+    outputPath: string;
+    outputFormat: string;
+    outputBundlePath: string;
+    outputTocDir: string;
+    inputFolderPath: string;
+    outputFolderPath: string;
+}
+
+export type Resources = {
+    [key in ResourceType]?: string[];
+};
+
+export type YandexCloudTranslateGlossaryPair = {
+  sourceText: string;
+  translatedText: string;
+};
+
+export type CommitInfo = {
+    email: string;
+    hashCommit: string;
+};

package/src/packages/credentials/index.ts

@@ -0,0 +1 @@
+export * from './yandex-oauth';

package/src/packages/credentials/yandex-oauth.ts

@@ -0,0 +1,42 @@
+import {readFile} from 'fs/promises';
+import {env} from 'process';
+import {homedir} from 'os';
+import {join} from 'path';
+
+import {logger} from '../../utils';
+
+const YANDEX_OAUTH_TOKEN_FILENAME = '.ya_oauth_token';
+
+async function getYandexOAuthToken() {
+    const {YANDEX_OAUTH_TOKEN} = env;
+
+    return YANDEX_OAUTH_TOKEN ?? getYandexOAuthTokenFromHomeDir();
+}
+
+async function getYandexOAuthTokenFromHomeDir() {
+    const error = 'failed reading yandex oauth token';
+
+    const path = join(homedir(), YANDEX_OAUTH_TOKEN_FILENAME);
+
+    let token;
+
+    try {
+        token = await readFile(path, {encoding: 'utf8'});
+
+        token = token.trim();
+
+        if (!token?.length) {
+            throw new Error(error);
+        }
+    } catch (err) {
+        logger.error(error);
+
+        throw err;
+    }
+
+    return token;
+}
+
+export {getYandexOAuthToken};
+
+export default {getYandexOAuthToken};

package/src/resolvers/index.ts

@@ -0,0 +1,3 @@
+export * from './md2md';
+export * from './md2html';
+export * from './lintPage';

package/src/resolvers/lintPage.ts

@@ -0,0 +1,119 @@
+import {dirname, relative, resolve} from 'path';
+import log from '@doc-tools/transform/lib/log';
+import {
+    default as yfmlint,
+    LintMarkdownFunctionOptions,
+    PluginOptions,
+} from '@doc-tools/transform/lib/yfmlint';
+import {readFileSync} from 'fs';
+import {bold} from 'chalk';
+
+import {ArgvService, PluginService} from '../services';
+import {getVarsPerFile, getVarsPerRelativeFile} from '../utils';
+import {liquidMd2Html} from './md2html';
+import {liquidMd2Md} from './md2md';
+
+interface FileTransformOptions {
+    path: string;
+    root?: string;
+}
+
+const FileLinter: Record<string, Function> = {
+    '.md': MdFileLinter,
+};
+
+export interface ResolverLintOptions {
+    inputPath: string;
+    fileExtension: string;
+    onFinish?: () => void;
+}
+
+export function lintPage(options: ResolverLintOptions) {
+    const {inputPath, fileExtension, onFinish} = options;
+    const {input} = ArgvService.getConfig();
+    const resolvedPath: string = resolve(input, inputPath);
+
+    try {
+        const content: string = readFileSync(resolvedPath, 'utf8');
+
+        const lintFn: Function = FileLinter[fileExtension];
+        if (!lintFn) {
+            return;
+        }
+
+        lintFn(content, {path: inputPath});
+    } catch (e) {
+        const message = `No such file or has no access to ${bold(resolvedPath)}`;
+        console.error(message, e);
+        log.error(message);
+    }
+
+    if (onFinish) {
+        onFinish();
+    }
+}
+
+function MdFileLinter(content: string, lintOptions: FileTransformOptions): void {
+    const {
+        input,
+        lintConfig,
+        disableLiquid,
+        outputFormat,
+        ...options
+    } = ArgvService.getConfig();
+    const {path: filePath} = lintOptions;
+
+    const plugins = outputFormat === 'md' ? [] : PluginService.getPlugins();
+    const vars = getVarsPerFile(filePath);
+    const root = resolve(input);
+    const path: string = resolve(input, filePath);
+    let preparedContent = content;
+
+    /* Relative path from folder of .md file to root of user' output folder */
+    const assetsPublicPath = relative(dirname(path), root);
+
+    const lintMarkdown = function lintMarkdown(opts: LintMarkdownFunctionOptions) {
+        const {input: localInput, path: localPath, sourceMap} = opts;
+
+        const pluginOptions: PluginOptions = {
+            ...options,
+            vars,
+            root,
+            path: localPath,
+            lintMarkdown, // Should pass the function for linting included files
+            assetsPublicPath,
+            disableLiquid,
+            log,
+            getVarsPerFile: getVarsPerRelativeFile,
+        };
+
+        yfmlint({
+            input: localInput,
+            lintConfig,
+            pluginOptions,
+            plugins,
+            defaultLintConfig: PluginService.getDefaultLintConfig(),
+            customLintRules: PluginService.getCustomLintRules(),
+            sourceMap,
+        });
+    };
+
+    let sourceMap;
+    if (!disableLiquid) {
+        let liquidResult;
+        if (outputFormat === 'md') {
+            liquidResult = liquidMd2Md(content, vars, path);
+        } else {
+            liquidResult = liquidMd2Html(content, vars, path);
+        }
+
+        preparedContent = liquidResult.output;
+        sourceMap = liquidResult.sourceMap;
+    }
+
+    lintMarkdown({
+        input: preparedContent,
+        path,
+        sourceMap,
+    });
+}