@diplodoc/cli 4.0.0

package/src/vcs-connector/client/github.ts

@@ -0,0 +1,52 @@
+import log from '@doc-tools/transform/lib/log';
+import {Octokit} from '@octokit/core';
+import {ArgvService} from '../../services';
+import {
+    GithubCommitDTO,
+    GitHubConnectorFields,
+    SourceType,
+    GithubUserDTO,
+} from '../connector-models';
+import {validateConnectorFields} from '../connector-validator';
+
+async function getRepoUser(octokit: Octokit, username: string): Promise<GithubUserDTO | null> {
+    try {
+        const user = await octokit.request('GET /users/{username}', {
+            username,
+        });
+
+        return user.data as GithubUserDTO;
+    } catch (error) {
+        log.warn(`Getting user for GitHub has been failed. Username: ${username}. Error: ${error}`);
+        return null;
+    }
+}
+
+async function getRepoCommitByHash(httpClientByToken: Octokit, hashCommit: string): Promise<GithubCommitDTO | null> {
+    const {connector} = ArgvService.getConfig();
+
+    const neededProperties = [GitHubConnectorFields.OWNER, GitHubConnectorFields.REPO];
+    const validatedFileds = validateConnectorFields(SourceType.GITHUB, neededProperties, connector);
+
+    if (Object.keys(validatedFileds).length === 0) {
+        return null;
+    }
+
+    try {
+        const commit = await httpClientByToken.request('GET /repos/{owner}/{repo}/commits/{commit_sha}', {
+            owner: validatedFileds[GitHubConnectorFields.OWNER] as string,
+            repo: validatedFileds[GitHubConnectorFields.REPO] as string,
+            commit_sha: hashCommit,
+        });
+
+        return commit.data;
+    } catch (error) {
+        log.warn(`Getting commit by sha has been failed for GitHub. SHA commit: ${hashCommit}. Error: ${error}`);
+        return null;
+    }
+}
+
+export default {
+    getRepoUser,
+    getRepoCommitByHash,
+};

package/src/vcs-connector/connector-models.ts

@@ -0,0 +1,76 @@
+import {
+    Contributors,
+    ContributorsByPathFunction,
+    ExternalAuthorByPathFunction,
+    NestedContributorsForPathFunction,
+    UserByLoginFunction,
+} from '../models';
+
+/* eslint-disable camelcase */
+export interface ConnectorValidatorProps {
+    validateFn: (value: unknown) => Boolean;
+    defaultValue?: unknown;
+    errorMessage?: string;
+    warnMessage?: string;
+    relatedValidator?: Record<string, ConnectorValidatorProps>;
+}
+
+export enum SourceType {
+    GITHUB = 'github',
+}
+
+export enum GitHubConnectorFields {
+    OWNER = 'owner',
+    REPO = 'repo',
+    TOKEN = 'token',
+    ENDPOINT = 'endpoint',
+}
+
+export interface VCSConnector {
+    getExternalAuthorByPath: ExternalAuthorByPathFunction;
+    addNestedContributorsForPath: NestedContributorsForPathFunction;
+    getContributorsByPath: ContributorsByPathFunction;
+    getUserByLogin: UserByLoginFunction;
+}
+
+export interface VCSConnectorConfig {
+    type: string;
+    [SourceType.GITHUB]?: {
+        endpoint: string;
+        token: string;
+        owner: string;
+        repo: string;
+    };
+}
+
+export interface FileContributors {
+    contributors: Contributors;
+    hasIncludes: boolean;
+}
+
+export interface GithubCommitDTO {
+    commit: {
+        author: {
+            name: string;
+            email: string;
+        };
+    };
+    author: {
+        login: string;
+        avatar_url: string;
+        html_url: string;
+    };
+}
+
+export interface GithubUserDTO {
+    avatar_url: string;
+    html_url: string;
+    email: string;
+    login: string;
+    name: string;
+}
+
+export interface GitLogsDTO {
+    author_email: string;
+    author_name: string;
+}

package/src/vcs-connector/connector-validator.ts

@@ -0,0 +1,114 @@
+import log from '@doc-tools/transform/lib/log';
+import {ConnectorValidatorProps, GitHubConnectorFields, SourceType, VCSConnectorConfig} from './connector-models';
+import {getMsgСonfigurationMustBeProvided} from '../constants';
+
+const githubConnectorValidator: Record<string, ConnectorValidatorProps> = {
+    [GitHubConnectorFields.ENDPOINT]: {
+        warnMessage: `'${GitHubConnectorFields.ENDPOINT}' must be provided for GitHub repo.`,
+        validateFn: notEmptyValue,
+        defaultValue: process.env.GITHUB_BASE_URL,
+    },
+    [GitHubConnectorFields.TOKEN]: {
+        warnMessage: `'${GitHubConnectorFields.TOKEN}' must be provided for GitHub repo.`,
+        validateFn: notEmptyValue,
+        defaultValue: process.env.GITHUB_TOKEN,
+    },
+    [GitHubConnectorFields.OWNER]: {
+        warnMessage: `'${GitHubConnectorFields.OWNER}' must be provided for GitHub repo.`,
+        validateFn: notEmptyValue,
+        defaultValue: process.env.GITHUB_OWNER,
+    },
+    [GitHubConnectorFields.REPO]: {
+        warnMessage: `'${GitHubConnectorFields.REPO}' must be provided for GitHub repo.`,
+        validateFn: notEmptyValue,
+        defaultValue: process.env.GITHUB_REPO,
+    },
+};
+
+const connectorValidator: Record<string, ConnectorValidatorProps> = {
+    'type': {
+        warnMessage: '\'type\' must be provided for repo.',
+        validateFn: notEmptyValue,
+    },
+    [SourceType.GITHUB]: {
+        warnMessage: `'${SourceType.GITHUB}' object must be filled needed fields.`,
+        validateFn: notEmptyObject,
+        relatedValidator: githubConnectorValidator,
+    },
+};
+
+function notEmptyObject(filed?: unknown): boolean {
+    if (typeof filed === 'object') {
+        return Boolean(filed && Object.getOwnPropertyNames(filed).length);
+    }
+
+    return false;
+}
+
+function notEmptyValue(value?: unknown): boolean {
+    if (typeof value === 'string') {
+        return Boolean(value);
+    }
+
+    return false;
+}
+
+export function validateConnectorFields(
+    sourceType: SourceType,
+    fieldNames: GitHubConnectorFields[],
+    repoProperties?: VCSConnectorConfig): Record<string, unknown> {
+
+    const repoValidator = connectorValidator[sourceType];
+
+    if (!repoValidator) {
+        log.error(`Invalid repo type: ${repoValidator}`);
+        return {};
+    }
+
+    const isValidRepo = repoValidator.validateFn(repoProperties && repoProperties[sourceType]);
+    const relatedRepoValidator = repoValidator.relatedValidator;
+    if (!repoProperties || !isValidRepo || !relatedRepoValidator) {
+        createLog(repoValidator);
+        return {};
+    }
+
+    let isValidProperties = true;
+    const validatedFields: Record<string, unknown> = {};
+
+    for (const property of fieldNames) {
+        const propertyValidator = relatedRepoValidator[property];
+
+        if (!propertyValidator) {
+            log.warn(`The property '${property}' doesn't exist in ${sourceType} repo.`);
+            continue;
+        }
+
+        const propertyValue = propertyValidator.defaultValue || repoProperties[sourceType]?.[property];
+
+        if (!propertyValidator.validateFn(propertyValue)) {
+            createLog(propertyValidator);
+            isValidProperties = false;
+        }
+
+        validatedFields[property] = propertyValue;
+    }
+
+    if (isValidProperties) {
+        return validatedFields;
+    }
+
+    log.warn(getMsgСonfigurationMustBeProvided(sourceType));
+    return {};
+}
+
+function createLog(validator: ConnectorValidatorProps): void {
+    if (validator.errorMessage) {
+        return log.error(validator.errorMessage);
+    }
+
+    if (validator.warnMessage) {
+        return log.warn(validator.warnMessage);
+    }
+
+    throw new Error(`Invalid validator: ${JSON.stringify(validator)}.`);
+}

package/src/vcs-connector/github.ts

@@ -0,0 +1,333 @@
+import {Octokit} from '@octokit/core';
+import {join, normalize} from 'path';
+import simpleGit, {SimpleGitOptions} from 'simple-git';
+import {minimatch} from 'minimatch';
+
+import github from './client/github';
+import {ArgvService} from '../services';
+import {
+    CommitInfo,
+    Contributor,
+    Contributors,
+    ContributorsByPathFunction,
+    ExternalAuthorByPathFunction,
+    NestedContributorsForPathFunction,
+} from '../models';
+import {
+    FileContributors,
+    GitHubConnectorFields,
+    SourceType,
+    VCSConnector,
+} from './connector-models';
+import {
+    ALL_CONTRIBUTORS_RECEIVED,
+    FIRST_COMMIT_FROM_ROBOT_IN_GITHUB,
+    GETTING_ALL_CONTRIBUTORS,
+} from '../constants';
+import {addSlashPrefix, logger} from '../utils';
+import {validateConnectorFields} from './connector-validator';
+import process from 'process';
+
+const authorByGitEmail: Map<string, Contributor | null> = new Map();
+const authorByPath: Map<string, Contributor | null> = new Map();
+const contributorsByPath: Map<string, FileContributors> = new Map();
+const contributorsData: Map<string, Contributor | null> = new Map();
+
+async function getGitHubVCSConnector(): Promise<VCSConnector | undefined> {
+    const {contributors} = ArgvService.getConfig();
+
+    const httpClientByToken = getHttpClientByToken();
+    if (!httpClientByToken) {
+        return undefined;
+    }
+
+    let addNestedContributorsForPath: NestedContributorsForPathFunction = () => { };
+    let getContributorsByPath: ContributorsByPathFunction = () => Promise.resolve({} as FileContributors);
+    const getExternalAuthorByPath: ExternalAuthorByPathFunction = (path: string) => authorByPath.get(path) ?? null;
+
+    if (contributors) {
+        await getAllContributorsTocFiles(httpClientByToken);
+        addNestedContributorsForPath = (path: string, nestedContributors: Contributors) =>
+            addNestedContributorsForPathFunction(path, nestedContributors);
+        getContributorsByPath = async (path: string) => getFileContributorsByPath(path);
+    }
+
+    return {
+        getExternalAuthorByPath,
+        addNestedContributorsForPath,
+        getContributorsByPath,
+        getUserByLogin: (login: string) => getUserByLogin(httpClientByToken, login),
+    };
+}
+
+function getHttpClientByToken(): Octokit | null {
+    const {connector, contributors} = ArgvService.getConfig();
+
+    if (!contributors) {
+        return null;
+    }
+
+    const neededProperties = [GitHubConnectorFields.TOKEN, GitHubConnectorFields.ENDPOINT];
+    const validatedFileds = validateConnectorFields(SourceType.GITHUB, neededProperties, connector);
+
+    if (Object.keys(validatedFileds).length === 0) {
+        return null;
+    }
+
+    const octokit = new Octokit({
+        auth: validatedFileds[GitHubConnectorFields.TOKEN] as string,
+        baseUrl: validatedFileds[GitHubConnectorFields.ENDPOINT] as string,
+    });
+
+    return octokit;
+}
+
+async function getAllContributorsTocFiles(httpClientByToken: Octokit): Promise<void> {
+    const {rootInput} = ArgvService.getConfig();
+
+    const options: Partial<SimpleGitOptions> = {
+        baseDir: rootInput,
+    };
+
+    logger.info('', GETTING_ALL_CONTRIBUTORS);
+
+    const masterDir = './_yfm-master';
+    const tmpMasterBranch = 'yfm-tmp-master';
+
+    try {
+        await simpleGit(options).raw('worktree', 'add', '-b', tmpMasterBranch, masterDir, 'origin/master');
+        const fullRepoLogString = await simpleGit({
+            baseDir: join(rootInput, masterDir),
+        }).raw(
+            'log',
+            `${FIRST_COMMIT_FROM_ROBOT_IN_GITHUB}..HEAD`,
+            '--pretty=format:%ae, %an, %H',
+            '--name-only',
+        );
+        const repoLogs = fullRepoLogString.split('\n\n');
+        if (process.env.ENABLE_EXPERIMANTAL_AUTHORS) {
+            const fullAuthorRepoLogString = await simpleGit({
+                baseDir: join(rootInput, masterDir),
+            }).raw(
+                'log',
+                `${FIRST_COMMIT_FROM_ROBOT_IN_GITHUB}..HEAD`,
+                '--diff-filter=A',
+                '--pretty=format:%ae;%an;%H',
+                '--name-only',
+            );
+            const authorRepoLog = fullAuthorRepoLogString.split('\n\n');
+            await matchAuthorsForEachPath(authorRepoLog, httpClientByToken);
+        }
+        await matchContributionsForEachPath(repoLogs, httpClientByToken);
+    } finally {
+        await simpleGit(options).raw('worktree', 'remove', masterDir);
+        await simpleGit(options).raw('branch', '-d', tmpMasterBranch);
+    }
+
+    logger.info('', ALL_CONTRIBUTORS_RECEIVED);
+}
+
+async function matchContributionsForEachPath(repoLogs: string[], httpClientByToken: Octokit): Promise<void> {
+
+    for (const repoLog of repoLogs) {
+        if (!repoLog) {
+            continue;
+        }
+
+        const dataArray = repoLog.split('\n');
+        const userData = dataArray[0];
+        const [email, name, hashCommit] = userData.split(', ');
+
+        if (shouldAuthorBeIgnored({email, name})) {
+            continue;
+        }
+
+        const hasContributorData = contributorsData.get(email);
+
+        let contributorDataByHash;
+
+        if (hasContributorData === undefined) {
+            logger.info('Contributors: Getting data for', email);
+
+            contributorDataByHash = await getContributorDataByHashCommit(httpClientByToken, hashCommit);
+
+            if (contributorDataByHash) {
+                const paths = dataArray.splice(1);
+                addContributorForPath(paths, {
+                    [email]: contributorDataByHash,
+                });
+            }
+            contributorsData.set(email, contributorDataByHash);
+        } else if (hasContributorData) {
+            const paths = dataArray.splice(1);
+            addContributorForPath(paths, {
+                [email]: hasContributorData,
+            });
+        }
+    }
+}
+
+async function matchAuthorsForEachPath(authorRepoLogs: string[], httpClientByToken: Octokit) {
+    for (const repoLog of authorRepoLogs) {
+        if (!repoLog) {
+            continue;
+        }
+
+        const dataArray = repoLog.split('\n');
+        const [userData, ...paths] = dataArray;
+        const [email, name, hashCommit] = userData.split(';');
+
+        if (shouldAuthorBeIgnored({email, name})) {
+            continue;
+        }
+
+        await getAuthorByPaths({email, hashCommit}, paths, httpClientByToken);
+    }
+}
+
+async function getContributorDataByHashCommit(httpClientByToken: Octokit, hashCommit: string,
+): Promise<Contributor | null> {
+    const repoCommit = await github.getRepoCommitByHash(httpClientByToken, hashCommit);
+
+    if (!repoCommit) {
+        return null;
+    }
+
+    const {author, commit} = repoCommit;
+
+    if (!author) {
+        return null;
+    }
+
+    const {avatar_url: avatar, html_url: url, login} = author;
+
+    return {
+        avatar,
+        email: commit.author.email,
+        login,
+        name: commit.author.name,
+        url,
+    };
+}
+
+async function getAuthorByPaths(commitInfo: CommitInfo, paths: string[], httpClientByToken: Octokit) {
+    for (const path of paths) {
+        if (!path) {
+            continue;
+        }
+        const normalizePath = normalize(addSlashPrefix(path));
+        const {email, hashCommit} = commitInfo;
+
+        let authorToReturn = authorByGitEmail.get(email) || null;
+
+        if (!authorToReturn) {
+            logger.info('Authors: Getting data for', email);
+
+            const repoCommit = await github.getRepoCommitByHash(httpClientByToken, hashCommit);
+            if (!repoCommit) {
+                continue;
+            }
+
+            const {author, commit} = repoCommit;
+            if (!author) {
+                continue;
+            }
+
+            const {avatar_url: avatar, html_url: url, login} = author;
+            authorToReturn = {
+                avatar,
+                email: commit.author.email,
+                login,
+                name: commit.author.name,
+                url,
+            };
+            authorByGitEmail.set(email, authorToReturn);
+        }
+
+        authorByPath.set(normalizePath, authorToReturn);
+    }
+}
+
+async function getFileContributorsByPath(path: string): Promise<FileContributors> {
+    if (contributorsData.size === 0 || !contributorsByPath.has(path)) {
+        return {} as FileContributors;
+    }
+
+    return contributorsByPath.get(path) as FileContributors;
+}
+
+async function getUserByLogin(octokit: Octokit, userLogin: string): Promise<Contributor | null> {
+    const user = await github.getRepoUser(octokit, userLogin);
+
+    if (!user) {
+        return null;
+    }
+
+    const {avatar_url: avatar, html_url: url, email, login, name} = user;
+
+    return {
+        avatar,
+        email,
+        login,
+        name,
+        url,
+    };
+}
+
+function addNestedContributorsForPathFunction(path: string, nestedContributors: Contributors): void {
+    addContributorForPath([path], nestedContributors, true);
+}
+
+function addContributorForPath(paths: string[], newContributor: Contributors, hasIncludes = false): void {
+    paths.forEach((path: string) => {
+        const normalizePath = normalize(addSlashPrefix(path));
+
+        if (!contributorsByPath.has(normalizePath)) {
+            contributorsByPath.set(normalizePath, {
+                contributors: newContributor,
+                hasIncludes,
+            });
+            return;
+        }
+
+        const oldContributors = contributorsByPath.get(normalizePath);
+
+        contributorsByPath.set(normalizePath, {
+            contributors: {
+                ...oldContributors?.contributors,
+                ...newContributor,
+            },
+            hasIncludes,
+        });
+    });
+}
+
+type ShouldAuthorBeIgnoredArgs = {
+    email?: string;
+    name?: string;
+};
+
+function shouldAuthorBeIgnored({email, name}: ShouldAuthorBeIgnoredArgs) {
+    if (!(email || name)) {
+        return false;
+    }
+
+    const {ignoreAuthorPatterns} = ArgvService.getConfig();
+    if (!ignoreAuthorPatterns) {
+        return false;
+    }
+
+    for (const pattern of ignoreAuthorPatterns) {
+        if (email && minimatch(email, pattern)) {
+            return true;
+        }
+
+        if (name && minimatch(name, pattern)) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+export default getGitHubVCSConnector;

package/src/vcs-connector/index.ts

@@ -0,0 +1,15 @@
+import {ArgvService} from '../services';
+import getGitHubVCSConnector from './github';
+import {VCSConnector, SourceType} from './connector-models';
+
+export async function getVCSConnector(): Promise<VCSConnector | undefined> {
+    const {connector} = ArgvService.getConfig();
+    const connectorType = process.env.VCS_CONNECTOR_TYPE || connector && connector.type;
+
+    switch (connectorType) {
+        case SourceType.GITHUB:
+            return getGitHubVCSConnector();
+        default:
+            return undefined;
+    }
+}

package/src/workers/linter/index.ts

@@ -0,0 +1,62 @@
+import log from '@doc-tools/transform/lib/log';
+import {extname} from 'path';
+import {Observable, Subject} from 'threads/observable';
+import {expose} from 'threads';
+
+import {ArgvService, PluginService, PresetService, TocService} from '../../services';
+import {TocServiceData} from '../../services/tocs';
+import {PresetStorage} from '../../services/preset';
+import {YfmArgv} from '../../models';
+import {lintPage} from '../../resolvers';
+
+let processedPages = new Subject();
+
+interface ProcessLinterWorkerOptions {
+    argvConfig: YfmArgv;
+    navigationPaths: TocServiceData['navigationPaths'];
+    presetStorage: PresetStorage;
+}
+
+async function run({
+    argvConfig,
+    presetStorage,
+    navigationPaths,
+}: ProcessLinterWorkerOptions) {
+    ArgvService.set(argvConfig);
+    PresetService.setPresetStorage(presetStorage);
+    TocService.setNavigationPaths(navigationPaths);
+    PluginService.setPlugins();
+
+    TocService.getNavigationPaths().forEach((pathToFile) => {
+        lintPage({
+            inputPath: pathToFile,
+            fileExtension: extname(pathToFile),
+            onFinish: () => {
+                processedPages.next(pathToFile);
+            },
+        });
+    });
+}
+
+async function finish() {
+    processedPages.complete();
+    processedPages = new Subject();
+
+    return log.get();
+}
+
+function getProcessedPages() {
+    return Observable.from(processedPages);
+}
+
+export type ProcessLinterWorker = {
+    run: typeof run;
+    finish: typeof finish;
+    getProcessedPages: typeof getProcessedPages;
+};
+
+expose({
+    run,
+    finish,
+    getProcessedPages,
+});