@diplodoc/cli 4.7.0 → 4.9.0

package/package.json

@@ -3,7 +3,7 @@
   "author": "Yandex Data UI Team <data-ui@yandex-team.ru>",
   "description": "Make documentation using yfm-docs in Markdown and HTML formats",
   "license": "MIT",
-  "version": "4.7.0",
+  "version": "4.9.0",
   "repository": {
     "type": "git",
     "url": "git@github.com:diplodoc-platform/cli.git"
@@ -14,6 +14,7 @@
   "main": "build/index.js",
   "files": [
     "build",
+    "assets",
     "src"
   ],
   "scripts": {
@@ -21,20 +22,21 @@
     "start": "node build/index.js",
     "lint": "eslint \"src/**/*.{js,jsx,ts,tsx}\"",
     "lint:fix": "npm run lint -- --fix",
+    "typecheck": "tsc --noEmit",
     "prepublishOnly": "npm run lint && npm run build",
     "git:head": "git checkout master && git pull"
   },
   "engines": {
-    "node": ">=18.*",
-    "npm:": ">=9.*"
+    "node": ">=18.*"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.369.0",
-    "@diplodoc/client": "^1.2.0",
+    "@diplodoc/client": "^2.0.0",
+    "@diplodoc/latex-extension": "^1.0.3",
     "@diplodoc/markdown-translation": "^1.0.4",
     "@diplodoc/mermaid-extension": "^1.2.1",
     "@diplodoc/openapi-extension": "^1.4.10",
-    "@diplodoc/transform": "^4.7.3",
+    "@diplodoc/transform": "^4.8.1",
     "@octokit/core": "4.2.4",
     "@yandex-cloud/nodejs-sdk": "^2.2.2",
     "ajv": "^8.11.0",
@@ -92,20 +94,6 @@
       "git add"
     ]
   },
-  "pkg": {
-    "scripts": [
-      "build/index.js",
-      "build/linter.js",
-      "build/plugins/*.js",
-      "build/lint-rules/*.js",
-      "build/default-lint-config.js",
-      "build/head-content.js"
-    ],
-    "assets": [
-      "build/app.client.js",
-      "build/app.client.css"
-    ]
-  },
   "keywords": [
     "markdown",
     "yandex",
@@ -115,8 +103,5 @@
     "tool",
     "tools",
     "generator"
-  ],
-  "overrides": {
-    "markdown-it": "^13.0.2"
-  }
+  ]
 }

package/src/cmd/publish/index.ts

@@ -67,19 +67,31 @@ function builder<T>(argv: Argv<T>) {
         });
 }
 
+type Args = {
+    input: string;
+    endpoint: string;
+    region: string;
+    bucket: string;
+    prefix: string;
+    ignore: string[];
+    accessKeyId: string;
+    secretAccessKey: string;
+};
+
 /* eslint-disable-next-line @typescript-eslint/no-explicit-any */
-async function handler(args: Arguments<any>) {
+async function handler(args: Arguments<Args>) {
     ArgvService.init({
         ...args,
     });
 
-    const {input, endpoint, bucket, prefix} = ArgvService.getConfig();
+    const config = ArgvService.getConfig() as unknown as Args;
+    const {input, endpoint, bucket, prefix} = config;
 
     logger.info('', `Upload artifacts from ${input} to ${join(endpoint, bucket, prefix)}`);
 
     try {
-        await upload(ArgvService.getConfig());
-    } catch (error) {
+        await upload(config);
+    } catch (error: any) {
         logger.error('', error.message);
     }
 }

package/src/constants.ts

@@ -1,4 +1,4 @@
-import {dirname} from 'path';
+import {resolve} from 'path';
 const os = require('os');
 const notes = require('@diplodoc/transform/lib/plugins/notes');
 const anchors = require('@diplodoc/transform/lib/plugins/anchors');
@@ -19,16 +19,15 @@ const term = require('@diplodoc/transform/lib/plugins/term');
 const blockAnchor = require('@diplodoc/transform/lib/plugins/block-anchor');
 const changelog = require('@diplodoc/transform/lib/plugins/changelog');
 const mermaid = require('@diplodoc/mermaid-extension');
+const latex = require('@diplodoc/latex-extension');
 const openapi = require('@diplodoc/openapi-extension');
 
 includes.collect = require('@diplodoc/transform/lib/plugins/includes/collect');
 images.collect = require('@diplodoc/transform/lib/plugins/images/collect');
 changelog.collect = require('@diplodoc/transform/lib/plugins/changelog/collect');
 
-export const BUILD_FOLDER = 'build';
+export const ASSETS_FOLDER = resolve(__dirname, '../assets');
 export const BUNDLE_FOLDER = '_bundle';
-export const BUNDLE_JS_FILENAME = 'app.client.js';
-export const BUNDLE_CSS_FILENAME = 'app.client.css';
 export const TMP_INPUT_FOLDER = '.tmp_input';
 export const TMP_OUTPUT_FOLDER = '.tmp_output';
 export const MAIN_TIMER_ID = 'Build time';
@@ -68,8 +67,6 @@ export enum ResourceType {
     script = 'script',
 }
 
-export const BUILD_FOLDER_PATH = dirname(require.resolve('@diplodoc/client'));
-
 export const YFM_PLUGINS = [
     meta,
     deflist,
@@ -89,6 +86,7 @@ export const YFM_PLUGINS = [
     term,
     openapi.transform(),
     mermaid.transform(),
+    latex.transform(),
     changelog,
     blockAnchor,
 ];

package/src/models.ts

@@ -19,6 +19,7 @@ export type NestedContributorsForPathFunction = (
 ) => void;
 export type UserByLoginFunction = (login: string) => Promise<Contributor | null>;
 export type CollectionOfPluginsFunction = (output: string, options: PluginOptions) => string;
+export type GetModifiedTimeByPathFunction = (filepath: string) => number | undefined;
 
 interface YfmConfig {
     varsPreset: VarsPreset;

package/src/resolvers/md2html.ts

@@ -50,6 +50,10 @@ export async function resolveMd2HTML(options: ResolverOptions): Promise<ResolveM
     const transformFn: Function = FileTransformer[fileExtension];
     const {result} = transformFn(content, {path: inputPath});
 
+    if (result.html) {
+        result.html = result.html.replace(/\n+/gm, '');
+    }
+
     const updatedMetadata =
         metadata && metadata.isContributorsEnabled
             ? await getUpdatedMetadata(metadata, content, result?.meta)

package/src/services/contributors.ts

@@ -94,7 +94,10 @@ async function getContributorsForNestedFiles(
     return Object.assign({}, ...includesContributors);
 }
 
-function getRelativeIncludeFilePaths(fileData: FileData, includeContents: string[]): Set<string> {
+function getRelativeIncludeFilePaths(
+    fileData: Pick<FileData, 'tmpInputFilePath'>,
+    includeContents: string[],
+): Set<string> {
     const {tmpInputFilePath} = fileData;
     const relativeIncludeFilePaths: Set<string> = new Set();
 
@@ -115,4 +118,45 @@ function getRelativeIncludeFilePaths(fileData: FileData, includeContents: string
     return relativeIncludeFilePaths;
 }
 
-export {getFileContributorsMetadata, getFileContributorsString};
+async function getFileIncludes(
+    fileData: Pick<FileData, 'fileContent' | 'tmpInputFilePath' | 'inputFolderPathLength'>,
+) {
+    const {fileContent, tmpInputFilePath, inputFolderPathLength} = fileData;
+
+    const results = new Set<string>();
+
+    const includeContents = fileContent.match(REGEXP_INCLUDE_CONTENTS);
+    if (!includeContents || includeContents.length === 0) {
+        return [];
+    }
+    const relativeIncludeFilePaths: Set<string> = getRelativeIncludeFilePaths(
+        {tmpInputFilePath},
+        includeContents,
+    );
+    for (const relativeIncludeFilePath of relativeIncludeFilePaths.values()) {
+        const relativeFilePath = relativeIncludeFilePath.substring(inputFolderPathLength + 1);
+        if (results.has(relativeFilePath)) continue;
+        results.add(relativeFilePath);
+
+        let contentIncludeFile: string;
+        try {
+            contentIncludeFile = await readFile(relativeIncludeFilePath, 'utf8');
+        } catch (err) {
+            if (err.code === 'ENOENT') {
+                continue;
+            }
+            throw err;
+        }
+
+        const includedPaths = await getFileIncludes({
+            inputFolderPathLength,
+            fileContent: contentIncludeFile,
+            tmpInputFilePath: relativeIncludeFilePath,
+        });
+        includedPaths.forEach((path) => results.add(path));
+    }
+
+    return Array.from(results.values());
+}
+
+export {getFileContributorsMetadata, getFileContributorsString, getFileIncludes};

package/src/services/metadata.ts

@@ -7,12 +7,16 @@ import {
     updateAuthorMetadataStringByAuthorLogin,
     updateAuthorMetadataStringByFilePath,
 } from './authors';
-import {getFileContributorsMetadata, getFileContributorsString} from './contributors';
+import {
+    getFileContributorsMetadata,
+    getFileContributorsString,
+    getFileIncludes,
+} from './contributors';
 import {isObject} from './utils';
 import {сarriage} from '../utils';
 import {REGEXP_AUTHOR, metadataBorder} from '../constants';
 import {dirname, relative, resolve} from 'path';
-import {ArgvService} from './index';
+import {ArgvService, TocService} from './index';
 
 async function getContentWithUpdatedMetadata(
     fileContent: string,
@@ -105,6 +109,11 @@ async function getContentWithUpdatedDynamicMetadata(
             newMetadatas.push(contributorsMetaData);
         }
 
+        const mtimeMetadata = await getModifiedTimeMetadataString(options, fileContent);
+        if (mtimeMetadata) {
+            newMetadatas.push(mtimeMetadata);
+        }
+
         let authorMetadata = '';
         if (fileMetadata) {
             const matchAuthor = fileMetadata.match(REGEXP_AUTHOR);
@@ -188,6 +197,37 @@ async function getContributorsMetadataString(
     return undefined;
 }
 
+async function getModifiedTimeMetadataString(options: MetaDataOptions, fileContent: string) {
+    const {isContributorsEnabled, vcsConnector, fileData} = options;
+
+    const {tmpInputFilePath, inputFolderPathLength} = fileData;
+
+    const relativeFilePath = tmpInputFilePath.substring(inputFolderPathLength + 1);
+
+    if (!isContributorsEnabled || !vcsConnector) {
+        return undefined;
+    }
+
+    const includedFiles = await getFileIncludes({...fileData, fileContent});
+    includedFiles.push(relativeFilePath);
+
+    const tocCopyFileMap = TocService.getCopyFileMap();
+
+    const mtimeList = includedFiles
+        .map((path) => {
+            const mappedPath = tocCopyFileMap.get(path) || path;
+            return vcsConnector.getModifiedTimeByPath(mappedPath);
+        })
+        .filter((v) => typeof v === 'number') as number[];
+
+    if (mtimeList.length) {
+        const mtime = Math.max(...mtimeList);
+        return `updatedAt: ${new Date(mtime * 1000).toISOString()}`;
+    }
+
+    return undefined;
+}
+
 function getUpdatedMetadataString(newMetadatas: string[], defaultMetadata = ''): string {
     const newMetadata = newMetadatas.join(сarriage) + (newMetadatas.length ? сarriage : '');
     const preparedDefaultMetadata = defaultMetadata.trimRight();

package/src/services/tocs.ts

@@ -24,6 +24,7 @@ export interface TocServiceData {
 const storage: TocServiceData['storage'] = new Map();
 let navigationPaths: TocServiceData['navigationPaths'] = [];
 const includedTocPaths: TocServiceData['includedTocPaths'] = new Set();
+const tocFileCopyMap = new Map<string, string>();
 
 async function add(path: string) {
     const {
@@ -207,6 +208,10 @@ function _copyTocDir(tocPath: string, destDir: string) {
         } else {
             shell.cp(from, to);
         }
+
+        const relFrom = relative(inputFolderPath, from);
+        const relTo = relative(inputFolderPath, to);
+        tocFileCopyMap.set(relTo, relFrom);
     });
 }
 
@@ -396,6 +401,10 @@ function setNavigationPaths(paths: TocServiceData['navigationPaths']) {
     navigationPaths = paths;
 }
 
+function getCopyFileMap() {
+    return tocFileCopyMap;
+}
+
 export default {
     add,
     getForPath,
@@ -403,4 +412,5 @@ export default {
     getTocDir,
     getIncludedTocPaths,
     setNavigationPaths,
+    getCopyFileMap,
 };

package/src/steps/processAssets.ts

@@ -1,10 +1,11 @@
 import walkSync from 'walk-sync';
 import shell from 'shelljs';
 
-import client from '../../scripts/client';
 import {ArgvService} from '../services';
 import {copyFiles} from '../utils';
 
+import {ASSETS_FOLDER} from '../constants';
+
 /**
  * Processes assets files (everything except .yaml and .md files)
  * @param {string} outputBundlePath
@@ -23,8 +24,5 @@ export function processAssets(outputBundlePath: string) {
 
     /* Copy js bundle to user' output folder */
     shell.mkdir('-p', outputBundlePath);
-
-    for (const path of Object.values(client.dst)) {
-        shell.cp(path, outputBundlePath);
-    }
+    shell.cp(ASSETS_FOLDER + '/*', outputBundlePath);
 }

package/src/utils/markup.ts

@@ -1,11 +1,15 @@
+import {join} from 'path';
 import {platform} from 'process';
 
 import {CUSTOM_STYLE, Platforms, ResourceType} from '../constants';
 import {Resources, SinglePageResult} from '../models';
 import {ArgvService, PluginService} from '../services';
 import {preprocessPageHtmlForSinglePage} from './singlePage';
-import {DocInnerProps, DocPageData, render} from '@diplodoc/client';
-import client from '../../scripts/client';
+
+import {DocInnerProps, DocPageData, render} from '@diplodoc/client/ssr';
+import manifest from '@diplodoc/client/manifest';
+
+const dst = (bundlePath: string) => (target: string) => join(bundlePath, target);
 
 export interface TitleMeta {
     title?: string;
@@ -44,7 +48,10 @@ export function generateStaticMarkup(
                         height: 100vh;
                     }
                 </style>
-                <link type="text/css" rel="stylesheet" href="${client.bundle.css(pathToBundle)}" />
+                ${manifest.css
+                    .map(dst(pathToBundle))
+                    .map((src: string) => `<link type="text/css" rel="stylesheet" href="${src}" />`)
+                    .join('\n')}
                 ${PluginService.getHeadContent()}
                 ${resources}
             </head>
@@ -54,9 +61,13 @@ export function generateStaticMarkup(
                    window.STATIC_CONTENT = ${staticContent}
                    window.__DATA__ = ${JSON.stringify(props)};
                 </script>
-                <script type="application/javascript" src="${client.bundle.js(
-                    pathToBundle,
-                )}"></script>
+                ${manifest.js
+                    .map(dst(pathToBundle))
+                    .map(
+                        (src: string) =>
+                            `<script type="application/javascript" src="${src}"></script>`,
+                    )
+                    .join('\n')}
             </body>
         </html>
     `;
@@ -126,7 +137,7 @@ export function joinSinglePageResults(
     root: string,
     tocDir: string,
 ): string {
-    const delimeter = `${сarriage}${сarriage}<hr class="yfm-page__delimeter">${сarriage}${сarriage}`;
+    const delimeter = `<hr class="yfm-page__delimeter">`;
     return singlePageResults
         .filter(({content}) => content)
         .map(({content, path, title}) =>

package/src/utils/path.ts

@@ -14,11 +14,3 @@ export function convertBackSlashToSlash(path: string): string {
 
     return path;
 }
-
-export function convertSlashToWindowsBackSlashes(path: string): string {
-    if (process.platform === Platforms.WINDOWS) {
-        return path.replace(/\//g, '\\\\');
-    }
-
-    return path;
-}

package/src/utils/singlePage.ts

@@ -3,7 +3,7 @@ import {parse} from 'node-html-parser';
 import {relative, resolve, sep} from 'path';
 import {resolveRelativePath} from '@diplodoc/transform/lib/utilsFS';
 import url from 'url';
-import _ from 'lodash';
+import escape from 'lodash/escapeRegExp';
 
 import {isExternalHref} from './url';
 
@@ -22,6 +22,11 @@ interface PreprocessSinglePageOptions {
 
 const HEADERS_SELECTOR = 'h1, h2, h3, h4, h5, h6';
 
+function toUrl(path: string) {
+    // replace windows backslashes
+    return path.replace(new RegExp(escape(sep), 'g'), '/');
+}
+
 function getNewNode(options: ModifyNode): HTMLElement | null {
     const {rawTagName, innerHTML, attrEntries} = options;
 
@@ -92,7 +97,7 @@ export function replaceLinks(rootEl: HTMLElement, options: PreprocessSinglePageO
             }
         }
 
-        node.setAttribute('href', preparedHref);
+        node.setAttribute('href', toUrl(preparedHref));
     });
 }
 
@@ -110,7 +115,7 @@ export function replaceImages(rootEl: HTMLElement, options: PreprocessSinglePage
         const linkFullPath = resolveRelativePath(resolvedPath, href);
         const preparedHref = relative(tocDir, linkFullPath);
 
-        node.setAttribute('src', preparedHref);
+        node.setAttribute('src', toUrl(preparedHref));
     });
 }
 
@@ -129,7 +134,7 @@ function prepareAnchorAttrs(node: HTMLElement, pageId: string) {
     for (const [name, value] of Object.entries(node.attributes)) {
         const preparedValue = prepareAnchorAttr(name, value, pageId);
 
-        node.setAttribute(name, preparedValue);
+        node.setAttribute(name, toUrl(preparedValue));
     }
 }
 
@@ -197,7 +202,7 @@ export function getSinglePageAnchorId(args: {
     resultAnchor = resultAnchor
         .replace(root, '')
         .replace(/\.(md|ya?ml|html)$/i, '')
-        .replace(new RegExp(_.escapeRegExp(sep), 'gi'), '_');
+        .replace(new RegExp(escape(sep), 'gi'), '_');
 
     if (hash) {
         resultAnchor = resultAnchor + '_' + hash.slice(1);

package/src/vcs-connector/connector-models.ts

@@ -2,6 +2,7 @@ import {
     Contributors,
     ContributorsByPathFunction,
     ExternalAuthorByPathFunction,
+    GetModifiedTimeByPathFunction,
     NestedContributorsForPathFunction,
     UserByLoginFunction,
 } from '../models';
@@ -31,6 +32,7 @@ export interface VCSConnector {
     addNestedContributorsForPath: NestedContributorsForPathFunction;
     getContributorsByPath: ContributorsByPathFunction;
     getUserByLogin: UserByLoginFunction;
+    getModifiedTimeByPath: GetModifiedTimeByPathFunction;
 }
 
 export interface VCSConnectorConfig {

package/src/vcs-connector/github.ts

@@ -33,9 +33,10 @@ const authorByPath: Map<string, Contributor | null> = new Map();
 const contributorsByPath: Map<string, FileContributors> = new Map();
 const contributorsData: Map<string, Contributor | null> = new Map();
 const loginUserMap: Map<string, Contributor | null> = new Map();
+const pathMTime = new Map<string, number>();
 
 async function getGitHubVCSConnector(): Promise<VCSConnector | undefined> {
-    const {contributors} = ArgvService.getConfig();
+    const {contributors, rootInput} = ArgvService.getConfig();
 
     const httpClientByToken = getHttpClientByToken();
     if (!httpClientByToken) {
@@ -49,6 +50,7 @@ async function getGitHubVCSConnector(): Promise<VCSConnector | undefined> {
         authorByPath.get(path) ?? null;
 
     if (contributors) {
+        await getFilesMTime(rootInput, pathMTime);
         await getAllContributorsTocFiles(httpClientByToken);
         addNestedContributorsForPath = (path: string, nestedContributors: Contributors) =>
             addNestedContributorsForPathFunction(path, nestedContributors);
@@ -60,6 +62,7 @@ async function getGitHubVCSConnector(): Promise<VCSConnector | undefined> {
         addNestedContributorsForPath,
         getContributorsByPath,
         getUserByLogin: (login: string) => getUserByLogin(httpClientByToken, login),
+        getModifiedTimeByPath: (filename: string) => pathMTime.get(filename),
     };
 }
 
@@ -364,4 +367,42 @@ function shouldAuthorBeIgnored({email, name}: ShouldAuthorBeIgnoredArgs) {
     return false;
 }
 
+async function getFilesMTime(repoDir: string, pathMTime: Map<string, number>) {
+    const timeFiles = await simpleGit({
+        baseDir: repoDir,
+    }).raw(
+        'log',
+        '--reverse',
+        '--before=now',
+        '--diff-filter=ADMR',
+        '--pretty=format:%ct',
+        '--name-status',
+    );
+
+    const parts = timeFiles.split(/\n\n/);
+    parts.forEach((part) => {
+        const lines = part.trim().split(/\n/);
+        const committerDate = lines.shift();
+        const unixtime = Number(committerDate);
+        lines.forEach((line) => {
+            const [status, from, to] = line.split(/\t/);
+            switch (status[0]) {
+                case 'R': {
+                    pathMTime.delete(from);
+                    pathMTime.set(to, unixtime);
+                    break;
+                }
+                case 'D': {
+                    pathMTime.delete(from);
+                    break;
+                }
+                default: {
+                    pathMTime.set(from, unixtime);
+                }
+            }
+        });
+    });
+    return pathMTime;
+}
+
 export default getGitHubVCSConnector;