@diplodoc/cli-tests 0.0.0-rc-fix-translate-openapi-includer-202507171205 → 0.0.0-rc-resolve-codeblock-202509151421

package/e2e/bundles.spec.ts

@@ -0,0 +1,15 @@
+import {describe, it} from 'vitest';
+import {TestAdapter, compareDirectories, getTestPaths} from '../fixtures';
+
+describe('Check bundles', () => {
+    it('bundles list is correct', async () => {
+        const {inputPath, outputPath} = getTestPaths('mocks/bundles');
+
+        await TestAdapter.testBuildPass(inputPath, outputPath, {
+            md2md: false,
+            md2html: true,
+            args: '-j2',
+        });
+        await compareDirectories(outputPath, false, true);
+    });
+});

package/e2e/errors.spec.ts

@@ -13,12 +13,12 @@ function test(path: string, expect: Function) {
         const {inputPath, outputPath} = getTestPaths(path);
 
         const md = await TestAdapter.build.run(inputPath, outputPath, ['-j2', '-f', 'md']);
-        const html = await TestAdapter.build.run(outputPath, outputPath + '-html', ['-j2', '-f', 'html']);
-
+        const html = await TestAdapter.build.run(inputPath, outputPath + '-html', ['-j2', '-f', 'html']);
         return expect({md, html});
     });
 }
 
+
 describe('Errors', () => {
     test('mocks/errors/unreachable-link', ({html}: TestResult) => {
         expectErrors(html, [
@@ -26,6 +26,29 @@ describe('Errors', () => {
             'ERR index.md: 2: YFM003 / unreachable-link Link is unreachable [Context: "[Unreachable link: "missed.html"][missed file](./missed.md)"]'
         ]);
     });
+
+    it('translate extract with filtered links', async () => {
+        const {inputPath, outputPath} = getTestPaths('mocks/errors/extract-filtered-link');
+
+        const result = await TestAdapter.extract.run(inputPath, outputPath, ['--source', 'ru-RU', '--target', 'es-ES', '--filter']);
+        
+        expect(result.code).toEqual(1);
+        
+        expect(result.errors).toEqual([
+            "ERR File index.md contains link to filtered.md, which was filtered from toc.yaml or it's not been included initially",
+            "ERR File index.md contains link to filtered2.md, which was filtered from toc.yaml or it's not been included initially",
+            "ERR File index.md contains link to filtered2.md, which was filtered from toc.yaml or it's not been included initially",
+            "ERR File index.md contains link to filtered3.md, which was filtered from toc.yaml or it's not been included initially",
+        ])
+    });
+});
+
+describe('Warnings', () => {
+    test('mocks/warning/unreachable-autotitle', ({html}: TestResult) => {
+        expectWarnings(html, [
+            'WARN index.md: 1: YFM010 / unreachable-autotitle-anchor Auto title anchor is unreachable [Context: "[Unreachable autotitle anchor: "link.html#unknown_yfm010"][{#T}](./link.md#unknown_yfm010)"]',
+        ]);
+    });
 });
 
 function expectErrors(report: Report, errors: string[]) {
@@ -35,3 +58,11 @@ function expectErrors(report: Report, errors: string[]) {
         expect(report.errors).toContain(error);
     }
 }
+
+function expectWarnings(report: Report, warnings: string[]) {
+    expect(report.warns.length).toEqual(warnings.length);
+
+    for (const warn of warnings) {
+        expect(report.warns).toContain(warn);
+    }
+}

package/e2e/generate-map.spec.ts

@@ -1,6 +1,7 @@
 import {describe, expect, test} from 'vitest';
 import {TestAdapter, bundleless, getFileContent, getTestPaths} from '../fixtures';
-import {join} from 'path';
+import {join} from 'node:path';
+import {readFile} from 'node:fs/promises';
 
 const generateMapTestTemplate = (
     testTitle: string,
@@ -11,11 +12,17 @@ const generateMapTestTemplate = (
     test(testTitle, async () => {
         const {inputPath, outputPath} = getTestPaths(testRootPath);
 
-        await TestAdapter.testBuildPass(inputPath, outputPath, {md2md, md2html, args: '--add-map-file'});
+        await TestAdapter.testBuildPass(inputPath, outputPath, {
+            md2md,
+            md2html,
+            args: '--add-map-file --build-manifest',
+        });
 
         const content = getFileContent(join(outputPath, 'files.json'));
+        const manifestContent = await readFile(join(outputPath, 'yfm-build-manifest.json'), 'utf-8');
 
         expect(bundleless(content)).toMatchSnapshot();
+        expect(JSON.parse(manifestContent)).toMatchSnapshot();
     });
 };
 
@@ -28,7 +35,6 @@ describe('Generate map for', () => {
     generateMapTestTemplate(
         'project with single language and toc include - only md2html',
         'mocks/generate-map/test1',
-        false,
     );
 
     generateMapTestTemplate('project with multiple language', 'mocks/generate-map/test2');

package/e2e/preprocess.test.ts

@@ -0,0 +1,33 @@
+import {describe, test} from 'vitest';
+import {TestAdapter, compareDirectories, getTestPaths} from '../fixtures';
+
+const generateFilesYamlTestTemplate = (
+    testTitle: string,
+    testRootPath: string,
+    args: string[] = [],
+) => {
+    test(testTitle, async () => {
+        const {inputPath, outputPath} = getTestPaths(testRootPath);
+        await TestAdapter.testBuildPass(inputPath, outputPath, {
+            md2md:true,
+            md2html: false,
+            args: args.join(' '),
+        });
+        await TestAdapter.testBuildPass(outputPath, outputPath + '-html', {
+            md2md: false,
+            md2html: true,
+            args: args.join(' '),
+        });
+        await compareDirectories(outputPath);
+    });
+};
+
+describe('Preprocess', () => {
+    generateFilesYamlTestTemplate('HashIncludes=true,Autotitles=false', 'mocks/preprocess', [
+        '--no-merge-autotitles'
+    ]);
+
+    generateFilesYamlTestTemplate('HashIncludes=true,Autotitles=true', 'mocks/preprocess');
+
+    // generateFilesYamlTestTemplate('Nested toc restricted access', 'mocks/preprocess/test3');
+});

package/e2e/redirects-validation.spec.ts

@@ -0,0 +1,61 @@
+import {describe, expect, it} from 'vitest';
+import {TestAdapter, getTestPaths} from '../fixtures';
+
+describe('Redirects validation', () => {
+    it('should emit an error on an unparseable redirects.yaml', async () => {
+        const {inputPath, outputPath} = getTestPaths('mocks/redirects-validation/unparseable');
+
+        const report = await TestAdapter.build.run(inputPath, outputPath, ['-f', 'md']);
+
+        expect(report.code).toBe(1);
+        expect(report.errors).toContainEqual(expect.stringMatching(/redirects.yaml parsing error/));
+    });
+
+    it.skip('should emit a warning when supplied with a `redirects.yaml` that mentions file extensions', async () => {
+        const {inputPath, outputPath} = getTestPaths(
+            'mocks/redirects-validation/extensions-deprecation',
+        );
+
+        const report = await TestAdapter.build.run(inputPath, outputPath, ['-f', 'md']);
+
+        expect(report.code).toBe(0);
+        expect(report.warns).toContainEqual(
+            expect.stringMatching(/Redirects with explicit extensions are deprecated./),
+        );
+    });
+
+    it('should emit an error when an invalid regular expression pattern is encountered', async () => {
+        const {inputPath, outputPath} = getTestPaths('mocks/redirects-validation/invalid-regex');
+
+        const report = await TestAdapter.build.run(inputPath, outputPath, ['-f', 'md']);
+
+        expect(report.code).toBe(1);
+        expect(report.errors).toContainEqual(
+            expect.stringMatching(
+                /Redirects configuration results in a non-valid regular expression/,
+            ),
+        );
+    });
+
+    it('should emit an error when a redirect is malformed', async () => {
+        const {inputPath, outputPath} = getTestPaths('mocks/redirects-validation/malformed-redirect');
+
+        const report = await TestAdapter.build.run(inputPath, outputPath, ['-f', 'md']);
+
+        expect(report.code).toBe(1);
+        expect(report.errors).toContainEqual(
+            expect.stringMatching(/One of the two parameters is missing/),
+        );
+    });
+
+    it('should emit an error when a redirect leads to the same path', async () => {
+        const {inputPath, outputPath} = getTestPaths('mocks/redirects-validation/same-path');
+
+        const report = await TestAdapter.build.run(inputPath, outputPath, ['-f', 'md']);
+
+        expect(report.code).toBe(1);
+        expect(report.errors).toContainEqual(
+            expect.stringMatching(/Parameters must be different/),
+        );
+    });
+});

package/e2e/search.test.ts

@@ -8,7 +8,7 @@ describe('Local search', () => {
         await TestAdapter.testBuildPass(inputPath, outputPath, {
             md2md: false,
             md2html: true,
-            args: '-j2 --search',
+            args: '-j2 --search --interface-toc',
         });
         await compareDirectories(outputPath);
     });

package/e2e/single-page.spec.ts

@@ -0,0 +1,22 @@
+import {describe, test} from 'vitest';
+import {TestAdapter, compareDirectories, getTestPaths} from '../fixtures';
+
+const generateMapTestSinglePageTemplate = (
+    testTitle: string,
+    testRootPath: string,
+    {md2md = true, md2html = true, args = '--single-page'},
+) => {
+    test(testTitle, async () => {
+        const {inputPath, outputPath} = getTestPaths(testRootPath);
+        await TestAdapter.testBuildPass(inputPath, outputPath, {md2md, md2html, args});
+        await compareDirectories(outputPath);
+    });
+};
+
+describe('Single page mode', () => {
+    generateMapTestSinglePageTemplate(
+        'simple md2html single page with lang dirs',
+        'mocks/single-page',
+        {md2md: false},
+    );
+});

package/e2e/skip-html-extension.spec.ts

@@ -0,0 +1,15 @@
+import {describe, it} from 'vitest';
+import {TestAdapter, compareDirectories, getTestPaths} from '../fixtures';
+
+describe('Skip html extension', () => {
+    it('transforms links correctly', async () => {
+        const {inputPath, outputPath} = getTestPaths('mocks/skip-html-extension');
+
+        await TestAdapter.testBuildPass(inputPath, outputPath, {
+            md2md: false,
+            md2html: true,
+            args: '-j2 --skip-html-extension',
+        });
+        await compareDirectories(outputPath);
+    });
+});

package/e2e/translation.spec.ts

@@ -7,7 +7,7 @@ const generateMapTestTemplate = (
     args: TranslateRunArgs,
     ignoreFileContent = true,
 ) => {
-    test.skip(testTitle, async () => {
+    test(testTitle, async () => {
         const {inputPath, outputPath} = getTestPaths(testRootPath);
 
         await TestAdapter.testTranslatePass(inputPath, outputPath, args);
@@ -56,10 +56,17 @@ describe('Translate command', () => {
         target: 'es-ES',
     });
 
+    generateMapTestTemplate('do not filter files on extract', 'mocks/translation/dir-files', {
+        subcommand: 'extract',
+        source: 'ru-RU',
+        target: 'es-ES',
+    });
+
     generateMapTestTemplate('filter files on extract', 'mocks/translation/dir-files', {
         subcommand: 'extract',
         source: 'ru-RU',
         target: 'es-ES',
+        additionalArgs: '--filter'
     });
 
     generateMapTestTemplate(
@@ -69,7 +76,19 @@ describe('Translate command', () => {
             subcommand: 'extract',
             source: 'ru-RU',
             target: 'es-ES',
-            additionalArgs: '--exclude ru/_no-translate/*.md',
+            additionalArgs: '--exclude ru/to-be-excluded.md --filter',
+        },
+    );
+
+    const vars = {skip: 'prod'}
+    generateMapTestTemplate(
+        'filter files on extract with extra vars option',
+        'mocks/translation/dir-files',
+        {
+            subcommand: 'extract',
+            source: 'ru-RU',
+            target: 'es-ES',
+            additionalArgs: `--vars ${JSON.stringify(vars)} --filter`,
         },
     );
 

package/fixtures/cliAdapter.ts

@@ -34,11 +34,34 @@ class Build {
     }
 }
 
+class Extract {
+    private readonly runner: Runner;
+
+    constructor(runner: Runner) {
+        this.runner = runner;
+    }
+
+    run(input: string, output: string, args: string[]) {
+        return this.runner.runYfmDocs([
+            'translate',
+            'extract',
+            '--input',
+            input,
+            '--output',
+            output,
+            '--quiet',
+            ...args,
+        ]);
+    }
+}
+
 export class CliTestAdapter {
     readonly runner: Runner = createRunner();
 
     readonly build = new Build(this.runner);
 
+    readonly extract = new Extract(this.runner);
+
     async testBuildPass(
         inputPath: string,
         outputPath: string,

package/fixtures/globals.d.ts

@@ -0,0 +1,4 @@
+declare module "@diplodoc/cli/manifest" {
+    const manifest: Record<string, Record<string, string[]>>;
+    export = manifest;
+}

package/fixtures/runners/binary.ts

@@ -1,5 +1,6 @@
 import {Runner} from './types';
 import {execa} from 'execa';
+import strip from 'strip-ansi';
 
 export class BinaryRunner implements Runner {
     private readonly binaryPath: string;
@@ -29,7 +30,7 @@ export class BinaryRunner implements Runner {
 
 function fillLog(filter: RegExp, source: string) {
     return source.split('\n')
-        .filter((line) => line.match(filter))
-        .map((line) => line.trim())
-        .filter(Boolean);
+        .map((line) => strip(line).trim())
+        .filter(Boolean)
+        .filter((line) => line.match(filter));
 }

package/fixtures/runners/types.ts

@@ -2,9 +2,8 @@ export type Report = {
     code: number;
     warns: string[];
     errors: string[];
-}
+};
 
 export interface Runner {
     runYfmDocs(argv: string[]): Promise<Report>;
 }
-

package/fixtures/utils/file.ts

@@ -1,18 +1,42 @@
 import {readFileSync} from 'node:fs';
+import {rm} from 'node:fs/promises';
 import {join, resolve} from 'node:path';
 import {glob} from 'glob';
 import {bundleless, hashless, platformless} from './test';
 import {expect} from 'vitest';
-import {$} from 'execa';
+
+const SYSTEM_DIRS = ['_bundle/', '_search/'];
 
 export function getFileContent(filePath: string) {
-    return bundleless(platformless(readFileSync(filePath, 'utf8')));
+    return platformless(bundleless(readFileSync(filePath, 'utf8')));
 }
 
-const uselessFile = (file: string) =>
-    !['_bundle/', '_assets/', '_search/'].some((part) => file.includes(part));
+const uselessFile = (file: string, dirs: string[]) =>
+    !dirs.some((part) => file.includes(part));
+
+export function stripSystemLinks(content: string) {
+    const dirPattern = SYSTEM_DIRS.map(d => d.replace('/', '\\/')).join('|');
+
+    content = content.replace(
+        new RegExp(`<script[^>]+src="(?:${dirPattern})[^"]*"[^>]*></script>`, 'g'),
+        ''
+    );
+
+    content = content.replace(
+        new RegExp(`<link[^>]+href="(?:${dirPattern})[^"]*"[^>]*\\/?>`, 'g'),
+        ''
+    );
+
+    content = content.replace(/^[ \t]*\r?\n/gm, '');
+
+    return content;
+}
 
-export async function compareDirectories(outputPath: string, ignoreFileContent = false) {
+export async function compareDirectories(
+    outputPath: string,
+    ignoreFileContent = false,
+    checkBundle = false,
+) {
     const filesFromOutput = (
         await glob(`**/*`, {
             cwd: outputPath,
@@ -21,13 +45,32 @@ export async function compareDirectories(outputPath: string, ignoreFileContent =
             nodir: true,
             posix: true,
         })
-    ).sort();
+    ).map(bundleless).sort();
 
-    expect(hashless(bundleless(JSON.stringify(filesFromOutput, null, 2)))).toMatchSnapshot('filelist');
+    let filesForSnapshot;
+
+    if (checkBundle) {
+        filesForSnapshot = filesFromOutput;
+    } else {
+        filesForSnapshot = filesFromOutput.filter(file => uselessFile(file, SYSTEM_DIRS));
+    }
+
+    // Here we sort the order of the included files after all processing
+    // This is necessary for better test stability
+    // We do not care in what order these files were received and processed
+    // We sort only the final list and put it in the snapshot
+    filesForSnapshot = filesForSnapshot.map(hashless).sort(); 
+
+    expect(JSON.stringify(filesForSnapshot, null, 2)).toMatchSnapshot('filelist');
 
     if (!ignoreFileContent) {
-        filesFromOutput.filter(uselessFile).forEach((filePath) => {
-            const content = getFileContent(resolve(outputPath, filePath));
+        filesFromOutput.filter(file => uselessFile(file, ['_assets/', ...SYSTEM_DIRS])).forEach((filePath) => {
+            let content = getFileContent(resolve(outputPath, filePath));
+
+            if (!checkBundle && filePath.endsWith('.html')) {
+                content = stripSystemLinks(content);
+            }
+
             expect(content).toMatchSnapshot();
         });
     }
@@ -46,5 +89,5 @@ export function getTestPaths(testRootPath: string): TestPaths {
 }
 
 export function cleanupDirectory(path: string) {
-    return $`rm -rf ${path}`;
+    return rm(path, {recursive: true, force: true});
 }

package/fixtures/utils/test.ts

@@ -1,11 +1,25 @@
-import {readFileSync} from 'node:fs';
+import assets from '@diplodoc/cli/manifest';
 
 export function platformless(text: string): string {
+    let index = 1;
+
     return hashless(text)
         .replace(/\r\n/g, '\n')
+        // Fix for XML equiv-text attributes in Windows - handle various patterns
+        .replace(/equiv-text="[\r\n]+
"/g, 'equiv-text="
"')
+        .replace(/equiv-text="[\r\n]+
"/g, 'equiv-text="
"')
+        // Also normalize any other attributes that might have line ending issues
+        .replace(/(ctype|id)="[\r\n]+(.*?)"/g, '$1="$2"')
         .replace(/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/g, 'UUID')
-        .replace(/(content"?[:=]{1}[" ]{1}Diplodoc.*? )v\d+\.\d+\.\d+(?:-[\w-]+)?/g, `$1vDIPLODOC-VERSION`)
-        .replace(/(\\(?![/"'])){1,2}/g, '/');
+        .replace(
+            /(content"?[:=]{1}[" ]{1}Diplodoc.*? )v\d+\.\d+\.\d+(?:-[\w-]+)?/g,
+            `$1vDIPLODOC-VERSION`,
+        )
+        .replace(/(\\(?![/"'])){1,2}/g, '/')
+        .replace(
+            /id=\\"inline-code-id-[a-zA-Z0-9]{8}\\"/g,
+            () => `id="inline-code-id-${index++}"`
+        )
 }
 
 export function hashless(text: string): string {
@@ -15,8 +29,6 @@ export function hashless(text: string): string {
 }
 
 export function bundleless(text: string): string {
-    const assets = require('@diplodoc/client/manifest') as Record<string, Record<string, string[]>>;
-
     for (const [entryKey, entry] of Object.entries(assets)) {
         for (const [typeKey, type] of Object.entries(entry)) {
             for (let index = 0; index < type.length; index++) {
@@ -32,6 +44,3 @@ export function bundleless(text: string): string {
     return text;
 }
 
-export function getNormalizedContent(filePath: string): string {
-    return bundleless(platformless(readFileSync(filePath, 'utf8')));
-}

package/mocks/bundles/input/.yfm

@@ -0,0 +1,13 @@
+allowHTML: true
+
+meta:
+  rootPath: http://127.0.0.1:5000/
+
+interface:
+  toc-header: false
+  favicon-src: https://storage.yandexcloud.net/diplodoc-www-assets/favicon/favicon.ico
+
+search:
+  provider: local
+  tolerance: 2
+  confidense: phrased

package/mocks/bundles/input/index.md

@@ -0,0 +1,12 @@
+---
+  availableLangs:
+    - en
+    - ru
+---
+
+# Header
+
+Content
+
+[Link 1](page1.md)
+[Link 2](page2.md)

package/mocks/bundles/input/page1.md

@@ -0,0 +1,3 @@
+# Page 1
+
+[Link](page2.md#hash)

package/mocks/bundles/input/page2.md

@@ -0,0 +1,5 @@
+# Page 2
+
+## hash
+
+[External link](https://example.com)

package/mocks/bundles/input/toc.yaml

@@ -0,0 +1,9 @@
+title: Skip html extension
+href: index.md
+
+items:
+  - name: Title 1
+    href: page1.md
+  - name: Title 2
+    href: page2.md
+    

package/mocks/docs-viewer-interface/input/.yfm

@@ -0,0 +1,6 @@
+langs: ['en','ru']
+
+interface:
+    toc: false
+    feedback: false
+    search: false

package/mocks/docs-viewer-interface/input/index.md

@@ -0,0 +1,3 @@
+# Hello World!
+
+some text inside

package/mocks/docs-viewer-interface/input/toc.yaml

@@ -0,0 +1,5 @@
+title: Test123
+href: index.md
+items:
+  - name: Главная
+    href: index.md

package/mocks/errors/extract-filtered-link/input/filtered.md

@@ -0,0 +1 @@
+File exists but not attached to toc

package/mocks/errors/extract-filtered-link/input/index.md

@@ -0,0 +1,7 @@
+[filtered file](./filtered.md)
+
+{% if test == "dev" %}[will be deleted on translate-extract](./index.md){% endif %}
+
+[second missed link](./filtered2.md)
+[second missed link](./filtered2.md)
+[second missed link](./filtered3.md)

package/mocks/errors/extract-filtered-link/input/toc.yaml

@@ -0,0 +1 @@
+href: index.md

package/mocks/load-custom-resources/md2html-with-resources/input/page.md

@@ -1,7 +1,14 @@
 ---
+title: Page Title
+description: Some test description
+
+interface:
+  toc: false
+  favicon-src: /favicon.ico
+
 metadata:
   - name: yfm
-    content: builder
+    content: builder in page
 ---
 
-Lorem
+Lorem

package/mocks/load-custom-resources/md2md-with-resources/input/page.md

@@ -1,7 +1,14 @@
 ---
+title: Page Title
+description: Some test description
+
+interface:
+  toc: false
+  favicon-src: /favicon.ico
+
 metadata:
   - name: yfm
-    content: builder
+    content: builder in page
 ---
 
-Lorem
+Lorem

package/mocks/load-custom-resources/single-page-with-resources/input/page.md

@@ -1,7 +1,14 @@
 ---
+title: Page Title
+description: Some test description
+
+interface:
+  toc: false
+  favicon-src: /favicon.ico
+
 metadata:
   - name: yfm
-    content: builder
+    content: builder in page
 ---
 
-Lorem
+Lorem

package/mocks/metadata/md2html-with-metadata/input/page.md

@@ -1,7 +1,14 @@
 ---
+title: Page Title
+description: Some test description
+
+interface:
+  toc: false
+  favicon-src: /favicon.ico
+
 metadata:
   - name: yfm
-    value: builder in page
+    content: builder in page
 ---
 
 Lorem