doc-freshness-checker 2.0.23 → 2.2.0

package/dist/runner.test.js

@@ -4,11 +4,16 @@ import path from 'path';
 import { glob } from 'glob';
 import { run, runWithConfig } from './runner.js';
 import { BUILT_IN_RULE_TYPES } from './config/defaults.js';
+import { GraphBuilder } from './graph/graphBuilder.js';
 import { VectorSearch } from './semantic/vectorSearch.js';
+import { SourceIndex } from './source/sourceIndex.js';
+import { DocumentParser } from './parsers/documentParser.js';
+import { ManifestInventory } from './manifests/manifestInventory.js';
 import { ValidationEngine } from './validators/validationEngine.js';
 import { IncrementalChecker } from './utils/incremental.js';
 import { FileValidator } from './validators/fileValidator.js';
 import { CacheManager } from './cache/cacheManager.js';
+import { FreshnessScorer } from './scoring/freshnessScorer.js';
 import { withOutputFile } from './test-utils/tempFiles.js';
 import { captureConsoleLog, captureConsoleWarn } from './test-utils/console.js';
 vi.mock('fastembed', () => ({
@@ -139,15 +144,58 @@ describe('runner', () => {
         readSpy.mockRestore();
         await fs.promises.rm(path.join(process.cwd(), cacheDir), { recursive: true, force: true });
     });
-    it('registers custom extractors and validators', async () => {
-        const extract = vi.fn().mockReturnValue([]);
-        const validateBatch = vi.fn().mockResolvedValue([]);
-        await run({
-            ...baseConfig,
-            customExtractors: [{ extract, supportsFormat: () => true }],
-            customValidators: { custom: { validateBatch } },
+    it('round-trips a two-method custom extractor through an overriding custom validator', async () => {
+        await fs.promises.mkdir(cacheRoot, { recursive: true });
+        await withOutputFile(cacheRoot, 'custom-extractor.md', async (filePath) => {
+            const content = '# Custom\n\nprose';
+            await fs.promises.writeFile(filePath, content, 'utf-8');
+            vi.mocked(glob).mockResolvedValueOnce([filePath]);
+            const reference = {
+                type: 'file-path',
+                value: 'prose',
+                lineNumber: 3,
+                raw: 'prose',
+                sourceFile: 'custom-extractor.md',
+            };
+            const customExtractor = {
+                supportsFormat: vi.fn((format) => format === 'markdown'),
+                extract: vi.fn(() => [reference]),
+            };
+            const invalidResult = {
+                reference,
+                valid: false,
+                severity: 'error',
+                message: 'Custom reference is stale',
+            };
+            const customValidator = {
+                validateBatch: vi.fn(async () => [invalidResult]),
+            };
+            const config = {
+                ...baseConfig,
+                rootDir: cacheRoot,
+                include: ['**/*.md'],
+                rules: { ...baseConfig.rules, 'file-path': { enabled: true } },
+                customExtractors: [customExtractor],
+                customValidators: { 'file-path': customValidator },
+            };
+            const results = await run(config);
+            const document = {
+                path: 'custom-extractor.md',
+                absolutePath: filePath,
+                content,
+                format: 'markdown',
+                lines: ['# Custom', '', 'prose'],
+                references: [reference],
+            };
+            expect(vi.mocked(glob).mock.calls).toEqual([[['**/*.md'], { ignore: [], cwd: cacheRoot, absolute: true }]]);
+            expect(customExtractor.supportsFormat.mock.calls).toEqual([['markdown']]);
+            expect(customExtractor.extract.mock.calls).toEqual([[document]]);
+            expect(customValidator.validateBatch.mock.calls).toEqual([[[reference], document, config]]);
+            expect(results).toEqual({
+                documents: [{ path: 'custom-extractor.md', issues: [invalidResult] }],
+                summary: { total: 1, valid: 0, errors: 1, warnings: 0, info: 0, skipped: 0 },
+            });
         });
-        expect(true).toBe(true);
     });
     it.each(['supportsFormat', 'extract'])('propagates custom extractor %s failures', async (hook) => {
         vi.mocked(glob).mockResolvedValueOnce([path.join(process.cwd(), 'README.md')]);
@@ -194,9 +242,7 @@ describe('runner', () => {
             await expect(run({
                 ...baseConfig,
                 rules: { ...baseConfig.rules, 'file-path': { enabled: true } },
-                customExtractors: [
-                    { extract: () => [reference], supportsFormat: () => true },
-                ],
+                customExtractors: [{ extract: () => [reference], supportsFormat: () => true }],
             })).rejects.toThrow('file validator crashed');
         }
         finally {
@@ -237,6 +283,73 @@ describe('runner', () => {
             });
         });
     });
+    describe('shared manifest inventory', () => {
+        afterEach(() => vi.restoreAllMocks());
+        const manifestDocument = {
+            path: 'doc.md',
+            absolutePath: '/doc.md',
+            content: '',
+            format: 'markdown',
+            lines: [],
+            references: [
+                {
+                    type: 'version',
+                    value: 'TypeScript 5.0',
+                    technology: 'typescript',
+                    version: '5.0',
+                    lineNumber: 1,
+                    raw: 'TypeScript 5.0',
+                    sourceFile: 'doc.md',
+                },
+                { type: 'dependency', value: 'typescript', lineNumber: 2, raw: 'typescript', sourceFile: 'doc.md' },
+            ],
+        };
+        it('reads a manifest once for both built-in validators', async () => {
+            const rootDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'doc-freshness-runner-manifest-'));
+            await fs.promises.writeFile(path.join(rootDir, 'package.json'), JSON.stringify({ dependencies: { typescript: '5.9.0' } }));
+            vi.spyOn(DocumentParser.prototype, 'scanDocuments').mockResolvedValue([manifestDocument]);
+            const readFile = vi.spyOn(fs.promises, 'readFile');
+            try {
+                const results = await run({
+                    ...baseConfig,
+                    rootDir,
+                    manifestFiles: ['package.json'],
+                    rules: { ...baseConfig.rules, version: { enabled: true }, dependency: { enabled: true } },
+                });
+                expect(results.summary).toMatchObject({ total: 2, valid: 2 });
+                expect(readFile).toHaveBeenCalledOnce();
+            }
+            finally {
+                await fs.promises.rm(rootDir, { recursive: true, force: true });
+            }
+        });
+        it('does no manifest work when there are no references or both rules are disabled', async () => {
+            const dependencyNames = vi.spyOn(ManifestInventory.prototype, 'dependencyNames');
+            const packageVersions = vi.spyOn(ManifestInventory.prototype, 'packageVersions');
+            await run(baseConfig);
+            vi.spyOn(DocumentParser.prototype, 'scanDocuments').mockResolvedValue([manifestDocument]);
+            await run(baseConfig);
+            expect(dependencyNames).not.toHaveBeenCalled();
+            expect(packageVersions).not.toHaveBeenCalled();
+        });
+        it('does no built-in manifest work when custom validators replace both types', async () => {
+            vi.spyOn(DocumentParser.prototype, 'scanDocuments').mockResolvedValue([manifestDocument]);
+            const dependencyNames = vi.spyOn(ManifestInventory.prototype, 'dependencyNames');
+            const packageVersions = vi.spyOn(ManifestInventory.prototype, 'packageVersions');
+            const validateBatch = vi.fn().mockResolvedValue([]);
+            await run({
+                ...baseConfig,
+                rules: { ...baseConfig.rules, version: { enabled: true }, dependency: { enabled: true } },
+                customValidators: {
+                    version: { validateBatch },
+                    dependency: { validateBatch },
+                },
+            });
+            expect(validateBatch).toHaveBeenCalledTimes(2);
+            expect(dependencyNames).not.toHaveBeenCalled();
+            expect(packageVersions).not.toHaveBeenCalled();
+        });
+    });
     describe('verbose mode', () => {
         it('logs config file path and source patterns', async () => {
             const spy = captureLog();
@@ -271,9 +384,11 @@ describe('runner', () => {
         });
     });
     describe('reporters', () => {
-        it('generates console report', async () => {
+        it('routes the Console emitter to stdout incrementally', async () => {
             const spy = captureLog();
             await run({ ...baseConfig, reporters: ['console'] });
+            expect(spy.mock.calls.length).toBeGreaterThan(1);
+            expect(spy.mock.calls.every((call) => call.length === 1)).toBe(true);
             expect(spy.mock.calls.flat().join('\n')).toContain('Documentation Freshness Report');
         });
         it('falls back to console when the reporter list is empty', async () => {
@@ -331,6 +446,95 @@ describe('runner', () => {
                 expect(spy.mock.calls.flat().join('\n')).toContain('written to');
             });
         });
+        it('defaults an absent reporter list to Console', async () => {
+            const log = captureLog();
+            await run({ ...baseConfig, reporters: undefined });
+            expect(log.mock.calls.flat().join('\n')).toContain('Documentation Freshness Report');
+        });
+        it('preserves configured reporter order and duplicates', async () => {
+            const log = captureLog();
+            await run({ ...baseConfig, reporters: ['json', 'markdown', 'json'] });
+            const reports = log.mock.calls
+                .map(([value]) => value)
+                .filter((value) => typeof value === 'string' && (value.startsWith('{') || value.startsWith('#')));
+            expect(reports).toHaveLength(3);
+            expect(reports[0]).toMatch(/^\{/);
+            expect(reports[1]).toMatch(/^# Documentation Freshness Report/);
+            expect(reports[2]).toMatch(/^\{/);
+        });
+        it('treats inherited registry keys as unknown', async () => {
+            const inheritedKeys = ['toString', 'constructor', '__proto__'];
+            const log = captureLog();
+            const warn = captureWarn();
+            await expect(run({ ...baseConfig, reporters: inheritedKeys })).resolves.toBeDefined();
+            expect(warn.mock.calls).toEqual([
+                ['Unknown reporter type: toString'],
+                ['Unknown reporter type: constructor'],
+                ['Unknown reporter type: __proto__'],
+            ]);
+            expect(log.mock.calls.some(([value]) => typeof value === 'string' && value.startsWith('{'))).toBe(false);
+            const output = log.mock.calls.flat().join('\n');
+            expect(output).not.toContain('Documentation Freshness Report');
+            expect(output).not.toContain('Documentation Freshness Scan Report');
+        });
+        it('keeps Console on stdout and ignores outputPath', async () => {
+            await withOutputFile(cacheRoot, 'console-must-not-write.out', async (outputPath) => {
+                const log = captureLog();
+                await run({ ...baseConfig, reporters: ['console'], outputPath });
+                expect(log.mock.calls.flat().join('\n')).toContain('Documentation Freshness Report');
+                await expect(fs.promises.access(outputPath)).rejects.toThrow();
+            });
+        });
+        it('overwrites a shared outputPath sequentially so the last string reporter wins', async () => {
+            await withOutputFile(cacheRoot, 'shared-reporter.out', async (outputPath) => {
+                captureLog();
+                await run({ ...baseConfig, reporters: ['json', 'markdown'], outputPath });
+                expect(await fs.promises.readFile(outputPath, 'utf-8')).toMatch(/^# Documentation Freshness Report/);
+                await run({ ...baseConfig, reporters: ['markdown', 'json'], outputPath });
+                expect(await fs.promises.readFile(outputPath, 'utf-8')).toMatch(/^\{/);
+            });
+        });
+        it('creates recursive output directories and reports each verbose label', async () => {
+            const outputRoot = path.join(cacheRoot, 'reporter-routing');
+            const outputPath = path.join(outputRoot, 'nested', 'report.out');
+            const log = captureLog();
+            try {
+                await run({ ...baseConfig, reporters: ['json', 'markdown'], outputPath, verbose: true });
+                expect(await fs.promises.readFile(outputPath, 'utf-8')).toMatch(/^# Documentation Freshness Report/);
+                expect(log.mock.calls.flat()).toContain(`JSON report written to ${outputPath}`);
+                expect(log.mock.calls.flat()).toContain(`Markdown report written to ${outputPath}`);
+            }
+            finally {
+                await fs.promises.rm(outputRoot, { recursive: true, force: true });
+            }
+        });
+        it('rejects when writing a string report fails', async () => {
+            const outputPath = path.join(cacheRoot, 'reporter-write-directory');
+            await fs.promises.mkdir(outputPath, { recursive: true });
+            try {
+                captureLog();
+                await expect(run({ ...baseConfig, reporters: ['json'], outputPath })).rejects.toThrow();
+            }
+            finally {
+                await fs.promises.rm(outputPath, { recursive: true, force: true });
+            }
+        });
+        it('reads the clock only for timestamped reporter paths and propagates clock failures', async () => {
+            const toISOString = vi.spyOn(Date.prototype, 'toISOString').mockImplementation(() => {
+                throw new Error('timestamp failed');
+            });
+            const log = captureLog();
+            try {
+                await expect(run({ ...baseConfig, reporters: ['console'] })).resolves.toBeDefined();
+                await expect(run({ ...baseConfig, reporters: ['json'] })).resolves.toBeDefined();
+                expect(toISOString).not.toHaveBeenCalled();
+                expect(log).toHaveBeenCalled();
+                await expect(run({ ...baseConfig, reporters: ['markdown'] })).rejects.toThrow('timestamp failed');
+            }
+            finally {
+                toISOString.mockRestore();
+            }
+        });
     });
     describe('graph and scoring', () => {
         it('builds graph without persisting an unused graph cache', async () => {
@@ -345,33 +549,152 @@ describe('runner', () => {
                 await fs.promises.rm(path.join(process.cwd(), cacheDir), { recursive: true, force: true }).catch(() => { });
             }
         });
-        it('generates json with scores to file', async () => {
-            await withOutputFile(cacheRoot, 'test-scored.json', async (outputPath) => {
+        it('does not read the clock when runner scored-result spreading fails', async () => {
+            const validationResults = {
+                documents: [],
+                summary: { total: 0, valid: 0, errors: 0, warnings: 0, skipped: 0 },
+            };
+            const proxiedResults = new Proxy(validationResults, {
+                get(target, property, receiver) {
+                    if (property === 'documents') {
+                        throw new Error('runner result getter failed');
+                    }
+                    return Reflect.get(target, property, receiver);
+                },
+            });
+            const scores = {
+                projectScore: 100,
+                projectGrade: 'A',
+                documents: [],
+                summary: { total: 0, gradeA: 0, gradeB: 0, gradeC: 0, gradeD: 0, gradeF: 0 },
+            };
+            const validate = vi.spyOn(ValidationEngine.prototype, 'validate').mockResolvedValue(proxiedResults);
+            const calculateScores = vi.spyOn(FreshnessScorer.prototype, 'calculateProjectScores').mockReturnValue(scores);
+            const clock = vi.spyOn(Date.prototype, 'toISOString');
+            try {
                 captureLog();
-                await run({
+                await expect(run({
                     ...baseConfig,
                     reporters: ['json'],
-                    outputPath,
                     graph: { enabled: true },
                     freshnessScoring: { enabled: true },
                     cache: { enabled: false },
-                });
-                expect(JSON.parse(await fs.promises.readFile(outputPath, 'utf-8'))).toHaveProperty('summary');
-            });
+                })).rejects.toThrow('runner result getter failed');
+                expect(clock).not.toHaveBeenCalled();
+            }
+            finally {
+                validate.mockRestore();
+                calculateScores.mockRestore();
+                clock.mockRestore();
+            }
         });
-        it('generates markdown with scores to file', async () => {
-            await withOutputFile(cacheRoot, 'test-scored.md', async (outputPath) => {
-                captureLog();
+        it('captures Markdown result references before the clock, then renders the base before scores', async () => {
+            const events = [];
+            const summary = { total: 1, valid: 1, errors: 0, warnings: 0, skipped: 0 };
+            const documents = [];
+            let currentSummary = summary;
+            let currentDocuments = documents;
+            const validationResults = {
+                get summary() {
+                    events.push('results:summary');
+                    return currentSummary;
+                },
+                set summary(value) {
+                    currentSummary = value;
+                },
+                get documents() {
+                    events.push('results:documents');
+                    return currentDocuments;
+                },
+                set documents(value) {
+                    currentDocuments = value;
+                },
+            };
+            const scores = {
+                get projectScore() {
+                    events.push('scores:projectScore');
+                    summary.total = 99;
+                    return 100;
+                },
+                projectGrade: 'A',
+                documents: [],
+                summary: { total: 0, gradeA: 0, gradeB: 0, gradeC: 0, gradeD: 0, gradeF: 0 },
+            };
+            const validate = vi.spyOn(ValidationEngine.prototype, 'validate').mockResolvedValue(validationResults);
+            const calculateScores = vi.spyOn(FreshnessScorer.prototype, 'calculateProjectScores').mockReturnValue(scores);
+            const clock = vi.spyOn(Date.prototype, 'toISOString').mockImplementation(() => {
+                events.push('clock');
+                summary.total = 7;
+                documents.push({ path: 'docs/captured.md', issues: [] });
+                validationResults.summary = { total: 50, valid: 50, errors: 0, warnings: 0, skipped: 0 };
+                validationResults.documents = [];
+                return '2025-01-02T03:04:05.678Z';
+            });
+            try {
+                const log = captureLog();
                 await run({
                     ...baseConfig,
                     reporters: ['markdown'],
-                    outputPath,
                     graph: { enabled: true },
                     freshnessScoring: { enabled: true },
                     cache: { enabled: false },
                 });
-                expect(await fs.promises.readFile(outputPath, 'utf-8')).toContain('Freshness Scores');
-            });
+                const report = log.mock.calls.flat().find((value) => typeof value === 'string' && value.startsWith('#'));
+                expect(events).toEqual(['results:summary', 'results:documents', 'clock', 'scores:projectScore']);
+                expect(report).toContain('| Total Checked | 7 |');
+                expect(report).toContain('docs/captured.md');
+                expect(report).not.toContain('| Total Checked | 99 |');
+                expect(report).not.toContain('| Total Checked | 50 |');
+            }
+            finally {
+                validate.mockRestore();
+                calculateScores.mockRestore();
+                clock.mockRestore();
+            }
+        });
+        it('does not inspect Markdown documents, clock, or scores when the summary getter fails', async () => {
+            const events = [];
+            let scoreRead = false;
+            const validationResults = {
+                get summary() {
+                    events.push('results:summary');
+                    throw new Error('Markdown summary failed');
+                },
+                get documents() {
+                    events.push('results:documents');
+                    return [];
+                },
+            };
+            const scores = {
+                get projectScore() {
+                    scoreRead = true;
+                    return 100;
+                },
+                projectGrade: 'A',
+                documents: [],
+                summary: { total: 0, gradeA: 0, gradeB: 0, gradeC: 0, gradeD: 0, gradeF: 0 },
+            };
+            const validate = vi.spyOn(ValidationEngine.prototype, 'validate').mockResolvedValue(validationResults);
+            const calculateScores = vi.spyOn(FreshnessScorer.prototype, 'calculateProjectScores').mockReturnValue(scores);
+            const clock = vi.spyOn(Date.prototype, 'toISOString');
+            try {
+                captureLog();
+                await expect(run({
+                    ...baseConfig,
+                    reporters: ['markdown'],
+                    graph: { enabled: true },
+                    freshnessScoring: { enabled: true },
+                    cache: { enabled: false },
+                })).rejects.toThrow('Markdown summary failed');
+                expect(events).toEqual(['results:summary']);
+                expect(clock).not.toHaveBeenCalled();
+                expect(scoreRead).toBe(false);
+            }
+            finally {
+                validate.mockRestore();
+                calculateScores.mockRestore();
+                clock.mockRestore();
+            }
         });
         it('generates enhanced with scores to file', async () => {
             await withOutputFile(cacheRoot, 'test-enhanced-scored.md', async (outputPath) => {
@@ -941,6 +1264,7 @@ describe('runner', () => {
             const output = spy.mock.calls.flat().join('\n');
             expect(output).toContain('Indexing documentation');
             expect(output).toContain('Finding semantic mismatches');
+            expect(output).not.toContain('Building source code index');
         });
     });
     describe('runWithConfig', () => {
@@ -956,6 +1280,7 @@ describe('runner', () => {
         });
     });
     describe('vector search without prior graph', () => {
+        afterEach(() => vi.restoreAllMocks());
         it('builds source index independently when graph not enabled', async () => {
             const spy = captureLog();
             await run({
@@ -968,36 +1293,113 @@ describe('runner', () => {
             const output = spy.mock.calls.flat().join('\n');
             expect(output).toContain('Building source code index');
         });
-    });
-    describe('reporters with freshness scores', () => {
-        it('console reporter uses generateWithScores when scores available', async () => {
+        it('does not report index building when code-pattern validation already loaded it', async () => {
+            vi.spyOn(DocumentParser.prototype, 'scanDocuments').mockResolvedValue([
+                {
+                    path: 'doc.md',
+                    absolutePath: '/doc.md',
+                    content: '',
+                    format: 'markdown',
+                    lines: [],
+                    references: [{ type: 'code-pattern', value: 'RealSymbol', lineNumber: 1, raw: 'RealSymbol', sourceFile: 'doc.md' }],
+                },
+            ]);
             const spy = captureLog();
             await run({
                 ...baseConfig,
-                reporters: ['console'],
-                graph: { enabled: true },
-                freshnessScoring: { enabled: true },
-                cache: { enabled: false },
+                rules: { ...baseConfig.rules, 'code-pattern': { enabled: true } },
+                vectorSearch: { enabled: true },
+                graph: { enabled: false },
+                verbose: true,
             });
-            expect(spy.mock.calls.flat().join('\n')).toContain('Freshness Scores');
+            expect(spy.mock.calls.flat().join('\n')).not.toContain('Building source code index');
         });
-        it('json reporter generates with scores', async () => {
-            const spy = captureLog();
+    });
+    describe('shared source index', () => {
+        afterEach(() => vi.restoreAllMocks());
+        it('does not load source when no validator, graph, or vector consumer needs it', async () => {
+            const load = vi.spyOn(SourceIndex.prototype, 'load');
+            await run(baseConfig);
+            expect(load).not.toHaveBeenCalled();
+        });
+        it('uses one index instance for both built-in validators', async () => {
+            vi.spyOn(DocumentParser.prototype, 'scanDocuments').mockResolvedValue([
+                {
+                    path: 'doc.md',
+                    absolutePath: '/doc.md',
+                    content: '',
+                    format: 'markdown',
+                    lines: [],
+                    references: [
+                        { type: 'code-pattern', value: 'RealSymbol', lineNumber: 1, raw: 'RealSymbol', sourceFile: 'doc.md' },
+                        { type: 'code-snippet', kind: 'function-call', value: 'realFunction', lineNumber: 2, raw: '', sourceFile: 'doc.md' },
+                    ],
+                },
+            ]);
+            const load = vi.spyOn(SourceIndex.prototype, 'load');
+            await run({
+                ...baseConfig,
+                rules: {
+                    ...baseConfig.rules,
+                    'code-pattern': { enabled: true },
+                    'code-snippet': { enabled: true },
+                },
+            });
+            expect(load).toHaveBeenCalledTimes(2);
+            expect(new Set(load.mock.contexts).size).toBe(1);
+        });
+        it('feeds symbols to graph and all pattern files to vector search', async () => {
+            const symbols = new Map();
+            const patternFiles = new Map([['src/no-symbol.ts', { content: '// a useful comment', language: 'typescript' }]]);
+            const snapshot = {
+                symbols,
+                patternFiles,
+                snippetFiles: new Map(),
+                functionSignatures: new Map(),
+                interfaceKeys: new Map(),
+                exportsByFile: new Map(),
+                patternInputs: [],
+            };
+            const load = vi.spyOn(SourceIndex.prototype, 'load').mockResolvedValue(snapshot);
+            const buildGraph = vi.spyOn(GraphBuilder.prototype, 'buildGraph');
+            const indexCodeComments = vi.spyOn(VectorSearch.prototype, 'indexCodeComments');
+            captureLog();
+            await run({ ...baseConfig, graph: { enabled: true }, vectorSearch: { enabled: true } });
+            expect(buildGraph).toHaveBeenCalledWith([], symbols);
+            expect(indexCodeComments).toHaveBeenCalledWith([
+                { path: 'src/no-symbol.ts', content: '// a useful comment', language: 'typescript' },
+            ]);
+            expect(new Set(load.mock.contexts).size).toBe(1);
+        });
+        it('keeps custom validator overwrite independent from graph source loading', async () => {
+            const validateBatch = vi.fn().mockResolvedValue([]);
+            vi.spyOn(DocumentParser.prototype, 'scanDocuments').mockResolvedValue([
+                {
+                    path: 'doc.md',
+                    absolutePath: '/doc.md',
+                    content: '',
+                    format: 'markdown',
+                    lines: [],
+                    references: [{ type: 'code-pattern', value: 'Custom', lineNumber: 1, raw: 'Custom', sourceFile: 'doc.md' }],
+                },
+            ]);
+            const load = vi.spyOn(SourceIndex.prototype, 'load');
             await run({
                 ...baseConfig,
-                reporters: ['json'],
                 graph: { enabled: true },
-                freshnessScoring: { enabled: true },
-                cache: { enabled: false },
+                rules: { ...baseConfig.rules, 'code-pattern': { enabled: true } },
+                customValidators: { 'code-pattern': { validateBatch } },
             });
-            const jsonStr = spy.mock.calls.flat().find((a) => typeof a === 'string' && a.startsWith('{'));
-            expect(JSON.parse(jsonStr)).toHaveProperty('summary');
+            expect(validateBatch).toHaveBeenCalledOnce();
+            expect(load).toHaveBeenCalledOnce();
         });
-        it('markdown reporter generates with scores to stdout', async () => {
+    });
+    describe('reporters with freshness scores', () => {
+        it('console reporter uses generateWithScores when scores available', async () => {
             const spy = captureLog();
             await run({
                 ...baseConfig,
-                reporters: ['markdown'],
+                reporters: ['console'],
                 graph: { enabled: true },
                 freshnessScoring: { enabled: true },
                 cache: { enabled: false },