@testspectra/cli 1.0.70 → 1.1.0-rc.0

package/dist/linter/spec-linter.js

@@ -0,0 +1,595 @@
+import fs from 'fs';
+import path from 'path';
+import { ZodError } from 'zod';
+import { areTitlesAligned, extractDeclaredStepsFromSpec, extractModuleExports, extractSymbolsFromTestCode, } from './extractor.js';
+import { isInsideSharedLibrary } from './discovery.js';
+import { parseMarkdownFrontmatter } from './parser.js';
+import { SpecFrontmatterSchema, validateSpecSections } from './schemas/spec.schema.js';
+import { SuiteFrontmatterSchema, validateSuiteSections } from './schemas/suite.schema.js';
+export class SpecLinter {
+    /**
+     * Validates a single spec.md file.
+     */
+    static validateSpecFile(filePath, contentOverride, companionCodeOverrides) {
+        const diagnostics = [];
+        if (contentOverride === undefined && !fs.existsSync(filePath)) {
+            return {
+                diagnostics: [
+                    {
+                        code: 'testspectra/parse-error',
+                        severity: 'error',
+                        message: `File does not exist: ${filePath}`,
+                        filePath,
+                        line: 1,
+                    },
+                ],
+            };
+        }
+        const content = contentOverride !== undefined ? contentOverride : fs.readFileSync(filePath, 'utf-8');
+        const parsed = parseMarkdownFrontmatter(content, filePath);
+        diagnostics.push(...parsed.diagnostics);
+        if (!parsed.hasFrontmatter || !parsed.data) {
+            return { diagnostics };
+        }
+        const data = parsed.data;
+        const id = typeof data.id === 'string' ? data.id : undefined;
+        const idLine = parsed.keyLineMap.get('id') || 1;
+        try {
+            SpecFrontmatterSchema.parse(data);
+        }
+        catch (err) {
+            if (err instanceof ZodError) {
+                for (const issue of err.issues) {
+                    const field = issue.path[0] ? String(issue.path[0]) : '';
+                    const line = parsed.keyLineMap.get(field) || 1;
+                    if (issue.code === 'invalid_enum_value') {
+                        diagnostics.push({
+                            code: 'testspectra/invalid-frontmatter-enum',
+                            severity: 'error',
+                            message: issue.message,
+                            filePath,
+                            line,
+                        });
+                    }
+                    else if (issue.code === 'invalid_type' &&
+                        (issue.received === 'undefined' || issue.message.toLowerCase().includes('missing required'))) {
+                        diagnostics.push({
+                            code: 'testspectra/missing-frontmatter',
+                            severity: 'error',
+                            message: issue.message,
+                            filePath,
+                            line: 1,
+                        });
+                    }
+                    else if (['priority', 'caseType', 'status'].includes(field)) {
+                        diagnostics.push({
+                            code: 'testspectra/invalid-frontmatter-enum',
+                            severity: 'error',
+                            message: issue.message,
+                            filePath,
+                            line,
+                        });
+                    }
+                    else {
+                        diagnostics.push({
+                            code: 'testspectra/missing-frontmatter',
+                            severity: 'error',
+                            message: issue.message,
+                            filePath,
+                            line,
+                        });
+                    }
+                }
+            }
+            else {
+                diagnostics.push({
+                    code: 'testspectra/parse-error',
+                    severity: 'error',
+                    message: err.message,
+                    filePath,
+                    line: 1,
+                });
+            }
+        }
+        // Validate markdown body sections
+        const sectionDiagnostics = validateSpecSections(parsed.body, filePath, parsed.bodyStartLine);
+        diagnostics.push(...sectionDiagnostics);
+        // ─────────────────────────────────────────────────────────────────────────
+        // Stage 3 & 4: AST Step Parity & Test Structure Enforcement
+        // ─────────────────────────────────────────────────────────────────────────
+        const declaredSteps = extractDeclaredStepsFromSpec(parsed.body, parsed.bodyStartLine);
+        const specDir = path.dirname(filePath);
+        let companionTestFiles = [];
+        try {
+            const entries = fs.readdirSync(specDir, { withFileTypes: true });
+            companionTestFiles = entries
+                .filter((e) => e.isFile() &&
+                (e.name.endsWith('.test.ts') ||
+                    e.name.endsWith('.web.ts') ||
+                    e.name.endsWith('.mobile.ts') ||
+                    e.name.endsWith('.android.ts') ||
+                    e.name.endsWith('.ios.ts') ||
+                    e.name.endsWith('.common.ts')))
+                .map((e) => path.join(specDir, e.name));
+        }
+        catch { }
+        if (companionCodeOverrides) {
+            const overrideKeys = companionCodeOverrides instanceof Map
+                ? Array.from(companionCodeOverrides.keys())
+                : Object.keys(companionCodeOverrides);
+            for (const overridePath of overrideKeys) {
+                if (path.dirname(overridePath) === specDir && !companionTestFiles.includes(overridePath)) {
+                    companionTestFiles.push(overridePath);
+                }
+            }
+        }
+        if (companionTestFiles.length === 0) {
+            if (declaredSteps.length > 0) {
+                for (const step of declaredSteps) {
+                    diagnostics.push({
+                        code: 'testspectra/missing-step-call',
+                        severity: 'error',
+                        message: `Parity Error: Step "@step:${step.name}" declared in spec.md line ${step.line} is missing in test code (no companion test files found)`,
+                        filePath,
+                        line: step.line,
+                    });
+                }
+            }
+        }
+        else {
+            const coverage = data.coverage;
+            for (const testFilePath of companionTestFiles) {
+                const testFileName = path.basename(testFilePath);
+                let testCode = '';
+                if (companionCodeOverrides) {
+                    if (companionCodeOverrides instanceof Map) {
+                        testCode = companionCodeOverrides.get(testFilePath) || '';
+                    }
+                    else {
+                        testCode = companionCodeOverrides[testFilePath] || '';
+                    }
+                }
+                if (!testCode) {
+                    try {
+                        testCode = fs.readFileSync(testFilePath, 'utf-8');
+                    }
+                    catch {
+                        continue;
+                    }
+                }
+                const symbols = extractSymbolsFromTestCode(testCode, testFilePath);
+                // testspectra/no-manual-import: Manual import statement detected (must be zero-import)
+                for (const imp of symbols.importStatements) {
+                    diagnostics.push({
+                        code: 'testspectra/no-manual-import',
+                        severity: 'error',
+                        message: 'Manual imports forbidden; all TestSpectra APIs, Page Objects, Steps, and Fixtures are ambient globals',
+                        filePath: testFilePath,
+                        line: imp.line,
+                        column: imp.column,
+                    });
+                }
+                // testspectra/no-test-block: forbidden test() calls detected
+                for (const tb of symbols.testBlocks) {
+                    diagnostics.push({
+                        code: 'testspectra/no-test-block',
+                        severity: 'error',
+                        message: 'test() is forbidden; TestSpectra test scripts must strictly use it() instead of test()',
+                        filePath: testFilePath,
+                        line: tb.line,
+                        column: tb.column,
+                    });
+                }
+                // testspectra/single-it-required: Multiple it() blocks (or 0 it blocks) detected in a single test case script
+                if (symbols.itBlocks.length === 0) {
+                    if (symbols.testBlocks.length === 0) {
+                        diagnostics.push({
+                            code: 'testspectra/single-it-required',
+                            severity: 'error',
+                            message: 'Test case script must contain exactly one it() block (found 0)',
+                            filePath: testFilePath,
+                            line: 1,
+                        });
+                    }
+                }
+                else if (symbols.itBlocks.length > 1) {
+                    for (let i = 1; i < symbols.itBlocks.length; i++) {
+                        const extra = symbols.itBlocks[i];
+                        diagnostics.push({
+                            code: 'testspectra/single-it-required',
+                            severity: 'error',
+                            message: `Multiple it() blocks detected (${symbols.itBlocks.length}); each test script must contain exactly one it() block`,
+                            filePath: testFilePath,
+                            line: extra.line,
+                            column: extra.column,
+                        });
+                    }
+                }
+                // testspectra/no-describe-block: describe(...) block detected (suites are strictly structured via directory hierarchy)
+                for (const desc of symbols.describeBlocks) {
+                    diagnostics.push({
+                        code: 'testspectra/no-describe-block',
+                        severity: 'error',
+                        message: 'describe() blocks are forbidden; test suites are structured via directory hierarchy with suite.md',
+                        filePath: testFilePath,
+                        line: desc.line,
+                        column: desc.column,
+                    });
+                }
+                // testspectra/title-mismatch: it("...") title in test script differs from title declared in spec.md frontmatter
+                if (symbols.itBlocks.length > 0 && typeof data.title === 'string') {
+                    const itTitle = symbols.itBlocks[0].title;
+                    if (!areTitlesAligned(data.title, itTitle)) {
+                        diagnostics.push({
+                            code: 'testspectra/title-mismatch',
+                            severity: 'warning',
+                            message: `Test title "${itTitle}" in ${testFileName}:L${symbols.itBlocks[0].line} does not match spec.md title "${data.title}"`,
+                            filePath: testFilePath,
+                            line: symbols.itBlocks[0].line,
+                            column: symbols.itBlocks[0].column,
+                        });
+                    }
+                }
+                // Determine if platform coverage is explicitly marked manual or unsupported
+                let isPlatformAutomated = true;
+                if (coverage) {
+                    if (testFileName.startsWith('web.') && (coverage.web === 'manual' || coverage.web === 'unsupported')) {
+                        isPlatformAutomated = false;
+                    }
+                    else if ((testFileName.startsWith('mobile.') ||
+                        testFileName.startsWith('android.') ||
+                        testFileName.startsWith('ios.')) &&
+                        (coverage.mobile === 'manual' || coverage.mobile === 'unsupported')) {
+                        isPlatformAutomated = false;
+                    }
+                }
+                const codeStepNames = new Set(symbols.stepCalls.map((s) => s.name));
+                // testspectra/missing-step-call: Step declared as @step:<name> in spec.md is absent in companion test script
+                if (isPlatformAutomated) {
+                    for (const declaredStep of declaredSteps) {
+                        if (!codeStepNames.has(declaredStep.name)) {
+                            diagnostics.push({
+                                code: 'testspectra/missing-step-call',
+                                severity: 'error',
+                                message: `Parity Error: Step "@step:${declaredStep.name}" declared in spec.md line ${declaredStep.line} is missing in ${testFileName}`,
+                                filePath,
+                                line: declaredStep.line,
+                            });
+                        }
+                    }
+                }
+                // testspectra/undocumented-step: Step.<name>() called in test script is not documented with @step:<name> in spec.md
+                const declaredStepNames = new Set(declaredSteps.map((s) => s.name));
+                for (const stepCall of symbols.stepCalls) {
+                    if (!declaredStepNames.has(stepCall.name)) {
+                        diagnostics.push({
+                            code: 'testspectra/undocumented-step',
+                            severity: 'warning',
+                            message: `Step "${stepCall.name}" called in ${testFileName}:L${stepCall.line} is not documented with @step:${stepCall.name} in spec.md`,
+                            filePath: testFilePath,
+                            line: stepCall.line,
+                            column: stepCall.column,
+                        });
+                    }
+                }
+            }
+        }
+        return { diagnostics, id, idLine };
+    }
+    /**
+     * Validates a single suite.md file.
+     */
+    static validateSuiteFile(filePath, contentOverride) {
+        const diagnostics = [];
+        if (contentOverride === undefined && !fs.existsSync(filePath)) {
+            return {
+                diagnostics: [
+                    {
+                        code: 'testspectra/parse-error',
+                        severity: 'error',
+                        message: `File does not exist: ${filePath}`,
+                        filePath,
+                        line: 1,
+                    },
+                ],
+            };
+        }
+        const content = contentOverride !== undefined ? contentOverride : fs.readFileSync(filePath, 'utf-8');
+        const parsed = parseMarkdownFrontmatter(content, filePath);
+        diagnostics.push(...parsed.diagnostics);
+        if (!parsed.hasFrontmatter || !parsed.data) {
+            return { diagnostics };
+        }
+        const data = parsed.data;
+        const id = typeof data.id === 'string' ? data.id : undefined;
+        const idLine = parsed.keyLineMap.get('id') || 1;
+        try {
+            SuiteFrontmatterSchema.parse(data);
+        }
+        catch (err) {
+            if (err instanceof ZodError) {
+                for (const issue of err.issues) {
+                    const field = issue.path[0] ? String(issue.path[0]) : '';
+                    const line = parsed.keyLineMap.get(field) || 1;
+                    if (issue.code === 'invalid_enum_value') {
+                        diagnostics.push({
+                            code: 'testspectra/invalid-frontmatter-enum',
+                            severity: 'error',
+                            message: issue.message,
+                            filePath,
+                            line,
+                        });
+                    }
+                    else if (issue.code === 'invalid_type' &&
+                        (issue.received === 'undefined' || issue.message.toLowerCase().includes('missing required'))) {
+                        diagnostics.push({
+                            code: 'testspectra/missing-frontmatter',
+                            severity: 'error',
+                            message: issue.message,
+                            filePath,
+                            line: 1,
+                        });
+                    }
+                    else if (field === 'status') {
+                        diagnostics.push({
+                            code: 'testspectra/invalid-frontmatter-enum',
+                            severity: 'error',
+                            message: issue.message,
+                            filePath,
+                            line,
+                        });
+                    }
+                    else {
+                        diagnostics.push({
+                            code: 'testspectra/missing-frontmatter',
+                            severity: 'error',
+                            message: issue.message,
+                            filePath,
+                            line,
+                        });
+                    }
+                }
+            }
+            else {
+                diagnostics.push({
+                    code: 'testspectra/parse-error',
+                    severity: 'error',
+                    message: err.message,
+                    filePath,
+                    line: 1,
+                });
+            }
+        }
+        // Validate markdown body sections
+        const sectionDiagnostics = validateSuiteSections(parsed.body, filePath, parsed.bodyStartLine);
+        diagnostics.push(...sectionDiagnostics);
+        return { diagnostics, id, idLine };
+    }
+    /**
+     * Validates all spec.md and suite.md files in a workspace or specific target path.
+     */
+    static validateWorkspace(workspaceDir, targetPath) {
+        const rootScan = targetPath ? path.resolve(workspaceDir, targetPath) : workspaceDir;
+        const specFiles = [];
+        const suiteFiles = [];
+        // Collect files
+        if (fs.existsSync(rootScan)) {
+            const stat = fs.statSync(rootScan);
+            if (stat.isFile()) {
+                const base = path.basename(rootScan);
+                if (base === 'spec.md')
+                    specFiles.push(rootScan);
+                else if (base === 'suite.md')
+                    suiteFiles.push(rootScan);
+            }
+            else {
+                // Directory traversal
+                const traverse = (dir) => {
+                    const entries = fs.readdirSync(dir, { withFileTypes: true });
+                    for (const entry of entries) {
+                        const fullPath = path.join(dir, entry.name);
+                        if (entry.isDirectory()) {
+                            if (entry.name === 'node_modules' ||
+                                entry.name === 'dist' ||
+                                entry.name === '.git' ||
+                                entry.name === '.testspectra') {
+                                continue;
+                            }
+                            traverse(fullPath);
+                        }
+                        else if (entry.isFile()) {
+                            if (entry.name === 'spec.md') {
+                                specFiles.push(fullPath);
+                            }
+                            else if (entry.name === 'suite.md') {
+                                suiteFiles.push(fullPath);
+                            }
+                        }
+                    }
+                };
+                // If target not specified and `specs/` exists, scan `specs/` by default
+                const defaultSpecsDir = path.join(rootScan, 'specs');
+                if (!targetPath && fs.existsSync(defaultSpecsDir)) {
+                    traverse(defaultSpecsDir);
+                }
+                else {
+                    traverse(rootScan);
+                }
+            }
+        }
+        const allDiagnostics = [];
+        const idTracker = new Map();
+        let passedSpecs = 0;
+        let passedSuites = 0;
+        // Validate suites first
+        for (const suitePath of suiteFiles) {
+            const { diagnostics, id, idLine } = this.validateSuiteFile(suitePath);
+            let fileValid = diagnostics.filter((d) => d.severity === 'error').length === 0;
+            if (id) {
+                if (idTracker.has(id)) {
+                    const prev = idTracker.get(id);
+                    diagnostics.push({
+                        code: 'testspectra/duplicate-spec-id',
+                        severity: 'error',
+                        message: `Duplicate suite id "${id}" detected across the workspace (first seen in ${path.relative(workspaceDir, prev.filePath)})`,
+                        filePath: suitePath,
+                        line: idLine || 1,
+                    });
+                    fileValid = false;
+                }
+                else {
+                    idTracker.set(id, { filePath: suitePath, line: idLine || 1 });
+                }
+            }
+            if (fileValid) {
+                passedSuites++;
+            }
+            allDiagnostics.push(...diagnostics);
+        }
+        // Validate specs
+        for (const specPath of specFiles) {
+            const { diagnostics, id, idLine } = this.validateSpecFile(specPath);
+            let fileValid = diagnostics.filter((d) => d.severity === 'error').length === 0;
+            if (id) {
+                if (idTracker.has(id)) {
+                    const prev = idTracker.get(id);
+                    diagnostics.push({
+                        code: 'testspectra/duplicate-spec-id',
+                        severity: 'error',
+                        message: `Duplicate test case id "${id}" detected across the workspace (first seen in ${path.relative(workspaceDir, prev.filePath)})`,
+                        filePath: specPath,
+                        line: idLine || 1,
+                    });
+                    fileValid = false;
+                }
+                else {
+                    idTracker.set(id, { filePath: specPath, line: idLine || 1 });
+                }
+            }
+            if (fileValid) {
+                passedSpecs++;
+            }
+            allDiagnostics.push(...diagnostics);
+        }
+        // ─────────────────────────────────────────────────────────────────────────
+        // testspectra/no-specs-in-shared-lib: Specs or test scripts detected inside shared testing library
+        // ─────────────────────────────────────────────────────────────────────────
+        for (const specPath of specFiles) {
+            if (isInsideSharedLibrary(specPath)) {
+                allDiagnostics.push({
+                    code: 'testspectra/no-specs-in-shared-lib',
+                    severity: 'error',
+                    message: `Test specification detected inside shared testing library; specs must be placed in feature package directories`,
+                    filePath: specPath,
+                    line: 1,
+                });
+            }
+        }
+        for (const suitePath of suiteFiles) {
+            if (isInsideSharedLibrary(suitePath)) {
+                allDiagnostics.push({
+                    code: 'testspectra/no-specs-in-shared-lib',
+                    severity: 'error',
+                    message: `Suite specification detected inside shared testing library; suites must be placed in feature package directories`,
+                    filePath: suitePath,
+                    line: 1,
+                });
+            }
+        }
+        // ─────────────────────────────────────────────────────────────────────────
+        // testspectra/no-manual-import (extended) & testspectra/require-anonymous-default-export: Scan shared scripts workspace-wide
+        // ─────────────────────────────────────────────────────────────────────────
+        const sharedScriptExtensions = ['.step.ts', '.action.ts', '.hook.ts'];
+        const collectSharedScripts = (dir) => {
+            const entries = fs.readdirSync(dir, { withFileTypes: true });
+            for (const entry of entries) {
+                const fullPath = path.join(dir, entry.name);
+                if (entry.isDirectory()) {
+                    if (entry.name === 'node_modules' ||
+                        entry.name === 'dist' ||
+                        entry.name === '.git' ||
+                        entry.name === '.testspectra') {
+                        continue;
+                    }
+                    collectSharedScripts(fullPath);
+                }
+                else if (entry.isFile()) {
+                    const isSharedScript = sharedScriptExtensions.some((ext) => entry.name.endsWith(ext));
+                    if (!isSharedScript)
+                        continue;
+                    let content = '';
+                    try {
+                        content = fs.readFileSync(fullPath, 'utf-8');
+                    }
+                    catch {
+                        continue;
+                    }
+                    // testspectra/no-manual-import: Check for manual imports in non-test scripts
+                    const symbols = extractSymbolsFromTestCode(content, fullPath);
+                    for (const imp of symbols.importStatements) {
+                        allDiagnostics.push({
+                            code: 'testspectra/no-manual-import',
+                            severity: 'error',
+                            message: 'Manual imports forbidden; all TestSpectra APIs, Page Objects, Steps, and Fixtures are ambient globals',
+                            filePath: fullPath,
+                            line: imp.line,
+                            column: imp.column,
+                        });
+                    }
+                    // testspectra/require-anonymous-default-export: Check for anonymous async default export
+                    const moduleExports = extractModuleExports(content, fullPath);
+                    // testspectra/require-anonymous-default-export: Named exports are forbidden
+                    for (const ne of moduleExports.namedExportLines) {
+                        allDiagnostics.push({
+                            code: 'testspectra/require-anonymous-default-export',
+                            severity: 'error',
+                            message: `Named export "${ne.name}" is forbidden; shared modules must use anonymous default export only`,
+                            filePath: fullPath,
+                            line: ne.line,
+                            column: ne.column,
+                        });
+                    }
+                    if (!moduleExports.hasAnonymousAsyncDefaultExport && moduleExports.namedExportLines.length === 0) {
+                        allDiagnostics.push({
+                            code: 'testspectra/require-anonymous-default-export',
+                            severity: 'error',
+                            message: 'Shared step/action/hook must export an anonymous async function as default export',
+                            filePath: fullPath,
+                            line: 1,
+                        });
+                    }
+                    // testspectra/require-anonymous-default-export: Test blocks are forbidden in shared modules
+                    for (const tb of moduleExports.testBlocks) {
+                        allDiagnostics.push({
+                            code: 'testspectra/require-anonymous-default-export',
+                            severity: 'error',
+                            message: `${tb.name}() block is forbidden inside shared step/action/hook modules`,
+                            filePath: fullPath,
+                            line: tb.line,
+                            column: tb.column,
+                        });
+                    }
+                }
+            }
+        };
+        try {
+            collectSharedScripts(rootScan);
+        }
+        catch { }
+        const errorCount = allDiagnostics.filter((d) => d.severity === 'error').length;
+        const warningCount = allDiagnostics.filter((d) => d.severity === 'warning').length;
+        const stats = {
+            totalSpecs: specFiles.length,
+            totalSuites: suiteFiles.length,
+            passedSpecs,
+            passedSuites,
+            errorCount,
+            warningCount,
+        };
+        return {
+            valid: errorCount === 0,
+            diagnostics: allDiagnostics,
+            stats,
+        };
+    }
+}

package/dist/linter/types.d.ts

@@ -0,0 +1,28 @@
+export type DiagnosticSeverity = 'error' | 'warning';
+export type LintDiagnosticCode = 'testspectra/parse-error' | 'testspectra/missing-frontmatter' | 'testspectra/invalid-frontmatter-enum' | 'testspectra/missing-step-call' | 'testspectra/undocumented-step' | 'testspectra/title-mismatch' | 'testspectra/no-manual-import' | 'testspectra/single-it-required' | 'testspectra/no-test-block' | 'testspectra/no-describe-block' | 'testspectra/duplicate-spec-id' | 'testspectra/no-specs-in-shared-lib' | 'testspectra/require-anonymous-default-export';
+export interface LintDiagnostic {
+    code: LintDiagnosticCode;
+    severity: DiagnosticSeverity;
+    message: string;
+    filePath: string;
+    line: number;
+    column?: number;
+}
+export interface SpecLintStats {
+    totalSpecs: number;
+    totalSuites: number;
+    passedSpecs: number;
+    passedSuites: number;
+    errorCount: number;
+    warningCount: number;
+}
+export interface SpecLintResult {
+    valid: boolean;
+    diagnostics: LintDiagnostic[];
+    stats: SpecLintStats;
+}
+export interface LintOptions {
+    strict?: boolean;
+    json?: boolean;
+    cwd?: string;
+}

package/dist/linter/types.js

@@ -0,0 +1 @@
+export {};

package/dist/mobile/__tests__/android-device-e2e.test.d.ts

@@ -0,0 +1 @@
+export {};