@testspectra/cli 1.1.7 → 1.1.8-rc.10

package/dist/linter/__tests__/spec-schema.test.js

@@ -0,0 +1,369 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { SpecLinter } from '../spec-linter.js';
+import { parseMarkdownFrontmatter } from '../parser.js';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+describe('Spec Schema Linter (Stage 1 & Stage 2)', () => {
+    const tempDir = path.join(__dirname, '__temp_specs__');
+    beforeEach(() => {
+        fs.mkdirSync(tempDir, { recursive: true });
+    });
+    afterEach(() => {
+        if (fs.existsSync(tempDir)) {
+            fs.rmSync(tempDir, { recursive: true, force: true });
+        }
+    });
+    describe('Spec File Validation (spec.md)', () => {
+        it('passes validation for a well-formed spec.md with frontmatter, H1 title, and Test Steps', () => {
+            const specPath = path.join(tempDir, 'spec.md');
+            fs.writeFileSync(specPath, `---
+id: TC-0001-valid
+title: Verify user authentication
+priority: High
+caseType: Positive
+status: ready-for-automation
+coverage:
+  web: automated
+  mobile: automated
+tags:
+  - auth
+---
+
+# Verify user authentication
+
+## Test Steps
+1. Open login page and submit credentials
+`);
+            const result = SpecLinter.validateSpecFile(specPath);
+            expect(result.diagnostics).toHaveLength(0);
+            expect(result.id).toBe('TC-0001-valid');
+        });
+        it('reports testspectra/missing-frontmatter when required frontmatter attributes are missing', () => {
+            const specPath = path.join(tempDir, 'spec.md');
+            fs.writeFileSync(specPath, `---
+id: TC-0002-missing-attrs
+title: Missing priority and caseType
+status: draft
+---
+
+# Missing priority and caseType
+
+## Test Steps
+1. Step one
+`);
+            const result = SpecLinter.validateSpecFile(specPath);
+            const codes = result.diagnostics.map((d) => d.code);
+            expect(codes).toContain('testspectra/missing-frontmatter');
+            const missingPriority = result.diagnostics.find((d) => d.code === 'testspectra/missing-frontmatter' &&
+                d.message.includes("Missing required frontmatter attribute 'priority'"));
+            expect(missingPriority).toBeDefined();
+            expect(missingPriority?.severity).toBe('error');
+            const missingCaseType = result.diagnostics.find((d) => d.code === 'testspectra/missing-frontmatter' &&
+                d.message.includes("Missing required frontmatter attribute 'caseType'"));
+            expect(missingCaseType).toBeDefined();
+            expect(missingCaseType?.severity).toBe('error');
+        });
+        it('differentiates missing status (testspectra/missing-frontmatter) from invalid status value (testspectra/invalid-frontmatter-enum)', () => {
+            // 1. Missing status -> testspectra/missing-frontmatter
+            const missingSpecPath = path.join(tempDir, 'missing-status.spec.md');
+            fs.writeFileSync(missingSpecPath, `---
+id: TC-0002-missing-status
+title: Missing Status Spec
+priority: High
+caseType: Positive
+---
+
+# Missing Status Spec
+
+## Test Steps
+1. Step one
+`);
+            const missingResult = SpecLinter.validateSpecFile(missingSpecPath);
+            const missingStatusDiag = missingResult.diagnostics.find((d) => d.code === 'testspectra/missing-frontmatter' &&
+                d.message === "Missing required frontmatter attribute 'status'");
+            expect(missingStatusDiag).toBeDefined();
+            expect(missingStatusDiag?.severity).toBe('error');
+            expect(missingResult.diagnostics.some((d) => d.code === 'testspectra/invalid-frontmatter-enum')).toBe(false);
+            // 2. Invalid status -> testspectra/invalid-frontmatter-enum
+            const invalidSpecPath = path.join(tempDir, 'invalid-status.spec.md');
+            fs.writeFileSync(invalidSpecPath, `---
+id: TC-0002-invalid-status
+title: Invalid Status Spec
+priority: High
+caseType: Positive
+status: not-a-valid-status
+---
+
+# Invalid Status Spec
+
+## Test Steps
+1. Step one
+`);
+            const invalidResult = SpecLinter.validateSpecFile(invalidSpecPath);
+            const invalidStatusDiag = invalidResult.diagnostics.find((d) => d.code === 'testspectra/invalid-frontmatter-enum' &&
+                d.message === 'Invalid status value. Expected one of: draft, ready-for-automation, automated, deprecated');
+            expect(invalidStatusDiag).toBeDefined();
+            expect(invalidStatusDiag?.severity).toBe('error');
+            expect(invalidResult.diagnostics.some((d) => d.message.includes("Missing required frontmatter attribute 'status'"))).toBe(false);
+        });
+        it('reports testspectra/invalid-frontmatter-enum when enum attributes have invalid values', () => {
+            const specPath = path.join(tempDir, 'spec.md');
+            fs.writeFileSync(specPath, `---
+id: TC-0003-invalid-enum
+title: Invalid priority value
+priority: SuperHigh
+caseType: MaybePositive
+status: unknown-status
+---
+
+# Invalid priority value
+
+## Test Steps
+1. Step one
+`);
+            const result = SpecLinter.validateSpecFile(specPath);
+            const enumErrors = result.diagnostics.filter((d) => d.code === 'testspectra/invalid-frontmatter-enum');
+            expect(enumErrors.length).toBeGreaterThanOrEqual(1);
+            const priorityErr = enumErrors.find((d) => d.message.toLowerCase().includes('priority'));
+            expect(priorityErr).toBeDefined();
+            expect(priorityErr?.line).toBe(4); // Priority is on line 4
+        });
+        it('reports testspectra/missing-frontmatter when required H1 title section is missing in spec.md body', () => {
+            const specPath = path.join(tempDir, 'spec.md');
+            fs.writeFileSync(specPath, `---
+id: TC-0004-no-h1
+title: No Heading Section
+priority: High
+caseType: Positive
+status: draft
+---
+
+## Test Steps
+1. Step without H1
+`);
+            const result = SpecLinter.validateSpecFile(specPath);
+            const missingHeading = result.diagnostics.find((d) => d.code === 'testspectra/missing-frontmatter' && d.message.includes('expected top-level test title'));
+            expect(missingHeading).toBeDefined();
+            expect(missingHeading?.severity).toBe('error');
+        });
+        it('reports testspectra/missing-frontmatter when required ## Test Steps section is missing in spec.md body', () => {
+            const specPath = path.join(tempDir, 'spec.md');
+            fs.writeFileSync(specPath, `---
+id: TC-0005-no-steps
+title: No Steps Heading
+priority: High
+caseType: Positive
+status: draft
+---
+
+# No Steps Heading
+
+Some narrative without Test Steps section.
+`);
+            const result = SpecLinter.validateSpecFile(specPath);
+            const missingSteps = result.diagnostics.find((d) => d.code === 'testspectra/missing-frontmatter' && d.message.includes('expected "## Test Steps"'));
+            expect(missingSteps).toBeDefined();
+            expect(missingSteps?.severity).toBe('error');
+        });
+        it('warns when multiple H1 headings are declared in spec.md body', () => {
+            const specPath = path.join(tempDir, 'spec.md');
+            fs.writeFileSync(specPath, `---
+id: TC-0006-multiple-h1
+title: Multiple Headings
+priority: High
+caseType: Positive
+status: draft
+---
+
+# First Title
+
+# Second Title
+
+## Test Steps
+1. Step
+`);
+            const result = SpecLinter.validateSpecFile(specPath);
+            const multipleH1 = result.diagnostics.find((d) => d.severity === 'warning' && d.message.includes('Multiple top-level H1 headings'));
+            expect(multipleH1).toBeDefined();
+        });
+        it('reports testspectra/parse-error on malformed YAML syntax', () => {
+            const specPath = path.join(tempDir, 'spec.md');
+            fs.writeFileSync(specPath, `---
+id: [broken yaml
+title: Test
+---
+`);
+            const result = SpecLinter.validateSpecFile(specPath);
+            expect(result.diagnostics.some((d) => d.code === 'testspectra/parse-error')).toBe(true);
+        });
+    });
+    describe('Suite File Validation (suite.md)', () => {
+        it('passes validation for a well-formed suite.md with H1 suite heading', () => {
+            const suitePath = path.join(tempDir, 'suite.md');
+            fs.writeFileSync(suitePath, `---
+id: suite-auth
+name: Authentication
+description: User authentication flows
+executionOrder: 1.0
+parallel: true
+status: active
+---
+
+# Suite: Authentication
+`);
+            const result = SpecLinter.validateSuiteFile(suitePath);
+            expect(result.diagnostics).toHaveLength(0);
+            expect(result.id).toBe('suite-auth');
+        });
+        it('reports testspectra/missing-frontmatter when suite id or title/name is missing', () => {
+            const suitePath = path.join(tempDir, 'suite.md');
+            fs.writeFileSync(suitePath, `---
+description: No id and no name
+---
+
+# Suite: Incomplete
+`);
+            const result = SpecLinter.validateSuiteFile(suitePath);
+            expect(result.diagnostics.some((d) => d.code === 'testspectra/missing-frontmatter')).toBe(true);
+        });
+        it('reports testspectra/invalid-frontmatter-enum when suite status has invalid enum value', () => {
+            const suitePath = path.join(tempDir, 'suite.md');
+            fs.writeFileSync(suitePath, `---
+id: suite-invalid-status
+name: Invalid Status Suite
+status: unknown-status
+---
+
+# Suite: Invalid Status Suite
+`);
+            const result = SpecLinter.validateSuiteFile(suitePath);
+            const enumErr = result.diagnostics.find((d) => d.code === 'testspectra/invalid-frontmatter-enum');
+            expect(enumErr).toBeDefined();
+            expect(enumErr?.severity).toBe('error');
+        });
+        it('supports in-memory content override for suite.md without writing to disk', () => {
+            const suitePath = path.join(tempDir, 'non-existent-suite.md');
+            const inMemoryContent = `---
+id: suite-in-memory
+title: In Memory Suite
+status: active
+---
+
+# Suite: In Memory Suite
+`;
+            const result = SpecLinter.validateSuiteFile(suitePath, inMemoryContent);
+            expect(result.diagnostics).toHaveLength(0);
+            expect(result.id).toBe('suite-in-memory');
+        });
+        it('reports testspectra/missing-frontmatter when suite H1 title heading is missing', () => {
+            const suitePath = path.join(tempDir, 'suite.md');
+            fs.writeFileSync(suitePath, `---
+id: suite-no-heading
+name: No Heading
+---
+
+Body without H1 suite heading.
+`);
+            const result = SpecLinter.validateSuiteFile(suitePath);
+            const headingErr = result.diagnostics.find((d) => d.code === 'testspectra/missing-frontmatter' && d.message.includes('expected top-level suite title'));
+            expect(headingErr).toBeDefined();
+        });
+    });
+    describe('Workspace Validation & Duplicate ID Detection (testspectra/duplicate-spec-id)', () => {
+        it('detects duplicate IDs across different spec files', () => {
+            const tc1Dir = path.join(tempDir, 'TC-1');
+            const tc2Dir = path.join(tempDir, 'TC-2');
+            fs.mkdirSync(tc1Dir, { recursive: true });
+            fs.mkdirSync(tc2Dir, { recursive: true });
+            const specContent = `---
+id: TC-DUPLICATE-001
+title: Duplicate test case
+priority: High
+caseType: Positive
+status: automated
+---
+
+# Duplicate test case
+
+## Test Steps
+1. Duplicate step
+`;
+            fs.writeFileSync(path.join(tc1Dir, 'spec.md'), specContent);
+            fs.writeFileSync(path.join(tc2Dir, 'spec.md'), specContent);
+            const result = SpecLinter.validateWorkspace(tempDir);
+            expect(result.valid).toBe(false);
+            const dupErrors = result.diagnostics.filter((d) => d.code === 'testspectra/duplicate-spec-id');
+            expect(dupErrors).toHaveLength(1);
+            expect(dupErrors[0].message).toContain('TC-DUPLICATE-001');
+        });
+        it('successfully validates workspace with valid specs', () => {
+            const validSuiteDir = path.join(tempDir, 'ValidSuite');
+            const validTcDir = path.join(validSuiteDir, 'TC-001');
+            fs.mkdirSync(validTcDir, { recursive: true });
+            fs.writeFileSync(path.join(validSuiteDir, 'suite.md'), `---
+id: suite-valid
+name: Valid Suite
+status: active
+---
+
+# Suite: Valid Suite
+`);
+            fs.writeFileSync(path.join(validTcDir, 'spec.md'), `---
+id: TC-VALID-001
+title: Valid Test Case
+priority: High
+caseType: Positive
+status: automated
+---
+
+# Valid Test Case
+
+## Test Steps
+1. Open page
+`);
+            fs.writeFileSync(path.join(validTcDir, 'web.test.ts'), `it('Valid Test Case', async () => {});\n`);
+            const result = SpecLinter.validateWorkspace(tempDir);
+            expect(result.valid).toBe(true);
+            expect(result.stats.errorCount).toBe(0);
+            expect(result.stats.totalSpecs).toBe(1);
+            expect(result.stats.totalSuites).toBe(1);
+        });
+    });
+    describe('parseMarkdownFrontmatter Delimiter Detection Cases', () => {
+        it('Case 1: detects missing opening delimiter when closing delimiter exists', () => {
+            const content = `id: suite-api-interception\nname: ApiInterception\n---\n\n# Suite: ApiInterception\n`;
+            const result = parseMarkdownFrontmatter(content, 'specs/api/suite.md');
+            expect(result.hasFrontmatter).toBe(false);
+            expect(result.diagnostics).toHaveLength(1);
+            expect(result.diagnostics[0].code).toBe('testspectra/missing-frontmatter');
+            expect(result.diagnostics[0].message).toBe('Missing opening YAML frontmatter delimiter (---) at line 1');
+        });
+        it('Case 2: detects missing closing delimiter when opening delimiter exists', () => {
+            const content = `---\nid: suite-api-interception\nname: ApiInterception\n\n# Suite: ApiInterception\n`;
+            const result = parseMarkdownFrontmatter(content, 'specs/api/suite.md');
+            expect(result.hasFrontmatter).toBe(false);
+            expect(result.diagnostics).toHaveLength(1);
+            expect(result.diagnostics[0].code).toBe('testspectra/missing-frontmatter');
+            expect(result.diagnostics[0].message).toBe('Missing closing YAML frontmatter delimiter (---). Expected closing "---"');
+        });
+        it('Case 3: detects missing delimiters when frontmatter properties exist', () => {
+            const content = `id: suite-api-interception\nname: ApiInterception\nexecutionOrder: 1.4\nparallel: true\n\n# Suite: ApiInterception\n`;
+            const result = parseMarkdownFrontmatter(content, 'specs/api/suite.md');
+            expect(result.hasFrontmatter).toBe(false);
+            expect(result.diagnostics).toHaveLength(1);
+            expect(result.diagnostics[0].code).toBe('testspectra/missing-frontmatter');
+            expect(result.diagnostics[0].message).toBe('Missing YAML frontmatter delimiters (---) at line 1');
+        });
+        it('Case 4: detects missing delimiter when file is empty or only has markdown heading', () => {
+            const content = `# Suite: ApiInterception\n`;
+            const result = parseMarkdownFrontmatter(content, 'specs/api/suite.md');
+            expect(result.hasFrontmatter).toBe(false);
+            expect(result.diagnostics).toHaveLength(1);
+            expect(result.diagnostics[0].code).toBe('testspectra/missing-frontmatter');
+            expect(result.diagnostics[0].message).toBe('Missing YAML frontmatter delimiter (---) at line 1');
+        });
+    });
+});

package/dist/linter/__tests__/step-parity.test.d.ts

@@ -0,0 +1 @@
+export {};