@testspectra/cli 1.0.70 → 1.1.0-rc.1

package/dist/commands/__tests__/run-e2e.test.js

@@ -0,0 +1,422 @@
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
+import { runCommand } from '../run.js';
+import { initCommand } from '../init.js';
+import { DemoServer } from '../../utils/demo-server.js';
+import { DemoApiServer } from '../../utils/api-server.js';
+import { RustCoreBridge } from '../../runner/bridge.js';
+describe('End-to-End (E2E) Test Plan: Native Orchestrator Execution (docs/v2/core/e2e-testing-plan.md)', () => {
+    let tmpDir;
+    let originalCwd;
+    let demoServer;
+    let apiServer;
+    let consoleLogs = [];
+    const originalLog = console.log;
+    beforeAll(async () => {
+        // 1. BeforeAll: Start demo web app server
+        demoServer = new DemoServer(5173);
+        try {
+            await demoServer.start();
+        }
+        catch {
+            // Server may already be running on port 5173
+        }
+        // The demo API server runs separately (port 5200). Tests that don't pass --demo still need it
+        // reachable: the web app now fetches the API cross-origin, so a JSON POST triggers a CORS
+        // preflight (OPTIONS) that isn't mocked and must reach a live API server.
+        apiServer = new DemoApiServer();
+        try {
+            await apiServer.start();
+        }
+        catch {
+            // API server may already be running on port 5200
+        }
+    });
+    afterAll(async () => {
+        // 4. AfterAll: Stop demo servers
+        try {
+            await demoServer.stop();
+        }
+        catch { }
+        try {
+            await apiServer.stop();
+        }
+        catch { }
+    });
+    beforeEach(async () => {
+        originalCwd = process.cwd();
+        tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'testspectra-e2e-'));
+        process.chdir(tmpDir);
+        // Scaffold full template project
+        await initCommand({ template: 'default', force: true });
+        consoleLogs = [];
+        console.log = vi.fn((...args) => {
+            consoleLogs.push(args.map((a) => (typeof a === 'string' ? a : JSON.stringify(a))).join(' '));
+            originalLog(...args);
+        });
+    });
+    afterEach(() => {
+        console.log = originalLog;
+        process.chdir(originalCwd);
+        try {
+            fs.rmSync(tmpDir, { recursive: true, force: true });
+        }
+        catch { }
+        vi.restoreAllMocks();
+    });
+    // E2E-01: Default Template Full Run
+    it('E2E-01: Default Template Full Run executes native runner and produces result.json', async () => {
+        // Ensure binary exists or mock bridge if binary is undergoing compilation
+        const binaryExists = (() => {
+            try {
+                RustCoreBridge.resolveBinaryPath();
+                return true;
+            }
+            catch {
+                return false;
+            }
+        })();
+        expect(binaryExists).toBe(true);
+        // Execute testspectra run -t web on full template project
+        await runCommand(undefined, { target: 'web', demo: true });
+        // Assert reports created
+        const resultJsonPath = path.join(tmpDir, '.testspectra', 'reports', 'result.json');
+        expect(fs.existsSync(resultJsonPath)).toBe(true);
+        const resultRaw = fs.readFileSync(resultJsonPath, 'utf-8');
+        const resultJson = JSON.parse(resultRaw);
+        // Validate result.json schema contract (Section 3.2.A)
+        expect(resultJson.version).toBe('2.0.0');
+        expect(resultJson.platform).toBe('web');
+        expect(resultJson.status).toBe('passed');
+        expect(resultJson.summary.passRate).toBe('100%');
+        expect(resultJson.summary.totalSuites).toBeGreaterThanOrEqual(1);
+        expect(resultJson.summary.totalTests).toBeGreaterThanOrEqual(1);
+        expect(resultJson.summary.passed).toBeGreaterThanOrEqual(1);
+        expect(resultJson.summary.failed).toBe(0);
+        // Performance RSS assertion (< 30 MB)
+        expect(resultJson.performance.peakMemoryRssMb).toBeLessThan(30);
+        expect(resultJson.suites.length).toBeGreaterThanOrEqual(1);
+    }, 30000);
+    // E2E-02: Single Suite Targeted Run
+    it('E2E-02: Single Suite Targeted Run executes only specified spec', async () => {
+        const targetSpec = 'specs/Playground/TC-0001-form-controls/web.test.ts';
+        expect(fs.existsSync(path.join(tmpDir, targetSpec))).toBe(true);
+        await runCommand([targetSpec], { target: 'web', demo: true });
+        const resultJsonPath = path.join(tmpDir, '.testspectra', 'reports', 'result.json');
+        expect(fs.existsSync(resultJsonPath)).toBe(true);
+        const resultJson = JSON.parse(fs.readFileSync(resultJsonPath, 'utf-8'));
+        expect(resultJson.version).toBe('2.0.0');
+        expect(resultJson.platform).toBe('web');
+        expect(resultJson.status).toBe('passed');
+        expect(resultJson.summary.totalSuites).toBe(1);
+        expect(resultJson.summary.totalTests).toBe(1);
+        expect(resultJson.summary.passed).toBe(1);
+        expect(resultJson.summary.failed).toBe(0);
+        expect(resultJson.summary.passRate).toBe('100%');
+    }, 30000);
+    // E2E-03: Network Recording Audit
+    it('E2E-03: Network Recording Audit creates network-resources.json with request entries', async () => {
+        await runCommand(undefined, { target: 'web', demo: true });
+        const networkJsonPath = path.join(tmpDir, '.testspectra', 'reports', 'network-resources.json');
+        expect(fs.existsSync(networkJsonPath)).toBe(true);
+        const networkRaw = fs.readFileSync(networkJsonPath, 'utf-8');
+        const networkResources = JSON.parse(networkRaw);
+        expect(Array.isArray(networkResources)).toBe(true);
+        expect(networkResources.length).toBeGreaterThanOrEqual(1);
+        const first = networkResources[0];
+        expect(first).toHaveProperty('requestId');
+        expect(first).toHaveProperty('url');
+        expect(first).toHaveProperty('method');
+        expect(first).toHaveProperty('status');
+    }, 30000);
+    // E2E-04: Semantic Logs Streaming & Summary Card Terminal
+    it('E2E-04: Semantic Logs Streaming displays natural logs and summary card', async () => {
+        await runCommand(undefined, { target: 'web', demo: true });
+        const allOutput = consoleLogs.join('\n');
+        expect(allOutput).toContain('TESTSPECTRA EXECUTION SUMMARY');
+        expect(allOutput).toContain('PASSED');
+        expect(allOutput).toContain('Platform');
+        expect(allOutput).toContain('Duration');
+        expect(allOutput).toContain('Report');
+    }, 30000);
+    // E2E-05: Failure Handling & Trace
+    it('E2E-05: Failure Handling & Trace properly records failed status and error details', async () => {
+        // Inject a failing step / invalid selector in a new test case
+        const failingDir = path.join(tmpDir, 'specs', 'Playground', 'TC-0099-failing');
+        fs.mkdirSync(failingDir, { recursive: true });
+        fs.writeFileSync(path.join(failingDir, 'web.test.ts'), `it("should fail when element does not exist", async () => {
+  await Step.verifyPlaygroundState();
+  await $("#non-existent-button-999").shouldBeVisible();
+});
+`, 'utf-8');
+        // Mock exit to prevent process termination during unit test assertion
+        const exitSpy = vi.spyOn(process, 'exit').mockImplementation((() => { }));
+        await runCommand(['specs/Playground/TC-0099-failing/web.test.ts'], { target: 'web', demo: true });
+        const resultJsonPath = path.join(tmpDir, '.testspectra', 'reports', 'result.json');
+        expect(fs.existsSync(resultJsonPath)).toBe(true);
+        const resultJson = JSON.parse(fs.readFileSync(resultJsonPath, 'utf-8'));
+        // If failure was triggered, status is failed and failed count >= 1
+        expect(resultJson.status).toBeDefined();
+        exitSpy.mockRestore();
+    }, 30000);
+    // E2E-06: Spectra Static API & Assertion Mismatch Failure in Real Chromium
+    it('E2E-06: Spectra static actions and negative assertion failures strictly fail in real Chromium', async () => {
+        // 1. Test a valid spec using pure Spectra static commands
+        const spectraStaticDir = path.join(tmpDir, 'specs', 'Playground', 'TC-0100-spectra-static');
+        fs.mkdirSync(spectraStaticDir, { recursive: true });
+        fs.writeFileSync(path.join(spectraStaticDir, 'web.test.ts'), `it("should execute via unified Spectra static object", async () => {
+  await Spectra.navigate("/#playground");
+  await Spectra.type("#demo-email-input-pg", "automated@spectra.dev");
+  await Spectra.get("#demo-email-input-pg").shouldHaveValue("automated@spectra.dev");
+  await Spectra.wait(50);
+  await Spectra.getAll(".demo-product-item").shouldHaveLengthGreaterThan(0);
+});
+`, 'utf-8');
+        await runCommand(['specs/Playground/TC-0100-spectra-static/web.test.ts'], { target: 'web', demo: true });
+        const resultStatic = JSON.parse(fs.readFileSync(path.join(tmpDir, '.testspectra', 'reports', 'result.json'), 'utf-8'));
+        expect(resultStatic.status).toBe('passed');
+        expect(resultStatic.summary.passed).toBe(1);
+        // 2. Test that assertion mismatch on existing element strictly FAILS (no false positive)
+        const mismatchDir = path.join(tmpDir, 'specs', 'Playground', 'TC-0101-mismatch');
+        fs.mkdirSync(mismatchDir, { recursive: true });
+        fs.writeFileSync(path.join(mismatchDir, 'web.test.ts'), `it("should strictly fail when assertion text does not match", async () => {
+  await Spectra.navigate("/#playground");
+  // Element exists, but text is wrong -> MUST throw and fail
+  await Spectra.get("#demo-email-input-pg").shouldHaveValue("completely_wrong_value_xyz");
+});
+`, 'utf-8');
+        const exitSpy = vi.spyOn(process, 'exit').mockImplementation((() => { }));
+        await runCommand(['specs/Playground/TC-0101-mismatch/web.test.ts'], { target: 'web', demo: true });
+        const resultMismatch = JSON.parse(fs.readFileSync(path.join(tmpDir, '.testspectra', 'reports', 'result.json'), 'utf-8'));
+        expect(resultMismatch.status).toBe('failed');
+        expect(resultMismatch.summary.failed).toBeGreaterThanOrEqual(1);
+        exitSpy.mockRestore();
+    }, 25000);
+    // E2E-07: Spectra.intercept GET Mocking & Dynamic respondWith on Web in Real Chromium
+    it('E2E-07: Spectra.intercept GET Mocking & Dynamic respondWith on Web in real Chromium', async () => {
+        const targetSpec = 'specs/ApiInterception/TC-0008-fetch-mocking/web.test.ts';
+        expect(fs.existsSync(path.join(tmpDir, targetSpec))).toBe(true);
+        await runCommand([targetSpec], { target: 'web' });
+        const resultJsonPath = path.join(tmpDir, '.testspectra', 'reports', 'result.json');
+        expect(fs.existsSync(resultJsonPath)).toBe(true);
+        const resultJson = JSON.parse(fs.readFileSync(resultJsonPath, 'utf-8'));
+        expect(resultJson.version).toBe('2.0.0');
+        expect(resultJson.platform).toBe('web');
+        expect(resultJson.status).toBe('passed');
+        expect(resultJson.summary.totalSuites).toBe(1);
+        expect(resultJson.summary.totalTests).toBe(1);
+        expect(resultJson.summary.passed).toBe(1);
+        expect(resultJson.summary.failed).toBe(0);
+        expect(resultJson.summary.passRate).toBe('100%');
+        // Assert network interception log was recorded in network-resources.json
+        const networkJsonPath = path.join(tmpDir, '.testspectra', 'reports', 'network-resources.json');
+        expect(fs.existsSync(networkJsonPath)).toBe(true);
+        const networkEntries = JSON.parse(fs.readFileSync(networkJsonPath, 'utf-8'));
+        const interceptedPostReq = networkEntries.find((entry) => entry.url && entry.url.includes('/api/posts'));
+        expect(interceptedPostReq).toBeDefined();
+        expect(interceptedPostReq.status).toBe(200);
+    }, 20000);
+    // E2E-08: Spectra.intercept POST Mocking with status 201 and payload verification on Web in Real Chromium
+    it('E2E-08: Spectra.intercept POST Mocking with status 201 and payload verification on Web', async () => {
+        const targetSpec = 'specs/ApiInterception/TC-0009-post-mocking/web.test.ts';
+        expect(fs.existsSync(path.join(tmpDir, targetSpec))).toBe(true);
+        await runCommand([targetSpec], { target: 'web' });
+        const resultJsonPath = path.join(tmpDir, '.testspectra', 'reports', 'result.json');
+        expect(fs.existsSync(resultJsonPath)).toBe(true);
+        const resultJson = JSON.parse(fs.readFileSync(resultJsonPath, 'utf-8'));
+        expect(resultJson.version).toBe('2.0.0');
+        expect(resultJson.platform).toBe('web');
+        expect(resultJson.status).toBe('passed');
+        expect(resultJson.summary.totalSuites).toBe(1);
+        expect(resultJson.summary.totalTests).toBe(1);
+        expect(resultJson.summary.passed).toBe(1);
+        expect(resultJson.summary.failed).toBe(0);
+        const networkJsonPath = path.join(tmpDir, '.testspectra', 'reports', 'network-resources.json');
+        expect(fs.existsSync(networkJsonPath)).toBe(true);
+        const networkEntries = JSON.parse(fs.readFileSync(networkJsonPath, 'utf-8'));
+        const postReq = networkEntries.find((entry) => entry.url && entry.url.includes('/api/posts') && entry.status === 201);
+        expect(postReq).toBeDefined();
+    }, 20000);
+    // E2E-09: Mobile API Interception & Socket Driver on Android target
+    it('E2E-09: Mobile API Interception and verification on Android target', async () => {
+        const net = await import('net');
+        // Start a mock Spectra Native Android Driver TCP Server on port 8200
+        const mockAndroidServer = net.createServer((socket) => {
+            let rxBuf = Buffer.alloc(0);
+            let fetchClickCount = 0;
+            socket.on('data', (chunk) => {
+                rxBuf = Buffer.concat([rxBuf, chunk]);
+                while (rxBuf.length >= 4) {
+                    const payloadLen = rxBuf.readUInt32BE(0);
+                    if (rxBuf.length < 4 + payloadLen)
+                        break;
+                    const payloadStr = rxBuf.slice(4, 4 + payloadLen).toString('utf8');
+                    rxBuf = rxBuf.slice(4 + payloadLen);
+                    try {
+                        const msg = JSON.parse(payloadStr);
+                        const cmd = msg.cmd;
+                        const id = msg.id;
+                        const params = msg.params || {};
+                        let responseData = {};
+                        if (cmd === 'FIND_ALL') {
+                            const sel = (params.selector || '').toLowerCase();
+                            // The driver now reads via a single full-hierarchy dump (`FIND_ALL xpath //*`) and
+                            // resolves selectors in memory, while actions use a targeted lookup for a live
+                            // nodeId. This mock returns the same node set for both, so either path resolves.
+                            //
+                            // TC-0008's mobile.test.ts drives 4 clicks in its single it() block: base mock
+                            // (2 items) -> respondWith dynamic update (1 item) -> respondOnce queued response
+                            // (1 item) -> reverted to base mock (2 items again).
+                            const items = fetchClickCount === 2
+                                ? [{ nodeId: 'item-dyn-1', text: 'Dynamic Updated Mock Title' }]
+                                : fetchClickCount === 3
+                                    ? [{ nodeId: 'item-once-1', text: 'One-Time Queued Response' }]
+                                    : [
+                                        { nodeId: 'item-1', text: 'Mocked Post 1 - TestSpectra Matchers' },
+                                        { nodeId: 'item-2', text: 'Mocked Post 2 - Cross-Platform Automation' },
+                                    ];
+                            const mkNode = (nodeId, text, id) => ({
+                                nodeId,
+                                text,
+                                resourceId: id,
+                                contentDesc: id,
+                                bounds: [0, 0, 10, 10],
+                                displayed: true,
+                                clickable: true,
+                                enabled: true,
+                                checked: false,
+                                selected: false,
+                                focused: false,
+                            });
+                            const tree = [
+                                mkNode('toggle-1', '☰', 'toggle-sidebar-btn'),
+                                mkNode('close-1', '✕', 'close-sidebar-btn'),
+                                mkNode('nav-1', 'API Fetch', 'nav-item-api-fetch'),
+                                mkNode('fetch-api-btn-node', 'Fetch Data', 'fetch-api-btn'),
+                                mkNode('badge-1', '200 OK', 'fetch-status-badge'),
+                                ...items.map((n, i) => mkNode(n.nodeId || `item-${i}`, n.text, 'api-post-item')),
+                                mkNode('badge-2', '201 Created', 'post-status-badge'),
+                                mkNode('banner-1', 'Successfully created post', 'post-success-banner'),
+                                mkNode('resp-json-1', '{"title": "TestSpectra Automated Post"}', 'post-response-json'),
+                            ];
+                            responseData = sel.startsWith('//')
+                                ? tree
+                                : tree.filter((n) => n.resourceId === sel || n.contentDesc === sel);
+                        }
+                        else if (cmd === 'CLICK' || cmd === 'PERFORM_ACTION') {
+                            if (params.node_id === 'fetch-api-btn-node' || params.nodeId === 'fetch-api-btn-node') {
+                                fetchClickCount++;
+                            }
+                            responseData = true;
+                        }
+                        else if (cmd === 'GET_ATTRIBUTE') {
+                            const attr = params.attr;
+                            if (attr === 'text') {
+                                responseData = '200 OK';
+                            }
+                            else {
+                                responseData = 'Mock Attribute';
+                            }
+                        }
+                        const respObj = { id, ok: true, data: responseData };
+                        const respStr = JSON.stringify(respObj);
+                        const respLen = Buffer.byteLength(respStr, 'utf8');
+                        const frame = Buffer.alloc(4 + respLen);
+                        frame.writeUInt32BE(respLen, 0);
+                        frame.write(respStr, 4, 'utf8');
+                        socket.write(frame);
+                    }
+                    catch { }
+                }
+            });
+        });
+        let isMockServerRunning = false;
+        await new Promise((resolve) => {
+            mockAndroidServer.once('error', (err) => {
+                if (err.code === 'EADDRINUSE') {
+                    resolve();
+                }
+            });
+            mockAndroidServer.listen(8200, '127.0.0.1', () => {
+                isMockServerRunning = true;
+                resolve();
+            });
+        });
+        try {
+            const configPath = path.join(tmpDir, 'spectra.config.ts');
+            if (fs.existsSync(configPath)) {
+                let conf = fs.readFileSync(configPath, 'utf-8');
+                conf = conf.replace(/appPackage:\s*["'][^"']*["'],/, 'appPackage: "",');
+                fs.writeFileSync(configPath, conf, 'utf-8');
+            }
+            const targetSpec = 'specs/ApiInterception/TC-0008-fetch-mocking/mobile.test.ts';
+            expect(fs.existsSync(path.join(tmpDir, targetSpec))).toBe(true);
+            await runCommand([targetSpec], { target: 'android', device: 'mock-device' });
+            const resultJsonPath = path.join(tmpDir, '.testspectra', 'reports', 'result.json');
+            expect(fs.existsSync(resultJsonPath)).toBe(true);
+            const resultJson = JSON.parse(fs.readFileSync(resultJsonPath, 'utf-8'));
+            expect(resultJson.version).toBe('2.0.0');
+            expect(resultJson.platform).toBe('android');
+            expect(resultJson.status).toBe('passed');
+            expect(resultJson.summary.passed).toBe(1);
+            expect(resultJson.summary.failed).toBe(0);
+        }
+        finally {
+            if (isMockServerRunning) {
+                await new Promise((resolve) => {
+                    mockAndroidServer.close(() => resolve());
+                });
+            }
+        }
+    }, 25000);
+    // E2E-10: Spectra.intercept FIFO respondOnce sequential polling and abort simulation
+    it('E2E-10: Spectra.intercept FIFO respondOnce sequential polling and abort simulation in real Chromium', async () => {
+        const fifoAbortDir = path.join(tmpDir, 'specs', 'ApiInterception', 'TC-0010-fifo-abort');
+        fs.mkdirSync(fifoAbortDir, { recursive: true });
+        fs.writeFileSync(path.join(fifoAbortDir, 'web.test.ts'), `it("should handle FIFO respondOnce queue and abort simulation", async () => {
+  await MatchersPage.open("api-fetch");
+
+  // Setup intercept with default fallback and FIFO queue
+  const mock = await Spectra.intercept("api/posts", "GET", [
+    { id: 999, title: "Default Fallback Post" }
+  ]);
+
+  await mock.respondOnce([
+    { id: 101, title: "FIFO Queued Post #1" }
+  ]);
+  await mock.respondOnce([
+    { id: 102, title: "FIFO Queued Post #2" }
+  ]);
+
+  // 1. First fetch triggers FIFO item #1
+  await MatchersPage.fetchApiButton.click();
+  await MatchersPage.fetchedItems.first().shouldContainText("FIFO Queued Post #1");
+
+  // 2. Second fetch triggers FIFO item #2
+  await MatchersPage.fetchApiButton.click();
+  await MatchersPage.fetchedItems.first().shouldContainText("FIFO Queued Post #2");
+
+  // 3. Third fetch falls back to default response
+  await MatchersPage.fetchApiButton.click();
+  await MatchersPage.fetchedItems.first().shouldContainText("Default Fallback Post");
+
+  // 4. Test abort simulation
+  await mock.abort("Failed");
+  await MatchersPage.fetchApiButton.click();
+  // On network failure, demo app shows fallback 200 OK from catch block
+  await MatchersPage.fetchStatusBadge.shouldHaveText("200 OK");
+});
+`, 'utf-8');
+        await runCommand(['specs/ApiInterception/TC-0010-fifo-abort/web.test.ts'], { target: 'web' });
+        const resultJsonPath = path.join(tmpDir, '.testspectra', 'reports', 'result.json');
+        expect(fs.existsSync(resultJsonPath)).toBe(true);
+        const resultJson = JSON.parse(fs.readFileSync(resultJsonPath, 'utf-8'));
+        expect(resultJson.version).toBe('2.0.0');
+        expect(resultJson.platform).toBe('web');
+        expect(resultJson.status).toBe('passed');
+        expect(resultJson.summary.passed).toBe(1);
+        expect(resultJson.summary.failed).toBe(0);
+    }, 25000);
+});

package/dist/commands/__tests__/run.test.d.ts

@@ -0,0 +1 @@
+export {};