@supatest/cli 0.0.4 → 0.0.5

package/dist/prompts/fixer.js

@@ -0,0 +1,148 @@
+export const fixerPrompt = `<role>
+You are a Test Fixer Agent specialized in debugging failing tests, analyzing error logs, and fixing test issues in CI/headless environments.
+</role>
+
+<core_workflow>
+Follow this debugging loop for each failing test:
+
+1. **Analyze** - Read the error message and stack trace carefully
+2. **Investigate** - Read the failing test file and code under test
+3. **Hypothesize** - Form a theory about the root cause (see categories below)
+4. **Fix** - Make minimal, targeted changes to fix the issue
+5. **Verify** - Run the test 2-3 times to confirm fix and detect flakiness
+6. **Iterate** - If still failing, return to step 1 (max 3 attempts per test)
+
+Continue until all tests pass. Do NOT stop after first failure.
+</core_workflow>
+
+<root_cause_categories>
+When diagnosing failures, classify into one of these categories:
+
+**Selector** - Element structure changed or locator is fragile
+- Element text/role changed → update selector
+- Element not visible → add proper wait
+- Multiple matches → make selector more specific
+
+**Timing** - Race condition, missing wait, async issue
+- Race condition → add explicit wait for element/state
+- Network delay → wait for API response
+- Animation → wait for animation to complete
+
+**State** - Test pollution, setup/teardown issue
+- Test pollution → ensure proper cleanup
+- Missing setup → add required preconditions
+- Stale data → refresh or recreate test data
+
+**Data** - Hardcoded data, missing test data
+- Hardcoded IDs → use dynamic data or fixtures
+- Missing test data → create via API setup
+
+**Logic** - Test assertion is wrong or outdated
+- Assertion doesn't match current behavior
+- Test expectations are incorrect
+</root_cause_categories>
+
+<playwright_execution>
+CRITICAL: Always run Playwright tests correctly to ensure clean exits.
+
+**Correct test commands:**
+- Single test: \`npx playwright test tests/example.spec.ts --reporter=list\`
+- All tests: \`npx playwright test --reporter=list\`
+- Retry failed: \`npx playwright test --last-failed --reporter=list\`
+
+**NEVER use:**
+- \`--ui\` flag (opens interactive UI that blocks)
+- \`--reporter=html\` without \`--reporter=list\` (may open server)
+- Commands without \`--reporter=list\` in CI/headless mode
+
+**Process management:**
+- Always use \`--reporter=list\` or \`--reporter=dot\` for clean output
+- Tests should exit automatically after completion
+- If a process hangs, kill it and retry with correct flags
+</playwright_execution>
+
+<debugging_with_mcp>
+When tests fail, use Playwright MCP tools to investigate:
+
+1. **Navigate**: Use \`mcp__playwright__playwright_navigate\` to load the failing page
+2. **Inspect DOM**: Use \`mcp__playwright__playwright_get_visible_html\` to see actual elements
+3. **Screenshot**: Use \`mcp__playwright__playwright_screenshot\` to capture current state
+4. **Console logs**: Use \`mcp__playwright__playwright_console_logs\` to check for JS errors
+5. **Interact**: Use click/fill tools to manually reproduce the flow
+
+**Workflow**: Navigate → inspect HTML → verify selectors → check console → fix
+</debugging_with_mcp>
+
+<flakiness_detection>
+After fixing, run the test 2-3 times. Watch for:
+
+- **Inconsistent results**: Passes sometimes, fails others
+- **Timing sensitivity**: Fails on slow runs, passes on fast
+- **Order dependence**: Fails when run with other tests
+- **Data coupling**: Relies on specific database state
+
+Common flakiness causes:
+- Arbitrary delays instead of condition waits
+- Shared state between tests
+- Hardcoded IDs or timestamps
+- Missing \`await\` on async operations
+- Race conditions in UI interactions
+</flakiness_detection>
+
+<fixing_patterns>
+**Selectors** - Prefer resilient locators:
+\`\`\`typescript
+// Good
+page.getByRole('button', { name: 'Submit' })
+page.getByTestId('submit-btn')
+
+// Avoid
+page.locator('.btn-primary')
+page.locator('div > button:nth-child(2)')
+\`\`\`
+
+**Timing** - Use condition-based waits, not arbitrary delays:
+\`\`\`typescript
+// Good
+await expect(element).toBeVisible({ timeout: 10_000 })
+
+// Avoid
+await page.waitForTimeout(5000)
+\`\`\`
+</fixing_patterns>
+
+<decision_gates>
+**Keep iterating if:**
+- You haven't tried 3 attempts yet
+- You have a new hypothesis to test
+- The error message changed (progress)
+
+**Escalate if:**
+- 3 attempts failed with no progress
+- Test identifies an actual app bug (don't mask bugs)
+- Test is fundamentally flaky by design
+- Requirements are ambiguous
+
+When escalating, report what you tried and why it didn't work.
+</decision_gates>
+
+<avoid>
+- Hard-coding values to make specific tests pass
+- Removing or skipping tests without understanding why they fail
+- Over-mocking that hides real integration issues
+- Making tests pass by weakening assertions
+- Introducing flakiness through timing-dependent fixes
+</avoid>
+
+<report_format>
+When reporting findings, use this structure:
+
+**Status**: fixed | escalated | in-progress
+**Test**: [test file and name]
+**Root Cause**: [Category] - [Specific cause]
+**Fix**: [What you changed]
+**Verification**: [N] runs, [all passed / some failed]
+**Flakiness Risk**: [none | low | medium | high] - [reason]
+
+Summarize final status: X/Y tests passing
+</report_format>`;

package/dist/prompts/index.js

@@ -0,0 +1,3 @@
+export { builderPrompt } from "./builder.js";
+export { fixerPrompt } from "./fixer.js";
+export { plannerPrompt } from "./planner.js";

package/dist/prompts/planner.js

@@ -0,0 +1,70 @@
+export const plannerPrompt = `<role>
+You are an E2E Test Planning Agent. Your job is to analyze applications, research user flows, and create detailed E2E test plans WITHOUT writing any code or making changes.
+</role>
+
+<core_principle>
+**Code Answers Everything.** Before asking ANY question:
+1. Search for the relevant component/API
+2. Read the implementation
+3. Check conditionals, handlers, and data flow
+
+Only ask about undefined business logic or incomplete implementations (TODOs).
+Never ask about routing, data scope, UI interactions, empty states, or error handling - these are in the code.
+</core_principle>
+
+<planning_focus>
+When planning E2E tests:
+1. Understand the application's user interface and user flows
+2. Identify critical user journeys that need test coverage
+3. Map out test scenarios with clear steps and expected outcomes
+4. Consider edge cases, error states, and boundary conditions
+5. Identify selectors and locators needed for each element
+6. Note any test data requirements or setup needed
+</planning_focus>
+
+<test_coverage>
+**Distribution target:**
+- 70% Happy paths - Standard successful flows
+- 20% Error paths - Validation failures, error states
+- 10% Edge cases - Boundaries, empty states, limits
+
+**What to cover:**
+- Critical user journeys (signup, login, checkout, core features)
+- Business-critical features (payments, data integrity)
+- High-risk areas (complex logic, integrations)
+- Error handling and validation
+
+**What NOT to cover:**
+- Every edge case permutation
+- Implementation details
+- Trivial functionality
+- Same pattern repeated across features
+</test_coverage>
+
+<analysis_tasks>
+- Explore the application structure and routes
+- Identify key UI components and their interactions
+- Map user authentication and authorization flows
+- Document form validations and error handling
+- Identify async operations that need proper waits
+- Note any third-party integrations or API dependencies
+</analysis_tasks>
+
+<plan_output>
+Your E2E test plan should include:
+1. Test suite overview - what user flows are being tested
+2. Test cases with clear descriptions and priority levels
+3. Step-by-step test actions (click, type, navigate, assert)
+4. Expected outcomes and assertions for each test
+5. Test data requirements (users, fixtures, mock data)
+6. Selector strategy (data-testid, aria-labels, etc.)
+7. Setup and teardown requirements
+8. Potential flakiness risks and mitigation strategies
+</plan_output>
+
+<constraints>
+- You can ONLY use read-only tools: Read, Glob, Grep, Task
+- Do NOT write tests, modify files, or run commands
+- Focus on research and planning, not implementation
+- Present findings clearly so the user can review before writing tests
+</constraints>`;

package/dist/services/api-client.js

@@ -0,0 +1,244 @@
+import { logger } from "../utils/logger";
+/**
+ * API Error class with user-friendly messages
+ *
+ * Based on Gemini CLI (Apache 2.0 License)
+ * https://github.com/google-gemini/gemini-cli
+ * Copyright 2025 Google LLC
+ */
+export class ApiError extends Error {
+    status;
+    isAuthError;
+    constructor(status, statusText, body) {
+        let message;
+        if (status === 401) {
+            message = "Authentication required. Use /login to authenticate.";
+        }
+        else if (status === 403) {
+            message = "Access denied. Your token may have been revoked.";
+        }
+        else {
+            message = `API error: ${status} ${statusText}`;
+            if (body) {
+                // Try to extract a cleaner error message from JSON body
+                try {
+                    const parsed = JSON.parse(body);
+                    if (parsed.error) {
+                        message = parsed.error;
+                    }
+                    else if (parsed.message) {
+                        message = parsed.message;
+                    }
+                }
+                catch {
+                    // If not JSON, append raw body if short
+                    if (body.length < 200) {
+                        message += ` - ${body}`;
+                    }
+                }
+            }
+        }
+        super(message);
+        this.name = "ApiError";
+        this.status = status;
+        this.isAuthError = status === 401 || status === 403;
+    }
+}
+/**
+ * API Client for CLI to communicate with Supatest backend
+ */
+export class ApiClient {
+    apiUrl;
+    apiKey;
+    constructor(apiUrl, apiKey) {
+        this.apiUrl = apiUrl;
+        this.apiKey = apiKey;
+    }
+    /**
+     * Update the API key (used when user logs in during session)
+     */
+    setApiKey(apiKey) {
+        this.apiKey = apiKey;
+    }
+    /**
+     * Clear the API key (used when user logs out)
+     */
+    clearApiKey() {
+        this.apiKey = undefined;
+    }
+    /**
+     * Check if the client has an API key set
+     */
+    hasApiKey() {
+        return !!this.apiKey;
+    }
+    /**
+     * Create a new CLI session on the backend
+     * @param title - The session title
+     * @param originMetadata - Optional metadata about the session origin
+     * @returns The session ID and web URL
+     */
+    async createSession(title, originMetadata) {
+        const url = `${this.apiUrl}/v1/agent/sessions`;
+        const response = await fetch(url, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${this.apiKey}`,
+            },
+            body: JSON.stringify({
+                title,
+                originMetadata,
+            }),
+        });
+        if (!response.ok) {
+            const errorText = await response.text();
+            throw new ApiError(response.status, response.statusText, errorText);
+        }
+        const data = await response.json();
+        return data;
+    }
+    /**
+     * Stream an event to the backend
+     * @param sessionId - The session ID
+     * @param event - The CLI event to stream
+     * @returns Success status
+     */
+    async streamEvent(sessionId, event) {
+        const url = `${this.apiUrl}/v1/agent/sessions/${sessionId}/events`;
+        const response = await fetch(url, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${this.apiKey}`,
+            },
+            body: JSON.stringify(event),
+        });
+        if (!response.ok) {
+            const errorText = await response.text();
+            throw new ApiError(response.status, response.statusText, errorText);
+        }
+        const data = await response.json();
+        return data;
+    }
+    /**
+     * Get session details from the backend
+     * @param sessionId - The session ID
+     * @returns Session details
+     */
+    async getSession(sessionId) {
+        const url = `${this.apiUrl}/v1/agent/sessions/${sessionId}`;
+        const response = await fetch(url, {
+            method: "GET",
+            headers: {
+                Authorization: `Bearer ${this.apiKey}`,
+            },
+        });
+        if (!response.ok) {
+            const errorText = await response.text();
+            throw new ApiError(response.status, response.statusText, errorText);
+        }
+        return await response.json();
+    }
+    /**
+     * Get paginated sessions accessible to the user
+     * @param limit - Maximum number of sessions to return
+     * @param offset - Number of sessions to skip
+     * @returns Paginated sessions with total count
+     */
+    async getSessions(limit, offset) {
+        const urlParams = new URLSearchParams({
+            limit: limit.toString(),
+            offset: offset.toString(),
+        });
+        const url = `${this.apiUrl}/v1/sessions?${urlParams.toString()}`;
+        logger.debug(`Fetching sessions: ${url}`);
+        const response = await fetch(url, {
+            method: "GET",
+            headers: {
+                Authorization: `Bearer ${this.apiKey}`,
+            },
+        });
+        if (!response.ok) {
+            const errorText = await response.text();
+            throw new ApiError(response.status, response.statusText, errorText);
+        }
+        const data = await response.json();
+        logger.debug(`Fetched ${data.sessions.length} sessions (${data.pagination.offset + 1}-${data.pagination.offset + data.sessions.length} of ${data.pagination.total})`);
+        return data;
+    }
+    /**
+     * Complete usage tracking for a message turn
+     * @param messageId - The assistant message ID
+     * @returns Success status
+     */
+    async completeUsage(messageId) {
+        const url = `${this.apiUrl}/v1/usage/complete`;
+        const response = await fetch(url, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${this.apiKey}`,
+            },
+            body: JSON.stringify({ messageId }),
+        });
+        if (!response.ok) {
+            const errorText = await response.text();
+            logger.warn(`Failed to complete usage tracking: ${response.status} ${response.statusText} - ${errorText}`);
+            // Don't throw - this is best effort
+            return { success: false };
+        }
+        const data = await response.json();
+        return data;
+    }
+    /**
+     * Get messages for a session
+     * @param sessionId - The session ID
+     * @returns Messages with pagination info
+     */
+    async getSessionMessages(sessionId) {
+        const url = `${this.apiUrl}/v1/sessions/${sessionId}/messages`;
+        logger.debug(`Fetching messages for session: ${sessionId}`);
+        const response = await fetch(url, {
+            method: "GET",
+            headers: {
+                Authorization: `Bearer ${this.apiKey}`,
+            },
+        });
+        if (!response.ok) {
+            const errorText = await response.text();
+            throw new ApiError(response.status, response.statusText, errorText);
+        }
+        const data = await response.json();
+        logger.debug(`Fetched ${data.messages.length} messages for session: ${sessionId}`);
+        return data;
+    }
+    /**
+     * Update a session (title, providerSessionId)
+     * @param sessionId - The session ID
+     * @param data - The data to update
+     * @returns Updated session
+     */
+    async updateSession(sessionId, data) {
+        const url = `${this.apiUrl}/v1/sessions/${sessionId}`;
+        logger.debug(`Updating session: ${sessionId}`, {
+            hasTitle: !!data.title,
+            hasProviderSessionId: !!data.providerSessionId,
+        });
+        const response = await fetch(url, {
+            method: "PATCH",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${this.apiKey}`,
+            },
+            body: JSON.stringify(data),
+        });
+        if (!response.ok) {
+            const errorText = await response.text();
+            throw new ApiError(response.status, response.statusText, errorText);
+        }
+        const result = await response.json();
+        logger.debug(`Session updated: ${sessionId}`);
+        return result;
+    }
+}

package/dist/services/event-streamer.js

@@ -0,0 +1,130 @@
+import { logger } from "../utils/logger";
+const BATCH_SIZE = 10; // Send events every 10 events
+const BATCH_INTERVAL_MS = 100; // Or every 100ms, whichever comes first
+const MAX_RETRY_ATTEMPTS = 3;
+const RETRY_DELAY_MS = 1000; // Start with 1s delay
+/**
+ * Event Streamer - Batches and streams CLI events to the backend with retry logic
+ */
+export class EventStreamer {
+    apiClient;
+    sessionId;
+    eventQueue = [];
+    batchTimer = null;
+    isFlushing = false;
+    isShutdown = false;
+    constructor(apiClient, sessionId) {
+        this.apiClient = apiClient;
+        this.sessionId = sessionId;
+    }
+    /**
+     * Queue an event for streaming
+     * @param event - The CLI event to queue
+     */
+    async queueEvent(event) {
+        if (this.isShutdown) {
+            logger.warn("EventStreamer is shutdown, cannot queue event");
+            return;
+        }
+        // Send streaming delta events immediately for real-time updates
+        // Batch other events to reduce API calls
+        if (event.type === "assistant_text" || event.type === "assistant_thinking") {
+            await this.sendEventWithRetry(event);
+            return;
+        }
+        this.eventQueue.push(event);
+        // Flush if we've reached batch size
+        if (this.eventQueue.length >= BATCH_SIZE) {
+            await this.flush();
+        }
+        else {
+            // Reset the batch timer
+            this.resetBatchTimer();
+        }
+    }
+    /**
+     * Reset the batch timer
+     */
+    resetBatchTimer() {
+        if (this.batchTimer) {
+            clearTimeout(this.batchTimer);
+        }
+        this.batchTimer = setTimeout(() => {
+            this.flush().catch((error) => {
+                logger.error(`Failed to flush events on timer: ${error.message}`);
+            });
+        }, BATCH_INTERVAL_MS);
+    }
+    /**
+     * Flush all queued events to the backend
+     */
+    async flush() {
+        if (this.isFlushing || this.eventQueue.length === 0) {
+            return;
+        }
+        this.isFlushing = true;
+        // Clear the batch timer
+        if (this.batchTimer) {
+            clearTimeout(this.batchTimer);
+            this.batchTimer = null;
+        }
+        // Take all events from the queue
+        const eventsToSend = [...this.eventQueue];
+        this.eventQueue = [];
+        // Send each event with retry logic
+        for (const event of eventsToSend) {
+            await this.sendEventWithRetry(event);
+        }
+        this.isFlushing = false;
+    }
+    /**
+     * Send a single event with retry logic
+     * @param event - The event to send
+     */
+    async sendEventWithRetry(event) {
+        let attempts = 0;
+        let lastError = null;
+        while (attempts < MAX_RETRY_ATTEMPTS) {
+            try {
+                await this.apiClient.streamEvent(this.sessionId, event);
+                return; // Success
+            }
+            catch (error) {
+                lastError = error;
+                attempts++;
+                if (attempts < MAX_RETRY_ATTEMPTS) {
+                    const delay = RETRY_DELAY_MS * Math.pow(2, attempts - 1); // Exponential backoff
+                    logger.warn(`Failed to stream event (attempt ${attempts}/${MAX_RETRY_ATTEMPTS}), retrying in ${delay}ms...`);
+                    await this.sleep(delay);
+                }
+            }
+        }
+        // All retries failed
+        logger.error(`Failed to stream event after ${MAX_RETRY_ATTEMPTS} attempts: ${lastError?.message}`);
+        // Don't throw - continue processing other events
+        // The event is lost, but we don't want to crash the CLI
+    }
+    /**
+     * Sleep for a given duration
+     * @param ms - Duration in milliseconds
+     */
+    sleep(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+    /**
+     * Shutdown the event streamer and flush any remaining events
+     */
+    async shutdown() {
+        if (this.isShutdown) {
+            return;
+        }
+        this.isShutdown = true;
+        // Clear the batch timer
+        if (this.batchTimer) {
+            clearTimeout(this.batchTimer);
+            this.batchTimer = null;
+        }
+        // Flush any remaining events
+        await this.flush();
+    }
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};