@zibby/core 0.1.0

package/templates/browser-test-automation/nodes/execute-live.js

@@ -0,0 +1,250 @@
+/**
+ * Execute Live Node
+ * 
+ * Purpose: Execute test in live browser using MCP Playwright tools
+ * 
+ * Configuration:
+ * - capabilities: Declares ['browser'] — framework injects the appropriate MCP server
+ * - outputSchema: Structured JSON with execution results, actions, assertions
+ * - Model: Configured in .zibby.config.js → agent.claude.model or agent.cursor.model
+ */
+
+import { z, SKILLS } from '@zibby/core';
+import { formatAssertionChecklist } from './utils.js';
+
+export const executeLiveNode = {
+  name: 'execute_live',
+  skills: [SKILLS.BROWSER, SKILLS.MEMORY],
+  timeout: 600000,
+
+  prompt: (state) => {
+    const ctx = state.context;
+    const contextInfo = ctx ? `
+Domain Knowledge & Environment:
+${ctx.global || ''}
+${ctx.pathBased ? `Test-Specific Info:\n${ctx.pathBased}\n` : ''}
+${ctx.env ? `Environment Config:\n${JSON.stringify(ctx.env, null, 2)}\n` : ''}
+---
+` : '';
+
+    const assertionChecklist = formatAssertionChecklist(state.preflight?.assertions);
+    
+    return `⚠️  CRITICAL: At the END, output ONLY the JSON object. NO explanations after the JSON.
+
+Execute this test using ONLY mcp_playwright-official_browser_* tools.
+
+🚨 HONESTY REQUIREMENT (STRICT):
+- If you DO NOT have access to browser tools → return {"success": false, "steps": [], "browserClosed": false, "notes": "No browser tools available"}
+- DO NOT hallucinate or pretend you executed the test
+- DO NOT return success: true unless you ACTUALLY called browser tools
+- BE HONEST - it's better to admit you can't do it than to lie
+
+🎯 YOUR GOAL: Execute the test steps and collect evidence for script generation.
+   You don't need perfect verification - just capture the key actions and results.
+   The next node will generate the actual test script from your execution.
+
+${contextInfo}
+${state.testSpec}
+${assertionChecklist ? `
+═══════════════════════════════════════════════════
+🎯 ASSERTION CHECKLIST (MANDATORY - from test spec)
+You MUST include ALL of these in your 'assertions' array.
+Report each as passed: true or passed: false with evidence.
+DO NOT skip any. DO NOT add extras.
+
+${assertionChecklist}
+═══════════════════════════════════════════════════
+` : ''}
+⚠️  CRITICAL RULES (STRICT ENFORCEMENT):
+1. DO NOT get stuck in read loops - if a snapshot is large, move on
+2. DO NOT over-analyze - just execute the steps
+3. **As soon as you complete the test → IMMEDIATELY return JSON**
+4. **NO screenshots required** - just execute and return JSON
+5. **If test is done, STOP - don't try to be perfect**
+6. **USE VALUES FROM THE TEST SPEC** - if the spec provides specific values, use them exactly. Do NOT replace them with random data.
+7. **USE UNIQUE DATA ONLY when CREATING new resources** (e.g., sign-up forms, new accounts) to avoid "already taken" conflicts:
+   - For NEW emails (not provided in test spec): use random digits like "test84729@example.com"
+   - For NEW names (not provided in test spec): append random digits like "John84729"
+   - This does NOT apply to login credentials or test data explicitly provided in the spec
+
+WHEN TO STOP (MANDATORY):
+✓ You've completed the test steps
+✓ Test outcome is visible (even briefly)
+→ **RETURN JSON IMMEDIATELY - DO NOT make any more tool calls**
+
+DO NOT:
+- Navigate to the same URL multiple times
+- Use browser_run_code or browser_evaluate
+- Read large snapshots repeatedly (max 2 snapshots per page)
+- Take screenshots (optional, skip if slowing you down)
+- Try to verify every single detail - focus on the MAIN outcome
+- Click/scroll/interact after seeing the expected result
+- Spend more than 2 minutes on any single page
+- Try to click elements that aren't immediately visible
+
+EXECUTION SEQUENCE (MANDATORY - FOLLOW STRICTLY):
+1. Execute the test steps efficiently (navigate, fill, click)
+   - Max 10-15 actions total
+   - If stuck, move on to next step
+2. Quick verification - check if main result is visible
+   - **If you see expected result → IMMEDIATELY return JSON**
+   - Don't try to make it perfect - good enough is enough
+3. **RETURN JSON AND STOP COMPLETELY**
+   - Format: { "success": true, "steps": ["step 1", ...], "browserClosed": true, "actions": [...] }
+   - MUST include: "success" (boolean), "steps" (array), "browserClosed" (boolean)
+   - Keep JSON CONCISE - short descriptions, no excessive detail
+   - After the closing brace }, DO NOT write ANYTHING
+   - NO commentary, NO explanations, NO additional text
+   - NO second JSON object
+   - Just the JSON, then STOP
+
+IMPORTANT for 'actions' array (STRICT 1:1 MAPPING):
+- Each entry MUST match EXACTLY ONE browser tool call.
+- DO NOT group multiple tool calls into one action.
+- DO NOT combine multiple 'fill' calls into one action.
+- If you call browser_type 3 times for 3 fields, you MUST have 3 actions in the array.
+- Include actual values/URLs in descriptions.
+- Keep descriptions SHORT (5-10 words max).
+
+IMPORTANT for 'assertions' array (USE THE CHECKLIST ABOVE):
+- Your assertions array MUST match the ASSERTION CHECKLIST exactly - one entry per item
+- If you verified it and it passed → "passed": true
+- If you could NOT verify it or it wasn't found → "passed": false with evidence of what you saw instead
+- Each assertion MUST include 'verifiedAfterAction' (0-based action index after which you checked)
+- Format: {"description": "...", "passed": true/false, "verifiedAfterAction": N, "evidence": "..."}
+
+🔍 CRITICAL: CAPTURE ROBUST SELECTORS (for script generation)
+For EACH action, capture multiple selector strategies in priority order:
+
+1. **Role + Name** (Most Robust - Accessibility-first):
+   - Role: button/textbox/link/etc
+   - Name: visible text or aria-label
+   Example: {"role": "button", "name": "Login"}
+
+2. **Stable Attributes** (Good Stability):
+   - name, type, placeholder, aria-label
+   Example: {"attributes": {"name": "username", "type": "text", "placeholder": "Enter username"}}
+
+3. **Partial Match** (For Dynamic Elements):
+   - Use starts-with for dynamic IDs/classes
+   Example: {"partialMatch": {"id": "^user-", "class": "^btn-"}}
+
+4. **Structural** (Fallback):
+   - Tag + position relative to stable landmark
+   Example: {"structure": "form input[type='text']:nth-of-type(1)"}
+
+Format for actions with selectors:
+{
+  "description": "Fill username field with 'joe'",
+  "reasoning": "Need to authenticate user",
+  "type": "fill",
+  "selectors": {
+    "role": {"role": "textbox", "name": "Username"},
+    "attributes": {"name": "username", "type": "text", "placeholder": "请输入账号"},
+    "structure": "form input[type='text']:first-of-type"
+  },
+  "value": "joe"
+}
+
+IMPORTANT for 'evidenceScreenshots' (array) - OPTIONAL:
+- Screenshots are OPTIONAL - only take if helpful
+- If you take screenshots, use descriptive filenames
+- Filename pattern: "{step-number}-{action-or-state}.png"
+- Keep it minimal - test execution is more important than documentation
+
+════════════════════════════════════════════════════════════
+🚨 CRITICAL JSON OUTPUT RULES 🚨
+
+YOU MUST OUTPUT JSON USING ONE OF THESE TWO FORMATS:
+
+✅ FORMAT 1 (BEST - Use This!):
+Think/plan/explain first, THEN output ONLY JSON:
+
+I'll navigate to the login page and fill the form...
+[... use browser tools ...]
+[... complete test execution ...]
+
+{"success": true, "steps": [...], "browserClosed": true}
+
+NO TEXT AFTER THE JSON! Stop immediately after }.
+
+✅ FORMAT 2 (If you need to explain after):
+Use delimiters to separate JSON from explanations:
+
+I'm executing the test now...
+[... use browser tools ...]
+
+===JSON_START===
+{"success": true, "steps": [...], "browserClosed": true}
+===JSON_END===
+
+Now let me explain what happened...
+
+❌ WRONG - DO NOT DO THIS:
+{"success": true, "steps": [...]} followed by more explanations
+
+❌ WRONG - DO NOT STREAM JSON LETTER BY LETTER:
+The test completed successfully, here's the result: { "success": t
+
+✅ CORRECT - Output complete JSON in one block:
+{"success": true, "steps": ["step 1", "step 2"], "browserClosed": true}
+
+REMEMBER: After the final }, you MUST STOP or use ===JSON_END===
+════════════════════════════════════════════════════════════
+`;
+  },
+  
+  outputSchema: z.object({
+    success: z.boolean()
+      .describe('Whether the test execution completed successfully'),
+    
+    steps: z.array(z.string())
+      .describe('Array of test steps executed'),
+    
+    finalUrl: z.string()
+      .optional()
+      .describe('Final URL after test execution'),
+    
+    actions: z.array(z.any())
+      .optional()
+      .describe('Detailed array of actions performed with descriptions and reasoning'),
+    
+    assertions: z.array(z.object({
+      description: z.string()
+        .describe('What was verified'),
+      passed: z.boolean()
+        .describe('Whether the assertion passed'),
+      verifiedAfterAction: z.number()
+        .describe('Index of the action after which this was verified (0-based, matches actions array index) - REQUIRED'),
+      evidence: z.string()
+        .optional()
+        .describe('Brief evidence of what was observed')
+    }))
+      .optional()
+      .describe('Array of assertions made during test'),
+    
+    waits: z.array(z.any())
+      .optional()
+      .describe('Array of waits needed for proper test execution'),
+    
+    evidenceScreenshots: z.array(z.object({
+      filename: z.string()
+        .describe('Descriptive filename pattern: {step-number}-{action-or-state}.png'),
+      
+      description: z.string()
+        .describe('What the screenshot shows and why it is evidence'),
+      
+      verdict: z.enum(['pass', 'fail', 'info'])
+        .describe('Test verdict: pass/fail for validation points, info for checkpoints')
+    }))
+      .optional()
+      .describe('Array of screenshots taken at key validation points throughout the test'),
+    
+    browserClosed: z.boolean()
+      .describe('Whether the browser was properly closed (should always be true)'),
+    
+    notes: z.string()
+      .optional()
+      .describe('Additional notes or observations. REQUIRED when success=false to explain why test failed or could not execute')
+  })
+};

package/templates/browser-test-automation/nodes/generate-script.js

@@ -0,0 +1,77 @@
+import { z } from '@zibby/core';
+import { formatRecordedActions, formatAssertionsWithResults } from './utils.js';
+
+const GenerateScriptOutputSchema = z.object({
+  success: z.boolean(),
+  scriptPath: z.string(),
+  method: z.string()
+});
+
+export const generateScriptNode = {
+  name: 'generate_script',
+  outputSchema: GenerateScriptOutputSchema,
+  timeout: 360000,
+
+  prompt: (state) => {
+    const exec = state.execute_live || {};
+    const preflight = state.preflight || {};
+
+    const actionsBlock = formatRecordedActions(state.sessionPath, exec.actions);
+    const assertionsBlock = formatAssertionsWithResults(
+      preflight.assertions,
+      exec.assertions,
+      exec.notes,
+      exec.finalUrl
+    );
+
+    return `Generate and verify Playwright test at ${state.outputPath}
+
+Test Spec:
+${state.testSpec}
+
+Live Execution Summary:
+- Success: ${exec.success}
+- Steps: ${JSON.stringify(exec.steps)}
+- Final URL: ${exec.finalUrl || 'unknown'}
+${actionsBlock}
+${assertionsBlock}
+
+IMPORTS AND PATTERN:
+\`\`\`javascript
+import { test, expect } from '@playwright/test';
+import { StableIdRuntime } from '@zibby/core';
+
+test('Test Name', async ({ page }) => {
+  await page.goto('https://...');
+  await StableIdRuntime.injectStableIds(page);
+  // Elements WITH stable IDs — use StableIdRuntime:
+  await StableIdRuntime.fillWithRetry(page, 'zibby-xxxxx', 'value');
+  await StableIdRuntime.clickWithRetry(page, 'zibby-xxxxx');
+  // Elements WITHOUT stable IDs (NO_STABLE_ID) — use native Playwright selectors:
+  await page.getByText('visible text').click();
+  await page.getByRole('button', { name: 'Submit' }).click();
+  await page.getByPlaceholder('placeholder text').fill('value');
+  await expect(page).toHaveURL(/expected-url/);
+});
+\`\`\`
+
+RULES:
+1. First navigate → page.goto(), skip subsequent navigates
+2. After goto, call StableIdRuntime.injectStableIds(page)
+3. Use EXACT stable IDs from recorded actions when available
+4. For [NO_STABLE_ID] actions, use the fallback selector (getByText, getByRole, getByPlaceholder). These are typically non-semantic elements like spans acting as buttons — use the visible text to target them.
+5. Skip duplicate consecutive clicks on same stableId
+6. No comments in generated code
+7. Implement ALL assertions from the list above
+8. If an assertion fails after retries, comment it out with a TODO (don't delete it)
+
+WORKFLOW:
+1. Write test to ${state.outputPath}
+2. Run: PLAYWRIGHT_HEADLESS=1 npx playwright test ${state.outputPath} --reporter=line --timeout=30000
+3. If fails: make ONE targeted fix (longer timeout, different selector, short wait)
+4. MAX 2 ATTEMPTS then STOP
+
+The test runs in: ${state.cwd || 'project root'}
+`;
+  },
+};

package/templates/browser-test-automation/nodes/index.js

@@ -0,0 +1,3 @@
+export { preflightNode } from './preflight.js';
+export { executeLiveNode } from './execute-live.js';
+export { generateScriptNode } from './generate-script.js';

package/templates/browser-test-automation/nodes/preflight.js

@@ -0,0 +1,59 @@
+/**
+ * Preflight Node
+ * 
+ * Pattern: Prompt-only node (no tools)
+ * Purpose: Analyze test spec and extract title + structured assertion checklist
+ * 
+ * This runs before execution to define:
+ * - A concise test title
+ * - The complete list of assertions that must be verified
+ * 
+ * Downstream nodes receive this as their contract:
+ * - execute_live: must report passed/failed for each assertion
+ * - generate_script: must implement each assertion in the test
+ */
+
+import { z } from '@zibby/core';
+import { writeFileSync } from 'fs';
+import { join } from 'path';
+
+const AssertionSchema = z.object({
+  description: z.string().describe('What to verify (e.g., "User is redirected to dashboard")'),
+  expected: z.string().describe('What the expected outcome looks like (e.g., "URL contains /dashboard")')
+});
+
+const PreflightOutputSchema = z.object({
+  title: z.string().describe('Concise test title (5-10 words, action-oriented). Prefix with ticket ID if found.'),
+  assertions: z.array(AssertionSchema).describe('Every expected result from the spec as a verifiable assertion')
+});
+
+export const preflightNode = {
+  name: 'preflight',
+
+  async onComplete(state, result) {
+    const sessionPath = state.sessionPath || process.env.ZIBBY_SESSION_PATH;
+    if (sessionPath && result.title) {
+      try {
+        writeFileSync(join(sessionPath, 'title.txt'), result.title, 'utf-8');
+        console.log(`Saved title: "${result.title}"`);
+      } catch (error) {
+        console.warn(`⚠️  Could not save title.txt: ${error.message}`);
+      }
+    }
+    return result;
+  },
+
+  prompt: (state) => `Analyze this test specification and extract:
+1. A concise test title (5-10 words, action-oriented). If you find a ticket ID (e.g., PROJ-123, ACME-456), prefix the title with it.
+2. Every expected result as a verifiable assertion. Each assertion must be something the browser can check after execution.
+
+Test Spec:
+${state.testSpec}
+
+IMPORTANT: You MUST create ONE assertion for EACH expected result in the spec. Do NOT skip any.
+
+Return ONLY this JSON:
+{ "title": "TICKET-ID: Short action title", "assertions": [ { "description": "...", "expected": "..." }, ... ] }`,
+
+  outputSchema: PreflightOutputSchema
+};

package/templates/browser-test-automation/nodes/utils.js

@@ -0,0 +1,154 @@
+import { readFileSync, existsSync } from 'fs';
+
+const ACTIONABLE_EVENTS = ['navigate', 'type', 'fill', 'click', 'select_option', 'select'];
+
+/**
+ * Load recorded browser events from a session's events.json,
+ * filtering to only actionable events (navigate, fill, click, etc.)
+ */
+export function loadRecordedActions(sessionPath, nodeName = 'execute_live') {
+  const eventsPath = `${sessionPath}/${nodeName}/events.json`;
+  if (!existsSync(eventsPath)) return [];
+  try {
+    return JSON.parse(readFileSync(eventsPath, 'utf-8'))
+      .filter(e => ACTIONABLE_EVENTS.includes(e.type))
+      .map(e => ({
+        type: e.type,
+        stableId: e.stableId || e.data?.stableId,
+        value: e.data?.text || e.data?.params?.text || e.data?.url || e.data?.params?.url || e.data?.values?.[0] || e.data?.params?.values?.[0],
+        element: e.data?.element || e.data?.params?.element,
+        reasoning: e.reasoning || e.description
+      }));
+  } catch (e) {
+    console.error('Failed to read events:', e.message);
+    return [];
+  }
+}
+
+/**
+ * Index AI-captured selectors by element description (lowercase)
+ * for fuzzy matching when building fallback selectors.
+ */
+export function buildSelectorMap(actions) {
+  const map = {};
+  for (const a of actions || []) {
+    if (a.selectors?.role) {
+      map[(a.description || '').toLowerCase()] = a.selectors;
+    }
+  }
+  return map;
+}
+
+/** Extract meaningful tokens from a description string for fuzzy matching. */
+function extractTokens(str) {
+  return str.replace(/[-–—]/g, ' ').split(/\s+/).filter(t => t.length > 1);
+}
+
+/** Score how well two descriptions match by counting shared tokens. */
+function tokenOverlapScore(a, b) {
+  const tokensA = extractTokens(a.toLowerCase());
+  const tokensB = extractTokens(b.toLowerCase());
+  return tokensA.filter(t => tokensB.some(tb => tb.includes(t) || t.includes(tb))).length;
+}
+
+/** Find a fallback selector for an element by fuzzy-matching against the selector map. */
+function findFallbackSelector(elementKey, selectorMap) {
+  let bestMatch = null;
+  let bestScore = 0;
+  for (const [k, selectors] of Object.entries(selectorMap)) {
+    const score = tokenOverlapScore(elementKey, k);
+    if (score > bestScore) {
+      bestScore = score;
+      bestMatch = selectors;
+    }
+  }
+  if (!bestMatch?.role || bestScore < 1) return '';
+  const { role, name } = bestMatch.role;
+  if (role === 'generic' || role === 'none') {
+    return name ? ` | fallback: getByText('${name}')` : '';
+  }
+  return ` | fallback: getByRole('${role}'${name ? `, { name: '${name}' }` : ''})`;
+}
+
+/** Extract visible text hint from element description for use as a getByText fallback. */
+function extractTextHint(elementDesc) {
+  if (!elementDesc) return '';
+  const cnMatch = elementDesc.match(/[\u4e00-\u9fff\u3000-\u303f]+/g);
+  if (cnMatch) return cnMatch.join('');
+  const afterDash = elementDesc.split(/[-–—]\s*/).pop()?.trim();
+  if (afterDash && afterDash !== elementDesc) return afterDash;
+  return '';
+}
+
+/** Format a single recorded event into a human-readable action line with stableId and fallback selector. */
+export function formatAction(event, index, selectorMap) {
+  const num = index + 1;
+  const reason = event.reasoning ? ` | reason: "${event.reasoning}"` : '';
+  const hasStableId = event.stableId && event.stableId !== 'undefined';
+  const fallback = findFallbackSelector((event.element || '').toLowerCase(), selectorMap);
+  const idPart = hasStableId ? `[stableId="${event.stableId}"]` : '[NO_STABLE_ID]';
+
+  let textFallback = '';
+  if (!hasStableId && !fallback) {
+    const hint = extractTextHint(event.element);
+    if (hint) textFallback = ` | fallback: getByText('${hint}')`;
+  }
+
+  switch (event.type) {
+    case 'navigate':
+      return `${num}. NAVIGATE to: ${event.value}${reason}`;
+    case 'click':
+      return `${num}. CLICK ${idPart} - ${event.element || 'element'}${fallback || textFallback}${reason}`;
+    case 'fill': case 'type':
+      return `${num}. FILL ${idPart} with "${event.value}" - ${event.element || 'field'}${fallback || textFallback}${reason}`;
+    case 'select': case 'select_option':
+      return `${num}. SELECT ${idPart} option "${event.value}" - ${event.element || 'dropdown'}${fallback || textFallback}${reason}`;
+    default:
+      return `${num}. ${event.type} ${hasStableId ? idPart : ''} ${event.value || ''}${fallback || textFallback}${reason}`;
+  }
+}
+
+/**
+ * Build the full RECORDED ACTIONS prompt block by loading events.json
+ * and merging with AI-captured selectors from execution output.
+ */
+export function formatRecordedActions(sessionPath, executionActions) {
+  const recorded = loadRecordedActions(sessionPath);
+  if (recorded.length === 0) return '';
+  const selectorMap = buildSelectorMap(executionActions);
+  return `RECORDED ACTIONS:\n${recorded.map((e, i) => formatAction(e, i, selectorMap)).join('\n')}\n\nGenerate a CLEAN script from these — not a 1:1 replay. If stableId fails, use the fallback selector.`;
+}
+
+/** Format preflight assertions into a simple numbered checklist for the execute_live prompt. */
+export function formatAssertionChecklist(assertions) {
+  if (!assertions?.length) return '';
+  return assertions.map((a, i) =>
+    `${i + 1}. ${a.description} → expected: ${a.expected}`
+  ).join('\n');
+}
+
+/**
+ * Merge preflight assertion definitions with execution results (pass/fail/evidence)
+ * into an ASSERTIONS TO IMPLEMENT block for the generate_script prompt.
+ */
+export function formatAssertionsWithResults(preflightAssertions, executionAssertions, notes, finalUrl) {
+  if (!preflightAssertions?.length && !executionAssertions?.length) return '';
+
+  const lines = preflightAssertions?.length > 0
+    ? preflightAssertions.map((a, i) => {
+        const exec = executionAssertions?.[i];
+        const status = exec ? (exec.passed ? '✅ PASSED' : '❌ FAILED') : '⚠️ NOT CHECKED';
+        const evidence = exec?.evidence ? ` | evidence: "${exec.evidence}"` : '';
+        return `${i + 1}. [${status}] ${a.description} → expected: ${a.expected}${evidence}`;
+      })
+    : executionAssertions.map((a, i) =>
+        typeof a === 'object'
+          ? `${i + 1}. ${a.description || a.type}: expected "${a.expected}", actual "${a.actual}"`
+          : `${i + 1}. ${a}`
+      );
+
+  return `ASSERTIONS TO IMPLEMENT:
+${lines.join('\n')}
+
+Final URL: ${finalUrl || 'unknown'}${notes ? `\nAI OBSERVATION: ${notes}` : ''}`;
+}