@zibby/core 0.4.5 → 0.5.0

package/templates/browser-test-automation/nodes/execute-live.mjs

@@ -1,254 +0,0 @@
-/**
- * Execute Live Node
- * 
- * Purpose: Execute test in live browser using MCP Playwright tools
- * 
- * Configuration:
- * - capabilities: Declares ['browser'] — framework injects the appropriate MCP server
- * - outputSchema: Structured JSON with execution results, actions, assertions
- * - Model: Configured in .zibby.config.mjs → agent.claude.model or agent.cursor.model
- */
-
-import { z, SKILLS } from '@zibby/core';
-import { formatAssertionChecklist } from './utils.mjs';
-
-export const executeLiveNode = {
-  name: 'execute_live',
-  skills: [SKILLS.BROWSER, SKILLS.MEMORY],
-  timeout: 600000,
-
-  prompt: (state) => {
-    const ctx = state.context;
-    const contextInfo = ctx ? `
-Domain Knowledge & Environment:
-${ctx.global || ''}
-${ctx.pathBased ? `Test-Specific Info:\n${ctx.pathBased}\n` : ''}
-${ctx.env ? `Environment Config:\n${JSON.stringify(ctx.env, null, 2)}\n` : ''}
----
-` : '';
-
-    const assertionChecklist = formatAssertionChecklist(state.preflight?.assertions);
-    
-    return `⚠️  CRITICAL: At the END, output ONLY the JSON object. NO explanations after the JSON.
-
-🎯 YOUR GOAL: Execute the test steps and collect evidence for script generation.
-   You don't need perfect verification - just capture the key actions and results.
-   The next node will generate the actual test script from your execution.
-
-${contextInfo}
-${state.testSpec}
-${assertionChecklist ? `
-═══════════════════════════════════════════════════
-🎯 ASSERTION CHECKLIST (MANDATORY - from test spec)
-You MUST include ALL of these in your 'assertions' array.
-Report each as passed: true or passed: false with evidence.
-DO NOT skip any. DO NOT add extras.
-
-${assertionChecklist}
-═══════════════════════════════════════════════════
-` : ''}
-⚠️  CRITICAL RULES (STRICT ENFORCEMENT):
-1. DO NOT get stuck in read loops - if a snapshot is large, move on
-2. DO NOT over-analyze - just execute the steps
-3. **As soon as you complete the test → IMMEDIATELY return JSON**
-4. **NO screenshots required** - just execute and return JSON
-5. **If test is done, STOP - don't try to be perfect**
-6. **USE VALUES FROM THE TEST SPEC** - if the spec provides specific values, use them exactly. Do NOT replace them with random data.
-7. **USE UNIQUE DATA ONLY when CREATING new resources** (e.g., sign-up forms, new accounts) to avoid "already taken" conflicts:
-   - For NEW emails (not provided in test spec): use random digits like "test84729@example.com"
-   - For NEW names (not provided in test spec): append random digits like "John84729"
-   - This does NOT apply to login credentials or test data explicitly provided in the spec
-
-WHEN TO STOP (MANDATORY):
-✓ You've completed the test steps
-✓ Test outcome is visible (even briefly)
-→ **RETURN JSON IMMEDIATELY - DO NOT make any more tool calls**
-
-DO NOT:
-- Navigate to the same URL multiple times
-- Use browser_run_code or browser_evaluate
-- Read large snapshots repeatedly (max 2 snapshots per page)
-- Take screenshots (optional, skip if slowing you down)
-- Try to verify every single detail - focus on the MAIN outcome
-- Click/scroll/interact after seeing the expected result
-- Spend more than 2 minutes on any single page
-- Try to click elements that aren't immediately visible
-
-📋 BEFORE EXECUTING (MEMORY-AWARE START):
-If the "Domain Knowledge" section above contains a "### Known Pages on This Site"
-block with "expected fingerprint: zibby-..." entries for the URL you're about
-to land on, do this ONCE on first navigation:
-
-  1. Navigate to the page.
-  2. Call browser_snapshot just once to read the live DOM's stableIds.
-  3. Count how many of the expected fingerprint stableIds are present.
-  4. If ≥ 85% are present → MATCH: trust the cached selectors directly,
-     no further exploration on this page is needed.
-  5. If < 85% are present → MISMATCH: the page has drifted, ignore the
-     cached selectors and rediscover from the live snapshot.
-
-This costs ONE snapshot per page in exchange for skipping the rest. Don't
-skip this step on pages that have an expected fingerprint — it's the
-mechanism that makes repeat runs cheap.
-
-EXECUTION SEQUENCE (MANDATORY - FOLLOW STRICTLY):
-1. Execute the test steps efficiently (navigate, fill, click)
-   - Max 10-15 actions total
-   - If stuck, move on to next step
-2. Quick verification - check if main result is visible
-   - **If you see expected result → IMMEDIATELY return JSON**
-   - Don't try to make it perfect - good enough is enough
-3. **RETURN JSON AND STOP COMPLETELY**
-   - Format: { "success": true, "steps": ["step 1", ...], "browserClosed": true, "actions": [...] }
-   - MUST include: "success" (boolean), "steps" (array), "browserClosed" (boolean)
-   - Keep JSON CONCISE - short descriptions, no excessive detail
-   - After the closing brace }, DO NOT write ANYTHING
-   - NO commentary, NO explanations, NO additional text
-   - NO second JSON object
-   - Just the JSON, then STOP
-
-IMPORTANT for 'actions' array (STRICT 1:1 MAPPING):
-- Each entry MUST match EXACTLY ONE browser tool call.
-- DO NOT group multiple tool calls into one action.
-- DO NOT combine multiple 'fill' calls into one action.
-- If you call browser_type 3 times for 3 fields, you MUST have 3 actions in the array.
-- Include actual values/URLs in descriptions.
-- Keep descriptions SHORT (5-10 words max).
-- **'committed' field (REQUIRED on actions you DELIBERATELY chose):**
-  set committed: true when this action was your CHOSEN attempt for an
-  intent (the click/fill you actually wanted, regardless of outcome).
-  set committed: false (or omit) for exploratory probes you tried while
-  searching for the right element. Only committed actions feed the
-  negative cache, so probes don't pollute future runs' Avoid lists.
-  Example: tried selector A (probe, failed) → tried selector B
-  (deliberate, failed) → tried selector C (deliberate, succeeded). Mark
-  B and C as committed: true; A as committed: false.
-
-IMPORTANT for 'assertions' array (USE THE CHECKLIST ABOVE):
-- Your assertions array MUST match the ASSERTION CHECKLIST exactly - one entry per item
-- If you verified it and it passed → "passed": true
-- If you could NOT verify it or it wasn't found → "passed": false with evidence of what you saw instead
-- Each assertion MUST include 'verifiedAfterAction' (0-based action index after which you checked)
-- Format: {"description": "...", "passed": true/false, "verifiedAfterAction": N, "evidence": "..."}
-
-🔍 CRITICAL: CAPTURE ROBUST SELECTORS (for script generation)
-For EACH action, capture multiple selector strategies in priority order:
-
-1. **Role + Name** (Most Robust - Accessibility-first):
-   - Role: button/textbox/link/etc
-   - Name: visible text or aria-label
-   Example: {"role": "button", "name": "Login"}
-
-2. **Stable Attributes** (Good Stability):
-   - name, type, placeholder, aria-label
-   Example: {"attributes": {"name": "username", "type": "text", "placeholder": "Enter username"}}
-
-3. **Partial Match** (For Dynamic Elements):
-   - Use starts-with for dynamic IDs/classes
-   Example: {"partialMatch": {"id": "^user-", "class": "^btn-"}}
-
-4. **Structural** (Fallback):
-   - Tag + position relative to stable landmark
-   Example: {"structure": "form input[type='text']:nth-of-type(1)"}
-
-Format for actions with selectors:
-{
-  "description": "Fill username field with 'joe'",
-  "reasoning": "Need to authenticate user",
-  "type": "fill",
-  "selectors": {
-    "role": {"role": "textbox", "name": "Username"},
-    "attributes": {"name": "username", "type": "text", "placeholder": "请输入账号"},
-    "structure": "form input[type='text']:first-of-type"
-  },
-  "value": "joe"
-}
-
-IMPORTANT for 'evidenceScreenshots' (array) - OPTIONAL:
-- Screenshots are OPTIONAL - only take if helpful
-- If you take screenshots, use descriptive filenames
-- Filename pattern: "{step-number}-{action-or-state}.png"
-- Keep it minimal - test execution is more important than documentation
-`;
-  },
-  
-  outputSchema: z.object({
-    success: z.boolean()
-      .describe('Whether the test execution completed successfully'),
-    
-    steps: z.array(z.string())
-      .describe('Array of test steps executed'),
-    
-    finalUrl: z.string()
-      .nullish()
-      .describe('Final URL after test execution'),
-    
-    actions: z.array(z.object({
-      type: z.string()
-        .describe('Action type: navigate, click, fill, type, select, keypress, hover, drag'),
-      description: z.string()
-        .describe('Human-readable description of the action'),
-      reasoning: z.string().nullish()
-        .describe('Why this action was performed'),
-      committed: z.boolean().nullish()
-        .describe('true when this was a deliberate chosen attempt for an intent (feeds negative cache on failure); false/omit for exploratory probes'),
-      status: z.enum(['success', 'failed']).nullish()
-        .describe('Outcome of the action — set "failed" if the tool call errored or the post-condition was not met'),
-      error: z.string().nullish()
-        .describe('Error message when status=failed'),
-      selectors: z.object({
-        role: z.object({
-          role: z.string().describe('ARIA role (e.g. button, link, textbox, generic)'),
-          name: z.string().nullish().describe('Accessible name of the element')
-        }).nullish().describe('Role-based selector for fallback matching')
-      }).nullish()
-        .describe('Element selectors captured during the action'),
-      value: z.string().nullish()
-        .describe('Value entered for fill/type actions')
-    }))
-      .nullish()
-      .describe('Detailed array of actions performed with descriptions and reasoning'),
-    
-    assertions: z.array(z.object({
-      description: z.string()
-        .describe('What was verified'),
-      passed: z.boolean()
-        .describe('Whether the assertion passed'),
-      verifiedAfterAction: z.number()
-        .describe('Index of the action after which this was verified (0-based, matches actions array index) - REQUIRED'),
-      evidence: z.string()
-        .nullish()
-        .describe('Brief evidence of what was observed')
-    }))
-      .nullish()
-      .describe('Array of assertions made during test'),
-    
-    waits: z.array(z.object({
-      description: z.string().describe('What the wait is for'),
-      duration: z.number().nullish().describe('Wait duration in milliseconds'),
-      condition: z.string().nullish().describe('Wait condition expression')
-    }))
-      .nullish()
-      .describe('Array of waits needed for proper test execution'),
-    
-    evidenceScreenshots: z.array(z.object({
-      filename: z.string()
-        .describe('Descriptive filename pattern: {step-number}-{action-or-state}.png'),
-      
-      description: z.string()
-        .describe('What the screenshot shows and why it is evidence'),
-      
-      verdict: z.enum(['pass', 'fail', 'info'])
-        .describe('Test verdict: pass/fail for validation points, info for checkpoints')
-    }))
-      .nullish()
-      .describe('Array of screenshots taken at key validation points throughout the test'),
-    
-    browserClosed: z.boolean()
-      .describe('Whether the browser was properly closed (should always be true)'),
-    
-    notes: z.string()
-      .nullish()
-      .describe('Additional notes or observations. REQUIRED when success=false to explain why test failed or could not execute')
-  })
-};

package/templates/browser-test-automation/nodes/generate-script.mjs

@@ -1,108 +0,0 @@
-import { z, SKILLS } from '@zibby/core';
-import { formatRecordedActions, formatAssertionsWithResults, loadRecordedActions, detectLoginPattern, formatSetupHint } from './utils.mjs';
-
-const GenerateScriptOutputSchema = z.object({
-  success: z.boolean(),
-  scriptPath: z.string(),
-  method: z.string()
-});
-
-export const generateScriptNode = {
-  name: 'generate_script',
-  skills: [SKILLS.MEMORY],
-  outputSchema: GenerateScriptOutputSchema,
-  timeout: 1200000,
-
-  prompt: (state) => {
-    const exec = state.execute_live || {};
-    const preflight = state.preflight || {};
-
-    const actionsBlock = formatRecordedActions(state.sessionPath, exec.actions);
-    const assertionsBlock = formatAssertionsWithResults(
-      preflight.assertions,
-      exec.assertions,
-      exec.notes,
-      exec.finalUrl
-    );
-
-    const recorded = loadRecordedActions(state.sessionPath);
-    const setupHint = formatSetupHint(detectLoginPattern(recorded));
-
-    // `state.outputPath` is computed from `state.specPath` by the
-    // framework. For `zibby test <spec>` runs it's set. For
-    // `workflow trigger` runs there's no spec file on disk, so
-    // outputPath comes back undefined and the prompt rendered
-    // "Generate and verify Playwright test at undefined" — which
-    // caused the LLM to literally write to a file named `undefined`.
-    // Fall back to a path under the session dir so the test always
-    // lands somewhere sensible.
-    const outputPath = state.outputPath
-      || (state.sessionPath ? `${state.sessionPath}/generate_script/generated-test.spec.js` : 'tests/generated-test.spec.js');
-
-    return `Generate and verify Playwright test at ${outputPath}
-
-Test Spec:
-${state.testSpec}
-
-Live Execution Summary:
-- Success: ${exec.success}
-- Steps: ${JSON.stringify(exec.steps)}
-- Final URL: ${exec.finalUrl || 'unknown'}
-${actionsBlock}
-${assertionsBlock}
-${setupHint}
-IMPORTS AND PATTERN:
-\`\`\`javascript
-import { test, expect } from '@playwright/test';
-import { StableIdRuntime } from '@zibby/core';
-
-async function clickSafe(page, stableId, fallback) {
-  try { await StableIdRuntime.clickWithRetry(page, stableId); }
-  catch { await fallback.click(); }
-}
-
-test('Test Name', async ({ page }) => {
-  await page.goto('https://...');
-  await StableIdRuntime.injectStableIds(page);
-  // Elements WITH stable IDs + fallback — use clickSafe:
-  await clickSafe(page, 'zibby-xxxxx', page.getByRole('button', { name: '...' }));
-  await StableIdRuntime.fillWithRetry(page, 'zibby-xxxxx', 'value');
-  // Elements WITHOUT stable IDs (NO_STABLE_ID) — use native Playwright selectors:
-  await page.getByText('visible text').click();
-  await page.getByRole('button', { name: 'Submit' }).click();
-  await page.getByPlaceholder('placeholder text').fill('value');
-  await expect(page).toHaveURL(/expected-url/);
-});
-\`\`\`
-
-RULES:
-1. First navigate → page.goto(), skip subsequent navigates
-2. After goto, call StableIdRuntime.injectStableIds(page)
-3. Selector priority:
-   a. If memory/insights flag a stableId as unreliable → use the fallback selector instead
-   b. If action is marked [DUPLICATE_STABLE_ID] → always use the provided fallback
-   c. Cross-reference stableIds against memory "Reliable Selectors" and "Flaky Selectors" — prefer proven selectors, avoid flaky ones
-   d. Otherwise use EXACT stable IDs from recorded actions
-4. For [NO_STABLE_ID] actions, use the fallback selector (getByText, getByRole, getByPlaceholder)
-5. Skip duplicate consecutive clicks on same stableId
-6. No comments in generated code
-7. Implement ALL assertions from the list above
-8. If an assertion fails after retries, comment it out with a TODO (don't delete it)
-9. Selector failure handling:
-   - When a stableId fails and you switch to a fallback, IMMEDIATELY call memory_save_insight (category: selector_tip) with which stableId failed, which fallback worked, and the page URL
-   - Note the specific Playwright locator strategy that succeeded
-10. Navigation order: always complete setup/login FIRST from the base URL, then navigate to the target page. Never go to a deep URL before setup is done.
-11. The generated test runs in a FRESH browser with no prior state. Even if the live execution skipped setup steps, the test must include them. Check memory insights for any required setup.
-
-WORKFLOW:
-1. Study the codebase FIRST — search tests/ for existing helpers, fixtures, and shared setup files. Read them. Reuse what exists. Do NOT create files that duplicate existing ones.
-2. Write test to ${outputPath} (after the run, a copy is mirrored under ${state.sessionPath}/generate_script/ for Studio — you may also write directly there if you prefer)
-3. Verify syntax: run node --check on the file. If it fails, fix and re-check before proceeding.
-4. Run: PLAYWRIGHT_HEADLESS=1 npx playwright test ${outputPath} --reporter=line --timeout=60000
-5. If fails: try selectors in order — (a) getByRole (b) getByText (c) getByTestId (d) add waitForSelector. Never retry the same selector twice.
-6. MAX 2 ATTEMPTS then STOP
-
-The test runs in: ${state.cwd || 'project root'}
-`;
-  },
-};

package/templates/browser-test-automation/nodes/index.mjs

@@ -1,4 +0,0 @@
-export { preflightNode } from './preflight.mjs';
-export { cacheReplayNode } from './cache-replay.mjs';
-export { executeLiveNode } from './execute-live.mjs';
-export { generateScriptNode } from './generate-script.mjs';

package/templates/browser-test-automation/nodes/preflight.mjs

@@ -1,94 +0,0 @@
-/**
- * Preflight Node
- * 
- * Pattern: Prompt-only node (no tools)
- * Purpose: Analyze test spec and extract title + structured assertion checklist
- * 
- * This runs before execution to define:
- * - A concise test title
- * - The complete list of assertions that must be verified
- * 
- * Downstream nodes receive this as their contract:
- * - execute_live: must report passed/failed for each assertion
- * - generate_script: must implement each assertion in the test
- */
-
-import { z, invokeAgent } from '@zibby/core';
-import { writeFileSync } from 'fs';
-import { join } from 'path';
-
-const AssertionSchema = z.object({
-  description: z.string().describe('What to verify (e.g., "User is redirected to dashboard")'),
-  expected: z.string().describe('What the expected outcome looks like (e.g., "URL contains /dashboard")')
-});
-
-const PreflightOutputSchema = z.object({
-  title: z.string().describe('Concise test title (5-10 words, action-oriented). Prefix with ticket ID if found.'),
-  assertions: z.array(AssertionSchema).describe('Every expected result from the spec as a verifiable assertion')
-});
-
-// Detect "no usable spec" before invoking the LLM. Catches:
-//   - state.testSpec is undefined (workflow run with no input)
-//   - empty string / whitespace only
-//   - the literal string "undefined" (some upstream paths stringify
-//     undefined into state, e.g. when --param isn't passed)
-function isMissingSpec(spec) {
-  if (spec == null) return true;
-  const s = String(spec).trim();
-  return s === '' || s.toLowerCase() === 'undefined';
-}
-
-const PROMPT = (testSpec) => `Analyze this test specification and extract:
-1. A concise test title (5-10 words, action-oriented). If you find a ticket ID (e.g., PROJ-123, ACME-456), prefix the title with it.
-2. Every expected result as a verifiable assertion. Each assertion must be something the browser can check after execution.
-
-Test Spec:
-${testSpec}
-
-IMPORTANT: You MUST create ONE assertion for EACH expected result in the spec. Do NOT skip any.
-
-Return ONLY this JSON:
-{ "title": "TICKET-ID: Short action title", "assertions": [ { "description": "...", "expected": "..." }, ... ] }`;
-
-export const preflightNode = {
-  name: 'preflight',
-  outputSchema: PreflightOutputSchema,
-
-  async execute(state) {
-    // Early exit BEFORE the LLM call when there's no spec to analyze.
-    // Without this guard the node fires the LLM, gets back
-    // `{title: "No test specification provided", assertions: []}`, and
-    // the graph's conditional edge then skips execute_live — but we've
-    // still paid for one preflight LLM call we didn't need to make.
-    // Returning empty assertions here triggers the same skip-to-END
-    // path in graph.mjs's preflight conditional edge.
-    if (isMissingSpec(state.testSpec)) {
-      console.log('⚠️  No test spec provided — skipping browser run.');
-      console.log('   Pass a spec via:  zibby test "<inline spec>"  or  zibby test path/to/spec.txt');
-      return {
-        title: 'No test specification provided',
-        assertions: [],
-      };
-    }
-
-    const result = await invokeAgent(PROMPT(state.testSpec), {
-      state,
-      model: state.model || 'auto',
-      schema: PreflightOutputSchema,
-    });
-    return result?.structured || result;
-  },
-
-  async onComplete(state, result) {
-    const sessionPath = state.sessionPath || process.env.ZIBBY_SESSION_PATH;
-    if (sessionPath && result.title) {
-      try {
-        writeFileSync(join(sessionPath, 'title.txt'), result.title, 'utf-8');
-        console.log(`Saved title: "${result.title}"`);
-      } catch (error) {
-        console.warn(`⚠️  Could not save title.txt: ${error.message}`);
-      }
-    }
-    return result;
-  },
-};