@taodigital/agents 0.0.0-stage → 1.2.1

package/dist/visual-agent.js

@@ -0,0 +1,261 @@
+"use strict";
+/**
+ * ZeTa-AI Visual Intelligence Agent
+ *
+ * Handles visual regression testing:
+ * 1. On first crawl — save screenshots as baselines.
+ * 2. On subsequent crawls — compare new screenshots to baselines.
+ * 3. AI analyzes diffs: meaningful change vs. rendering noise.
+ * 4. Generates bug report drafts for critical visual changes.
+ *
+ * Phase 2: Vision AI integration (uses Claude's vision capability).
+ * Phase 1: Pixel-diff fallback (no vision AI needed yet).
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setVisualBaseline = setVisualBaseline;
+exports.compareToBaseline = compareToBaseline;
+exports.updateIgnoredRegions = updateIgnoredRegions;
+exports.setAllViewportBaselines = setAllViewportBaselines;
+exports.compareAllViewports = compareAllViewports;
+exports.aiVisualCompare = aiVisualCompare;
+const app_brain_1 = require("@taodigital/app-brain");
+const core_1 = require("@taodigital/core");
+const llm_js_1 = require("./llm.js");
+const llm_config_js_1 = require("./llm-config.js");
+const promises_1 = __importDefault(require("node:fs/promises"));
+const sharp_1 = __importDefault(require("sharp"));
+async function compressForVision(buf) {
+    const compressed = await (0, sharp_1.default)(buf)
+        .resize(768, 512, { fit: 'inside', withoutEnlargement: true })
+        .jpeg({ quality: 80, progressive: false })
+        .toBuffer();
+    return { base64: compressed.toString('base64'), mediaType: 'image/jpeg' };
+}
+/**
+ * Set or update the visual baseline for a screen.
+ * Called after the first successful crawl screenshot is stored.
+ */
+async function setVisualBaseline(opts) {
+    await app_brain_1.AppBrain.saveVisualBaseline(opts.tenantId, opts.projectId, opts.screenId, opts.screenshotPath, opts.viewport ?? 'DESKTOP', opts.approvedBy);
+}
+/**
+ * Compare a new screenshot against the stored baseline.
+ * Phase 1: returns a structural diff (element count change).
+ * Phase 2: Vision AI reads both images and produces rich analysis.
+ */
+async function compareToBaseline(opts) {
+    const baseline = await app_brain_1.AppBrain.getVisualBaseline(opts.tenantId, opts.screenId, opts.viewport ?? 'DESKTOP');
+    if (!baseline)
+        return null; // No baseline yet — this IS the baseline.
+    const bl = baseline;
+    // Phase 1 fallback: compare file sizes as a rough diff proxy.
+    // Phase 2: use Vision AI to compare actual images.
+    let diffScore = 0;
+    let changeType = 'NONE';
+    let changes = [];
+    let recommendation = 'APPROVE';
+    let aiSummary;
+    try {
+        const [baselineBuffer, newBuffer] = await Promise.all([
+            promises_1.default.readFile(bl.storagePath).catch(() => null),
+            promises_1.default.readFile(opts.newScreenshotPath).catch(() => null),
+        ]);
+        if (baselineBuffer && newBuffer) {
+            const [baselineImg, newImg] = await Promise.all([
+                compressForVision(baselineBuffer),
+                compressForVision(newBuffer),
+            ]);
+            const aiResult = await aiVisualCompare({
+                baselineImageBase64: baselineImg.base64,
+                newImageBase64: newImg.base64,
+                imageMediaType: baselineImg.mediaType,
+                screenName: opts.screenName,
+                screenPurpose: 'Application screen captured during automated crawl',
+                tenantId: opts.tenantId,
+                projectId: opts.projectId,
+            });
+            diffScore = aiResult.diffScore;
+            changeType = aiResult.changeType;
+            changes = aiResult.changes;
+            recommendation = aiResult.recommendation;
+            aiSummary = aiResult.summary;
+        }
+    }
+    catch {
+        // AI comparison failed — fall back to file-size proxy
+        try {
+            const [baselineStat, newStat] = await Promise.all([
+                promises_1.default.stat(bl.storagePath).catch(() => null),
+                promises_1.default.stat(opts.newScreenshotPath).catch(() => null),
+            ]);
+            if (baselineStat && newStat) {
+                const sizeDiff = Math.abs(newStat.size - baselineStat.size) / baselineStat.size;
+                diffScore = Math.min(100, sizeDiff * 100);
+                if (diffScore < 2) {
+                    changeType = 'NONE';
+                    recommendation = 'APPROVE';
+                }
+                else if (diffScore < 15) {
+                    changeType = 'MINOR';
+                    recommendation = 'APPROVE';
+                    changes = [{ description: 'Minor size change', location: 'Unknown', severity: 'Cosmetic' }];
+                }
+                else if (diffScore < 40) {
+                    changeType = 'SIGNIFICANT';
+                    recommendation = 'REVIEW';
+                    changes = [{ description: 'Significant visual change detected', location: 'Unknown', severity: 'Functional' }];
+                }
+                else {
+                    changeType = 'CRITICAL';
+                    recommendation = 'FLAG';
+                    changes = [{ description: 'Major visual change', location: 'Unknown', severity: 'Critical' }];
+                }
+            }
+        }
+        catch { /* ignore */ }
+    }
+    // Save the diff record.
+    const bugReportDraft = changeType === 'CRITICAL' || changeType === 'SIGNIFICANT'
+        ? {
+            title: `Visual regression on "${opts.screenName}"`,
+            severity: changeType === 'CRITICAL' ? 'High' : 'Medium',
+            steps: ['Navigate to the screen', 'Compare current view to baseline screenshot'],
+            expected: 'Screen appearance should match the stored baseline.',
+            actual: aiSummary || `Screen has changed (diff score: ${diffScore.toFixed(1)}%)`,
+        }
+        : undefined;
+    await app_brain_1.AppBrain.saveVisualDiff(opts.tenantId, {
+        baselineId: bl.id,
+        newScreenshotPath: opts.newScreenshotPath,
+        diffScore,
+        changeType,
+        changes,
+        recommendation,
+        bugReportDraft,
+        aiSummary,
+    });
+    return {
+        screenId: opts.screenId,
+        changeType,
+        diffScore,
+        changes,
+        recommendation,
+        bugReportDraft,
+    };
+}
+/**
+ * Update the ignored regions for a visual baseline.
+ * Regions are excluded from AI visual comparison (PHI masking, dynamic content).
+ * Each region is a rectangle in original image coordinates (before compression).
+ */
+async function updateIgnoredRegions(opts) {
+    await app_brain_1.AppBrain.updateBaselineIgnoredRegions(opts.tenantId, opts.screenId, opts.viewport ?? 'DESKTOP', opts.regions);
+}
+/**
+ * Set visual baselines for all standard viewports from a map of viewport→screenshotPath.
+ * Call after crawl completes with screenshots at each breakpoint.
+ */
+async function setAllViewportBaselines(opts) {
+    await Promise.all(Object.entries(opts.screenshots).map(([viewport, path]) => setVisualBaseline({
+        tenantId: opts.tenantId,
+        projectId: opts.projectId,
+        screenId: opts.screenId,
+        screenshotPath: path,
+        viewport,
+        approvedBy: opts.approvedBy,
+    })));
+}
+/**
+ * Compare a screen against baselines for ALL viewports that have baselines stored.
+ * Returns a map of viewport → comparison result. Null value = no baseline for that viewport.
+ */
+async function compareAllViewports(opts) {
+    const viewports = ['DESKTOP', 'TABLET', 'MOBILE'];
+    const results = {};
+    await Promise.all(viewports.map(async (viewport) => {
+        const screenshotPath = opts.screenshots[viewport];
+        if (!screenshotPath) {
+            results[viewport] = null;
+            return;
+        }
+        results[viewport] = await compareToBaseline({
+            tenantId: opts.tenantId,
+            projectId: opts.projectId,
+            screenId: opts.screenId,
+            newScreenshotPath: screenshotPath,
+            screenName: `${opts.screenName} (${viewport})`,
+            viewport,
+        });
+    }));
+    return results;
+}
+/**
+ * Phase 2: AI-powered visual comparison using Claude vision.
+ * This replaces the file-size proxy with real image understanding.
+ */
+async function aiVisualCompare(opts) {
+    const SYSTEM_DEFAULT = `You are ZeTa-AI's Visual Intelligence Agent. Your job is to compare two screenshots of the SAME application screen and identify REAL, VISIBLE differences between them.
+
+CRITICAL RULES:
+1. Only report changes you can ACTUALLY SEE in the provided images. NEVER invent or guess changes.
+2. If the images look identical or nearly identical, return changeType: "NONE" with an empty changes array.
+3. Ignore: font anti-aliasing, sub-pixel rendering, JPEG compression artifacts, identical scroll position differences.
+4. Only flag: missing UI elements, layout shifts, color changes, text content changes, error states, broken images.
+5. Be conservative — false positives are worse than false negatives.
+
+Return ONLY valid JSON (no markdown, no explanation):
+{
+  "changeType": "NONE" | "MINOR" | "SIGNIFICANT" | "CRITICAL",
+  "diffScore": <0-100, where 0=pixel-perfect identical, 100=completely different>,
+  "changes": [{"description": "<exact visible change>", "location": "<where on screen>", "severity": "COSMETIC" | "MINOR" | "SIGNIFICANT" | "CRITICAL"}],
+  "recommendation": "APPROVE" | "REVIEW" | "FLAG" | "BLOCK",
+  "summary": "<one sentence: either 'No meaningful changes detected' or brief description of actual changes>"
+}`;
+    const user = `Screen: "${opts.screenName}"
+
+Image 1 (BASELINE — the reference): provided above as the first image.
+Image 2 (CURRENT — the new capture): provided above as the second image.
+
+Compare them carefully. If they look the same, say so. Only list changes you can actually see.`;
+    try {
+        const t0 = Date.now();
+        const llmCfg = opts.tenantId ? await (0, llm_config_js_1.getTaskAgentLLMConfig)(opts.tenantId, 'visualDiff') : undefined;
+        const system = await (0, core_1.getAgentPrompt)('visual-diff-analysis', undefined, opts.tenantId) ?? SYSTEM_DEFAULT;
+        const mt = opts.imageMediaType ?? 'image/png';
+        const images = [
+            { base64: opts.baselineImageBase64, mediaType: mt },
+            { base64: opts.newImageBase64, mediaType: mt },
+        ];
+        const seed = (0, llm_js_1.deterministicSeed)(`visual-diff:${opts.tenantId ?? ''}:${opts.baselineImageBase64.slice(0, 200)}:${opts.newImageBase64.slice(0, 200)}`);
+        const { text: raw, usage } = await (0, llm_js_1.askTrackedWithImages)(system, user, images, llmCfg, { temperature: 0, seed });
+        const latencyMs = Date.now() - t0;
+        if (opts.tenantId) {
+            (0, app_brain_1.recordLLMCall)({
+                tenantId: opts.tenantId,
+                projectId: opts.projectId,
+                agentType: 'visual',
+                label: 'ai_visual_compare',
+                usage,
+                latencyMs,
+            }).catch(console.error);
+        }
+        const start = raw.indexOf('{');
+        const end = raw.lastIndexOf('}');
+        if (start === -1 || end === -1)
+            throw new Error('No JSON in response');
+        const result = JSON.parse(raw.slice(start, end + 1));
+        // Sanitize: drop any change items missing a description
+        if (Array.isArray(result.changes)) {
+            result.changes = result.changes.filter((c) => c?.description?.trim());
+        }
+        if (!result.summary)
+            result.summary = result.changeType === 'NONE' ? 'No meaningful changes detected.' : '';
+        return result;
+    }
+    catch {
+        return { changeType: 'NONE', changes: [], recommendation: 'APPROVE', diffScore: 0, summary: 'No meaningful changes detected.' };
+    }
+}

package/package.json

@@ -1,6 +1,28 @@
 {
   "name": "@taodigital/agents",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "1.2.1",
+  "private": false,
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "test": "tsx --test src/**/*.test.ts",
+    "e2e": "tsx scripts/agent-e2e.ts",
+    "eval": "tsx eval/run-eval.ts",
+    "eval:accept": "tsx eval/run-eval.ts --accept"
+  },
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.27.0",
+    "@taodigital/app-brain": "*",
+    "@taodigital/core": "*",
+    "openai": "^4.82.0",
+    "sharp": "^0.35.0"
+  }
+}