@vscjava/vscode-autotest 0.1.0

package/dist/operators/stepVerifier.js

@@ -0,0 +1,182 @@
+/**
+ * StepVerifier — runs verification checks for a test step.
+ *
+ * Supports both deterministic checks (verifyFile, verifyEditor, verifyProblems, etc.)
+ * and LLM-powered natural language verification (verify field + screenshot).
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+export class StepVerifier {
+    driver;
+    llm;
+    constructor(driver, options = {}) {
+        this.driver = driver;
+        this.llm = options.llmClient ?? null;
+    }
+    /**
+     * Verify a step against all its verification criteria.
+     * Runs deterministic checks first, then LLM verification if needed.
+     *
+     * @param screenshotPath — path to the after-action screenshot (for LLM verify)
+     */
+    async verify(step, screenshotPath) {
+        // If no verification defined, auto-pass
+        if (!step.verify && !step.verifyFile && !step.verifyNotification
+            && !step.verifyEditor && !step.verifyProblems && !step.verifyCompletion) {
+            return { passed: true };
+        }
+        // ── Deterministic verifications (run all, fail fast) ──
+        const fileResult = await this.verifyFile(step);
+        if (fileResult && !fileResult.passed)
+            return fileResult;
+        const notifResult = await this.verifyNotification(step);
+        if (notifResult && !notifResult.passed)
+            return notifResult;
+        const editorResult = await this.verifyEditor(step);
+        if (editorResult && !editorResult.passed)
+            return editorResult;
+        const problemsResult = await this.verifyProblems(step);
+        if (problemsResult && !problemsResult.passed)
+            return problemsResult;
+        const completionResult = await this.verifyCompletion(step);
+        if (completionResult && !completionResult.passed)
+            return completionResult;
+        // ── LLM verification for natural language `verify` field ──
+        if (step.verify) {
+            const llmResult = await this.verifyWithLLM(step.verify, screenshotPath);
+            return llmResult;
+        }
+        return { passed: true };
+    }
+    // ─── Deterministic Verifiers ─────────────────────────────
+    async verifyFile(step) {
+        if (!step.verifyFile)
+            return null;
+        const filePath = path.resolve(step.verifyFile.path);
+        if (step.verifyFile.exists === false) {
+            const exists = await this.driver.fileExists(filePath);
+            if (exists)
+                return { passed: false, reason: `File should not exist: ${filePath}` };
+        }
+        else {
+            if (!await this.driver.fileExists(filePath)) {
+                return { passed: false, reason: `File not found: ${filePath}` };
+            }
+            if (step.verifyFile.contains) {
+                const contains = await this.driver.fileContains(filePath, step.verifyFile.contains);
+                if (!contains) {
+                    return { passed: false, reason: `File does not contain: "${step.verifyFile.contains}"` };
+                }
+            }
+        }
+        return { passed: true };
+    }
+    async verifyNotification(step) {
+        if (!step.verifyNotification)
+            return null;
+        const notifications = await this.driver.getNotifications();
+        const found = notifications.some((n) => n.includes(step.verifyNotification));
+        if (!found) {
+            return {
+                passed: false,
+                reason: `Notification not found: "${step.verifyNotification}". Got: [${notifications.join(", ")}]`,
+            };
+        }
+        return { passed: true };
+    }
+    async verifyEditor(step) {
+        if (!step.verifyEditor?.contains)
+            return null;
+        const found = await this.driver.editorContains(step.verifyEditor.contains);
+        if (!found) {
+            return {
+                passed: false,
+                reason: `Editor does not contain: "${step.verifyEditor.contains}"`,
+            };
+        }
+        return { passed: true };
+    }
+    async verifyProblems(step) {
+        if (!step.verifyProblems)
+            return null;
+        const maxWait = (step.timeout ?? 30) * 1000;
+        const pollInterval = 3000;
+        const deadline = Date.now() + maxWait;
+        let lastCounts = { errors: 0, warnings: 0 };
+        while (Date.now() < deadline) {
+            lastCounts = await this.driver.getProblemsCount();
+            const errorsOk = step.verifyProblems.errors === undefined || (step.verifyProblems.atLeast
+                ? lastCounts.errors >= step.verifyProblems.errors
+                : lastCounts.errors === step.verifyProblems.errors);
+            const warningsOk = step.verifyProblems.warnings === undefined || (step.verifyProblems.atLeast
+                ? lastCounts.warnings >= step.verifyProblems.warnings
+                : lastCounts.warnings === step.verifyProblems.warnings);
+            if (errorsOk && warningsOk)
+                return { passed: true };
+            await this.driver.wait(pollInterval / 1000);
+        }
+        const parts = [];
+        if (step.verifyProblems.errors !== undefined) {
+            const cmp = step.verifyProblems.atLeast ? "at least " : "";
+            parts.push(`Expected ${cmp}${step.verifyProblems.errors} errors, got ${lastCounts.errors}`);
+        }
+        if (step.verifyProblems.warnings !== undefined) {
+            const cmp = step.verifyProblems.atLeast ? "at least " : "";
+            parts.push(`Expected ${cmp}${step.verifyProblems.warnings} warnings, got ${lastCounts.warnings}`);
+        }
+        return { passed: false, reason: parts.join("; ") };
+    }
+    async verifyCompletion(step) {
+        if (!step.verifyCompletion)
+            return null;
+        const items = await this.driver.triggerCompletion();
+        await this.driver.dismissCompletion();
+        if (step.verifyCompletion.notEmpty && items.length === 0) {
+            return { passed: false, reason: "Expected non-empty completion list, got empty" };
+        }
+        if (step.verifyCompletion.contains) {
+            for (const expected of step.verifyCompletion.contains) {
+                const found = items.some((item) => item.toLowerCase().includes(expected.toLowerCase()));
+                if (!found) {
+                    return {
+                        passed: false,
+                        reason: `Completion list missing "${expected}". Got: [${items.slice(0, 10).join(", ")}${items.length > 10 ? "..." : ""}]`,
+                    };
+                }
+            }
+        }
+        return { passed: true };
+    }
+    // ─── LLM Verification ───────────────────────────────────
+    async verifyWithLLM(description, screenshotPath) {
+        // If no LLM client or not configured, auto-pass with warning
+        if (!this.llm || !this.llm.isConfigured()) {
+            console.log(`   🤖 AI verify skipped (not configured): "${description}"`);
+            return { passed: true, reason: `LLM not configured — auto-pass: ${description}` };
+        }
+        // Read screenshot as base64
+        if (!screenshotPath || !fs.existsSync(screenshotPath)) {
+            console.log(`   🤖 AI verify skipped (no screenshot): "${description}"`);
+            return { passed: true, reason: `No screenshot for LLM verify: ${description}` };
+        }
+        console.log(`   🤖 AI verifying: "${description}"`);
+        const screenshotBase64 = fs.readFileSync(screenshotPath).toString("base64");
+        try {
+            const result = await this.llm.verifyScreenshot(screenshotBase64, description);
+            const icon = result.passed ? "✅" : "❌";
+            console.log(`   🤖 ${icon} confidence=${result.confidence.toFixed(2)}: ${result.reasoning}`);
+            return {
+                passed: result.passed,
+                reason: `[LLM ${result.confidence.toFixed(2)}] ${result.reasoning}`,
+            };
+        }
+        catch (e) {
+            console.log(`   🤖 ⚠️ LLM error: ${e.message}`);
+            return {
+                passed: true,
+                reason: `LLM error (auto-pass): ${e.message}`,
+            };
+        }
+    }
+}
+//# sourceMappingURL=stepVerifier.js.map

package/dist/operators/stepVerifier.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stepVerifier.js","sourceRoot":"","sources":["../../src/operators/stepVerifier.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAUlC,MAAM,OAAO,YAAY;IACf,MAAM,CAAe;IACrB,GAAG,CAAmB;IAE9B,YAAY,MAAoB,EAAE,UAA+B,EAAE;QACjE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC;IACvC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,MAAM,CACV,IAAc,EACd,cAAuB;QAEvB,wCAAwC;QACxC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,kBAAkB;eACzD,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC5E,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC1B,CAAC;QAED,yDAAyD;QAEzD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC/C,IAAI,UAAU,IAAI,CAAC,UAAU,CAAC,MAAM;YAAE,OAAO,UAAU,CAAC;QAExD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;QACxD,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,MAAM;YAAE,OAAO,WAAW,CAAC;QAE3D,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACnD,IAAI,YAAY,IAAI,CAAC,YAAY,CAAC,MAAM;YAAE,OAAO,YAAY,CAAC;QAE9D,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QACvD,IAAI,cAAc,IAAI,CAAC,cAAc,CAAC,MAAM;YAAE,OAAO,cAAc,CAAC;QAEpE,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAC3D,IAAI,gBAAgB,IAAI,CAAC,gBAAgB,CAAC,MAAM;YAAE,OAAO,gBAAgB,CAAC;QAE1E,6DAA6D;QAE7D,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;YACxE,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED,4DAA4D;IAEpD,KAAK,CAAC,UAAU,CAAC,IAAc;QACrC,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO,IAAI,CAAC;QAElC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;YACrC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YACtD,IAAI,MAAM;gBAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,0BAA0B,QAAQ,EAAE,EAAE,CAAC;QACrF,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5C,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,QAAQ,EAAE,EAAE,CAAC;YAClE,CAAC;YACD,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC;gBAC7B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBACpF,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,2BAA2B,IAAI,CAAC,UAAU,CAAC,QAAQ,GAAG,EAAE,CAAC;gBAC3F,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC1B,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,IAAc;QAC7C,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,IAAI,CAAC;QAE1C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAC3D,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,kBAAmB,CAAC,CAAC,CAAC;QAC9E,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,4BAA4B,IAAI,CAAC,kBAAkB,YAAY,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;aACnG,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC1B,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,IAAc;QACvC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE9C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAC3E,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,6BAA6B,IAAI,CAAC,YAAY,CAAC,QAAQ,GAAG;aACnE,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC1B,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,IAAc;QACzC,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC;QAEtC,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC;QAC5C,MAAM,YAAY,GAAG,IAAI,CAAC;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACtC,IAAI,UAAU,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QAE5C,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;YAC7B,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,KAAK,SAAS,IAAI,CAC3D,IAAI,CAAC,cAAc,CAAC,OAAO;gBACzB,CAAC,CAAC,UAAU,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM;gBACjD,CAAC,CAAC,UAAU,CAAC,MAAM,KAAK,IAAI,CAAC,cAAc,CAAC,MAAM,CACrD,CAAC;YACF,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,KAAK,SAAS,IAAI,CAC/D,IAAI,CAAC,cAAc,CAAC,OAAO;gBACzB,CAAC,CAAC,UAAU,CAAC,QAAQ,IAAI,IAAI,CAAC,cAAc,CAAC,QAAQ;gBACrD,CAAC,CAAC,UAAU,CAAC,QAAQ,KAAK,IAAI,CAAC,cAAc,CAAC,QAAQ,CACzD,CAAC;YACF,IAAI,QAAQ,IAAI,UAAU;gBAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YACpD,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;QAC9C,CAAC;QAED,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,gBAAgB,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9F,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC/C,MAAM,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,kBAAkB,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QACpG,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IACrD,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,IAAc;QAC3C,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAAE,OAAO,IAAI,CAAC;QAExC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;QACpD,MAAM,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAEtC,IAAI,IAAI,CAAC,gBAAgB,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,+CAA+C,EAAE,CAAC;QACpF,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC;YACnC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC;gBACtD,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAChC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CACpD,CAAC;gBACF,IAAI,CAAC,KAAK,EAAE,CAAC;oBACX,OAAO;wBACL,MAAM,EAAE,KAAK;wBACb,MAAM,EAAE,4BAA4B,QAAQ,YAAY,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG;qBAC1H,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED,2DAA2D;IAEnD,KAAK,CAAC,aAAa,CACzB,WAAmB,EACnB,cAAuB;QAEvB,6DAA6D;QAC7D,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,EAAE,CAAC;YAC1C,OAAO,CAAC,GAAG,CAAC,8CAA8C,WAAW,GAAG,CAAC,CAAC;YAC1E,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,mCAAmC,WAAW,EAAE,EAAE,CAAC;QACpF,CAAC;QAED,4BAA4B;QAC5B,IAAI,CAAC,cAAc,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;YACtD,OAAO,CAAC,GAAG,CAAC,6CAA6C,WAAW,GAAG,CAAC,CAAC;YACzE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,iCAAiC,WAAW,EAAE,EAAE,CAAC;QAClF,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,wBAAwB,WAAW,GAAG,CAAC,CAAC;QACpD,MAAM,gBAAgB,GAAG,EAAE,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAE5E,IAAI,CAAC;YACH,MAAM,MAAM,GAAuB,MAAM,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,gBAAgB,EAAE,WAAW,CAAC,CAAC;YAElG,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YACvC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,eAAe,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;YAE7F,OAAO;gBACL,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,MAAM,EAAE,QAAQ,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,SAAS,EAAE;aACpE,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,GAAG,CAAC,uBAAwB,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YAC3D,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,0BAA2B,CAAW,CAAC,OAAO,EAAE;aACzD,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}

package/dist/operators/testRunner.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Test Runner — orchestrates test plan execution.
+ *
+ * Delegates action resolution to ActionResolver and verification to StepVerifier.
+ * Handles lifecycle (launch/close), screenshots, and reporting.
+ */
+import type { TestPlan, TestReport } from "../types.js";
+export interface TestRunnerOptions {
+    /** Output directory for this test run. Contains screenshots/ and results.json. */
+    outputDir?: string;
+    /** Disable LLM verification (auto-pass all `verify` fields). */
+    noLLM?: boolean;
+}
+export declare class TestRunner {
+    private driver;
+    private plan;
+    private actionResolver;
+    private verifier;
+    private outputDir;
+    private screenshotDir;
+    private screenshotCounter;
+    constructor(plan: TestPlan, options?: TestRunnerOptions);
+    /** Force-close the VSCode instance (for signal handlers) */
+    cleanup(): Promise<void>;
+    run(): Promise<TestReport>;
+    private executeStep;
+    private takeScreenshot;
+    private cloneRepos;
+}

package/dist/operators/testRunner.js

@@ -0,0 +1,181 @@
+/**
+ * Test Runner — orchestrates test plan execution.
+ *
+ * Delegates action resolution to ActionResolver and verification to StepVerifier.
+ * Handles lifecycle (launch/close), screenshots, and reporting.
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { execSync } from "node:child_process";
+import { VscodeDriver } from "../drivers/vscodeDriver.js";
+import { ActionResolver } from "./actionResolver.js";
+import { LLMClient } from "./llmClient.js";
+import { StepVerifier } from "./stepVerifier.js";
+export class TestRunner {
+    driver;
+    plan;
+    actionResolver;
+    verifier;
+    outputDir;
+    screenshotDir;
+    screenshotCounter = 0;
+    constructor(plan, options = {}) {
+        this.plan = plan;
+        this.outputDir = options.outputDir ?? null;
+        this.screenshotDir = this.outputDir ? path.join(this.outputDir, "screenshots") : null;
+        this.driver = new VscodeDriver({
+            vscodeVersion: plan.setup.vscodeVersion,
+            extensionPath: plan.setup.extensionPath,
+            extensions: plan.setup.extensions,
+            workspacePath: plan.setup.workspace,
+            filePath: plan.setup.file,
+            settings: plan.setup.settings,
+        });
+        this.actionResolver = new ActionResolver(this.driver, {
+            lsTimeout: (plan.setup.timeout ?? 120) * 1000,
+        });
+        const llm = options.noLLM ? null : new LLMClient();
+        this.verifier = new StepVerifier(this.driver, { llmClient: llm });
+    }
+    /** Force-close the VSCode instance (for signal handlers) */
+    async cleanup() {
+        await this.driver.close();
+    }
+    async run() {
+        const startTime = new Date();
+        const results = [];
+        // Prepare output directory — clean stale data from previous runs
+        if (this.outputDir) {
+            if (fs.existsSync(this.outputDir)) {
+                fs.rmSync(this.outputDir, { recursive: true, force: true });
+            }
+            fs.mkdirSync(this.outputDir, { recursive: true });
+            fs.mkdirSync(this.screenshotDir, { recursive: true });
+            console.log(`📂 Output → ${this.outputDir}`);
+        }
+        try {
+            // Clone required repos if specified
+            if (this.plan.setup.repos?.length) {
+                await this.cloneRepos(this.plan.setup.repos);
+            }
+            console.log(`\n🚀 Launching VSCode for: ${this.plan.name}`);
+            await this.driver.launch();
+            console.log(`✅ VSCode ready\n`);
+            // Brief wait for UI to settle (not the full setup.timeout — that's for LS steps)
+            await this.driver.wait(3);
+            for (const step of this.plan.steps) {
+                const result = await this.executeStep(step);
+                results.push(result);
+                const icon = result.status === "pass" ? "✅" : result.status === "fail" ? "❌" : "⏭️";
+                console.log(`${icon} [${result.stepId}] ${result.action} (${result.duration}ms)`);
+                if (result.reason) {
+                    console.log(`   → ${result.reason}`);
+                }
+            }
+        }
+        catch (e) {
+            console.error(`\n💥 Fatal error: ${e.message}`);
+        }
+        finally {
+            await this.driver.close();
+        }
+        const endTime = new Date();
+        const summary = {
+            total: results.length,
+            passed: results.filter((r) => r.status === "pass").length,
+            failed: results.filter((r) => r.status === "fail").length,
+            skipped: results.filter((r) => r.status === "skip").length,
+            errors: results.filter((r) => r.status === "error").length,
+        };
+        console.log(`\n📊 Results: ${summary.passed}/${summary.total} passed`);
+        const report = {
+            planName: this.plan.name,
+            startTime: startTime.toISOString(),
+            endTime: endTime.toISOString(),
+            duration: endTime.getTime() - startTime.getTime(),
+            results,
+            summary,
+        };
+        // Save results.json into output directory
+        if (this.outputDir) {
+            const reportPath = path.join(this.outputDir, "results.json");
+            fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
+            console.log(`📄 Report → ${reportPath}`);
+        }
+        return report;
+    }
+    async executeStep(step) {
+        const start = Date.now();
+        let beforePath;
+        try {
+            if (step.waitBefore) {
+                await this.driver.wait(step.waitBefore);
+            }
+            beforePath = await this.takeScreenshot(step.id, "before");
+            // Delegate action execution to ActionResolver
+            await this.actionResolver.resolve(step.action);
+            const afterPath = await this.takeScreenshot(step.id, "after");
+            // Delegate verification to StepVerifier (pass screenshot for LLM)
+            const verifyResult = await this.verifier.verify(step, afterPath);
+            return {
+                stepId: step.id,
+                action: step.action,
+                status: verifyResult.passed ? "pass" : "fail",
+                reason: verifyResult.reason,
+                duration: Date.now() - start,
+                screenshot: afterPath,
+            };
+        }
+        catch (e) {
+            const errorPath = await this.takeScreenshot(step.id, "error");
+            return {
+                stepId: step.id,
+                action: step.action,
+                status: "error",
+                reason: e.message,
+                duration: Date.now() - start,
+                screenshot: errorPath,
+            };
+        }
+    }
+    async takeScreenshot(stepId, phase) {
+        if (!this.screenshotDir)
+            return undefined;
+        try {
+            // Dismiss notifications before every screenshot for clean captures
+            await this.driver.dismissAllNotifications();
+            const seq = String(++this.screenshotCounter).padStart(2, "0");
+            const fileName = `${seq}_${stepId}_${phase}.png`;
+            const filePath = path.join(this.screenshotDir, fileName);
+            await this.driver.screenshot(filePath);
+            return filePath;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    async cloneRepos(repos) {
+        for (const repo of repos) {
+            // Derive local path from URL if not specified
+            const repoName = repo.url.replace(/\.git$/, "").split("/").pop() ?? "repo";
+            const targetPath = repo.path ?? path.resolve(repoName);
+            if (fs.existsSync(targetPath)) {
+                console.log(`📦 Repo already exists: ${targetPath}`);
+                continue;
+            }
+            console.log(`📦 Cloning ${repo.url} → ${targetPath}`);
+            const branchArg = repo.branch ? `--branch ${repo.branch}` : "";
+            try {
+                execSync(`git clone --depth 1 ${branchArg} ${repo.url} "${targetPath}"`, {
+                    stdio: "pipe",
+                    timeout: 120_000,
+                });
+                console.log(`📦 Clone complete`);
+            }
+            catch (e) {
+                throw new Error(`Failed to clone ${repo.url}: ${e.message.slice(0, 200)}`);
+            }
+        }
+    }
+}
+//# sourceMappingURL=testRunner.js.map

package/dist/operators/testRunner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"testRunner.js","sourceRoot":"","sources":["../../src/operators/testRunner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE1D,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AASjD,MAAM,OAAO,UAAU;IACb,MAAM,CAAe;IACrB,IAAI,CAAW;IACf,cAAc,CAAiB;IAC/B,QAAQ,CAAe;IACvB,SAAS,CAAgB;IACzB,aAAa,CAAgB;IAC7B,iBAAiB,GAAG,CAAC,CAAC;IAE9B,YAAY,IAAc,EAAE,UAA6B,EAAE;QACzD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC;QAC3C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEtF,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,CAAC;YAC7B,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;YACvC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;YACvC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU;YACjC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS;YACnC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;YACzB,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ;SAC9B,CAAC,CAAC;QAEH,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE;YACpD,SAAS,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,GAAG,CAAC,GAAG,IAAI;SAC9C,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,SAAS,EAAE,CAAC;QACnD,IAAI,CAAC,QAAQ,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACpE,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,OAAO;QACX,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAiB,EAAE,CAAC;QAEjC,iEAAiE;QACjE,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBAClC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9D,CAAC;YACD,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAClD,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,aAAc,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACvD,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,CAAC;YACH,oCAAoC;YACpC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;gBAClC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/C,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC5D,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;YAEhC,iFAAiF;YACjF,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAE1B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBAC5C,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAErB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;gBACpF,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,KAAK,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;gBAClF,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;oBAClB,OAAO,CAAC,GAAG,CAAC,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;gBACvC,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,qBAAsB,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7D,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;YACzD,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;YACzD,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM;YAC1D,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM;SAC3D,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,iBAAiB,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,SAAS,CAAC,CAAC;QAEvE,MAAM,MAAM,GAAe;YACzB,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI;YACxB,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;YAClC,OAAO,EAAE,OAAO,CAAC,WAAW,EAAE;YAC9B,QAAQ,EAAE,OAAO,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE;YACjD,OAAO;YACP,OAAO;SACR,CAAC;QAEF,0CAA0C;QAC1C,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;YAC7D,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC9D,OAAO,CAAC,GAAG,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,IAAc;QACtC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,UAA8B,CAAC;QAEnC,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1C,CAAC;YAED,UAAU,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;YAE1D,8CAA8C;YAC9C,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAE/C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YAE9D,kEAAkE;YAClE,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAEjE,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;gBAC7C,MAAM,EAAE,YAAY,CAAC,MAAM;gBAC3B,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC5B,UAAU,EAAE,SAAS;aACtB,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YAE9D,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,EAAE;gBACf,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,OAAO;gBACf,MAAM,EAAG,CAAW,CAAC,OAAO;gBAC5B,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC5B,UAAU,EAAE,SAAS;aACtB,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,MAAc,EAAE,KAAmC;QAC9E,IAAI,CAAC,IAAI,CAAC,aAAa;YAAE,OAAO,SAAS,CAAC;QAC1C,IAAI,CAAC;YACH,mEAAmE;YACnE,MAAM,IAAI,CAAC,MAAM,CAAC,uBAAuB,EAAE,CAAC;YAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC9D,MAAM,QAAQ,GAAG,GAAG,GAAG,IAAI,MAAM,IAAI,KAAK,MAAM,CAAC;YACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;YACzD,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YACvC,OAAO,QAAQ,CAAC;QAClB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAkB;QACzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,8CAA8C;YAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC;YAC3E,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAEvD,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,GAAG,CAAC,2BAA2B,UAAU,EAAE,CAAC,CAAC;gBACrD,SAAS;YACX,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,GAAG,MAAM,UAAU,EAAE,CAAC,CAAC;YACtD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/D,IAAI,CAAC;gBACH,QAAQ,CAAC,uBAAuB,SAAS,IAAI,IAAI,CAAC,GAAG,KAAK,UAAU,GAAG,EAAE;oBACvE,KAAK,EAAE,MAAM;oBACb,OAAO,EAAE,OAAO;iBACjB,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;YACnC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,GAAG,KAAM,CAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YACxF,CAAC;QACH,CAAC;IACH,CAAC;CACF"}

package/dist/types.d.ts

@@ -0,0 +1,137 @@
+/**
+ * Core type definitions for VSCode AutoTest framework.
+ */
+export interface TestPlan {
+    name: string;
+    description?: string;
+    setup: TestSetup;
+    steps: TestStep[];
+}
+export interface TestSetup {
+    extension: string;
+    extensionPath?: string;
+    /** Marketplace extensions to install before launch (e.g. ["vscjava.vscode-java-pack"]) */
+    extensions?: string[];
+    vscodeVersion?: "stable" | "insiders";
+    /** Workspace folder to open. Mutually exclusive with `file`. */
+    workspace?: string;
+    /** Single file to open (no workspace). For testing LS in no-workspace mode. */
+    file?: string;
+    /** Git repos to clone before running. Each entry: { url, path?, branch? } */
+    repos?: RepoClone[];
+    settings?: Record<string, unknown>;
+    timeout?: number;
+}
+export interface RepoClone {
+    /** Git clone URL */
+    url: string;
+    /** Local path to clone into (relative to plan file). If omitted, derived from repo name. */
+    path?: string;
+    /** Branch or tag to checkout */
+    branch?: string;
+}
+export interface TestStep {
+    id: string;
+    action: string;
+    verify?: string;
+    verifyFile?: FileVerification;
+    verifyNotification?: string;
+    verifyEditor?: EditorVerification;
+    verifyProblems?: ProblemsVerification;
+    verifyCompletion?: CompletionVerification;
+    timeout?: number;
+    waitBefore?: number;
+}
+export interface FileVerification {
+    path: string;
+    exists?: boolean;
+    contains?: string;
+    matches?: string;
+}
+export interface EditorVerification {
+    fileName?: string;
+    contains?: string;
+    language?: string;
+}
+export interface ProblemsVerification {
+    errors?: number;
+    warnings?: number;
+    /** If true, check errors >= value instead of exact match */
+    atLeast?: boolean;
+}
+export interface CompletionVerification {
+    /** Completion list must not be empty */
+    notEmpty?: boolean;
+    /** Completion list must include these items (partial match on label) */
+    contains?: string[];
+}
+export interface A11yNode {
+    role: string;
+    name?: string;
+    value?: string;
+    description?: string;
+    focused?: boolean;
+    checked?: boolean;
+    disabled?: boolean;
+    expanded?: boolean;
+    children?: A11yNode[];
+}
+export interface Diagnostic {
+    severity: "error" | "warning" | "info" | "hint";
+    message: string;
+    source?: string;
+    file?: string;
+    line?: number;
+}
+export interface VscodeDriverOptions {
+    vscodeVersion?: "stable" | "insiders";
+    extensionPath?: string;
+    /** Marketplace extension IDs to install before launch */
+    extensions?: string[];
+    workspacePath?: string;
+    /** Single file to open (no workspace) */
+    filePath?: string;
+    userDataDir?: string;
+    settings?: Record<string, unknown>;
+    launchArgs?: string[];
+    /** Connect to an existing VSCode instance via CDP port instead of launching */
+    attachPort?: number;
+}
+export interface StepResult {
+    stepId: string;
+    action: string;
+    status: "pass" | "fail" | "skip" | "error";
+    reason?: string;
+    duration: number;
+    snapshot?: A11yNode;
+    screenshot?: string;
+}
+export interface TestReport {
+    planName: string;
+    startTime: string;
+    endTime: string;
+    duration: number;
+    results: StepResult[];
+    summary: {
+        total: number;
+        passed: number;
+        failed: number;
+        skipped: number;
+        errors: number;
+    };
+}
+export interface ActionMapping {
+    /** Original natural language action */
+    action: string;
+    /** Mapped driver method calls */
+    calls: DriverCall[];
+}
+export interface DriverCall {
+    method: string;
+    args: unknown[];
+}
+export interface VerificationResult {
+    passed: boolean;
+    reasoning: string;
+    confidence: number;
+}

package/dist/types.js

@@ -0,0 +1,5 @@
+/**
+ * Core type definitions for VSCode AutoTest framework.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "@vscjava/vscode-autotest",
+  "version": "0.1.0",
+  "description": "AI-driven VSCode extension E2E testing framework — YAML test plans + Playwright + LLM verification",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "autotest": "dist/cli/index.js"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "keywords": [
+    "vscode",
+    "testing",
+    "e2e",
+    "playwright",
+    "java",
+    "language-server",
+    "automation"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/wenytang-ms/javaext-autotest.git"
+  },
+  "homepage": "https://github.com/wenytang-ms/javaext-autotest#readme",
+  "bugs": {
+    "url": "https://github.com/wenytang-ms/javaext-autotest/issues"
+  },
+  "license": "MIT",
+  "author": "vscjava",
+  "engines": {
+    "node": ">=18"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "test": "vitest",
+    "lint": "eslint src/",
+    "prepublishOnly": "npm run build",
+    "run:test-plan": "tsx src/cli/index.ts run",
+    "run:all": "npm run build && node -e \"const{readdirSync}=require('fs'),{execSync}=require('child_process');readdirSync('test-plans').filter(f=>f.endsWith('.yaml')).forEach(f=>{console.log('\\n=== '+f+' ===');try{execSync('npx autotest run test-plans/'+f,{stdio:'inherit'})}catch{}})\""
+  },
+  "dependencies": {
+    "@azure/openai": "^2.0.0",
+    "@playwright/test": "^1.50.0",
+    "@vscode/test-electron": "^2.4.0",
+    "commander": "^12.0.0",
+    "js-yaml": "^4.1.0"
+  },
+  "devDependencies": {
+    "@types/js-yaml": "^4.0.9",
+    "@types/node": "^22.0.0",
+    "tsx": "^4.19.0",
+    "typescript": "^5.6.0",
+    "vitest": "^2.0.0"
+  }
+}