@vostride/agent-qa-runner-local 0.2.2 → 0.2.4

package/README.md

@@ -0,0 +1,78 @@
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/vostride/agent-qa/main/docs/assets/readme/banner-dark.jpg">
+  <source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/vostride/agent-qa/main/docs/assets/readme/banner-light.png">
+  <img src="https://raw.githubusercontent.com/vostride/agent-qa/main/docs/assets/readme/banner-light.png" alt="agent-qa by vostride — a self-improving QA agent for your software factory" width="100%">
+</picture>
+
+<p align="center">
+  <strong>English</strong> · <a href="https://github.com/vostride/agent-qa/blob/main/README.zh-CN.md">中文</a>
+</p>
+
+<p align="center">
+  <a href="https://www.npmjs.com/package/agent-qa"><img src="https://img.shields.io/npm/v/agent-qa?color=196872" alt="npm version"></a>
+  <a href="https://www.npmjs.com/package/agent-qa"><img src="https://img.shields.io/npm/dm/agent-qa?color=196872" alt="npm downloads per month"></a>
+  <a href="https://github.com/vostride/agent-qa/stargazers"><img src="https://img.shields.io/github/stars/vostride/agent-qa?color=196872" alt="GitHub stars"></a>
+  <a href="https://github.com/vostride/agent-qa/actions/workflows/ci.yml"><img src="https://github.com/vostride/agent-qa/actions/workflows/ci.yml/badge.svg?branch=main" alt="CI"></a>
+  <a href="https://vostride.com/docs/agent-qa"><img src="https://img.shields.io/badge/docs-agent--qa-196872" alt="Documentation"></a>
+  <a href="https://github.com/vostride/agent-qa/blob/main/LICENSE.md"><img src="https://img.shields.io/badge/license-FSL--1.1--ALv2-196872" alt="License: FSL-1.1-ALv2"></a>
+</p>
+
+<p align="center">
+  <a href="https://vostride.com/agent-qa">Website</a> · <a href="https://vostride.com/docs/agent-qa">Docs</a> · <a href="https://vostride.com/demo/agent-qa">Demo</a> · <a href="https://vostride.com/blog">Blog</a> · <a href="https://vostride.com/changelog">Changelog</a>
+</p>
+
+**Your AI QA engineer, with memory.** Describe a user journey in plain language. agent-qa tests it on web, Android or iOS and keeps the evidence. Use the CLI, the dashboard, or your coding agent through MCP and skills.
+
+- **memidex:** remembers facts and working procedures as reviewable files in your repository.
+- **Self-improvement:** validates lessons from runs, retires stale knowledge, and uses your feedback to correct mistakes.
+
+| Benchmark | Rank | Published result |
+| --- | --- | --- |
+| [AndroidWorld](https://vostride.com/benchmark/android-world) | **#1** | **100% pass@1** (116/116 tasks) |
+| [AppControlBench](https://vostride.com/benchmark/app-control-bench) | **#1** | **99.6% completion** (114.5/115 points) |
+
+Read the full benchmark reports, with run evidence and methodology, on [our blog](https://vostride.com/blog). Both used gpt-6-sol with memory and self-improvement off; AppControlBench gives half credit for partial completion.
+
+## See it in action
+
+**Inspect a failure.** The demo catches a cart showing $29 instead of $53, with the failed step and its screenshot side by side.
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/vostride/agent-qa/main/docs/assets/readme/evidence-dark.png">
+  <source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/vostride/agent-qa/main/docs/assets/readme/evidence-light.png">
+  <img src="https://raw.githubusercontent.com/vostride/agent-qa/main/docs/assets/readme/evidence-light.png" alt="The dashboard shows the deliberate cart-total bug, its failed assertion, and recorded browser evidence" width="100%">
+</picture>
+
+**See what memory contributed.** Inspect the knowledge supplied to a run and trace it back to the steps that used it.
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/vostride/agent-qa/main/docs/assets/readme/knowledge-dark.png">
+  <source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/vostride/agent-qa/main/docs/assets/readme/knowledge-light.png">
+  <img src="https://raw.githubusercontent.com/vostride/agent-qa/main/docs/assets/readme/knowledge-light.png" alt="The Knowledge tab shows stored facts retrieved for the demo run" width="100%">
+</picture>
+
+**Run from your terminal.** Real output from the demo suite: checkout passes; the deliberate cart-total bug fails.
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/vostride/agent-qa/main/docs/assets/readme/cli-dark.png">
+  <source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/vostride/agent-qa/main/docs/assets/readme/cli-light.png">
+  <img src="https://raw.githubusercontent.com/vostride/agent-qa/main/docs/assets/readme/cli-light.png" alt="CLI output excerpt showing one passed test, one failed test, and eight of nine steps passing" width="100%">
+</picture>
+
+## Get started
+
+Requires Node.js 24+.
+
+```sh
+npm install -D agent-qa
+npx agent-qa init
+```
+
+Follow the [quickstart](https://vostride.com/docs/agent-qa/quickstart) to connect a model and run the demo, or [set up with your coding agent](https://vostride.com/docs/agent-qa/agent-quickstart).
+
+> [!NOTE]
+> Early stage. Expect bugs. [Report an issue](https://github.com/vostride/agent-qa/issues) when something breaks.
+
+## License
+
+[Functional Source License (FSL-1.1-ALv2)](https://github.com/vostride/agent-qa/blob/main/LICENSE.md).

package/dist/index.cjs

@@ -55,6 +55,7 @@ __export(index_exports, {
   latestBuildOrdinal: () => latestBuildOrdinal,
   loadCompletedRunHandoff: () => loadCompletedRunHandoff,
   nextBuildOrdinal: () => nextBuildOrdinal,
+  purgeActionCache: () => purgeActionCache,
   readBuildTagMap: () => readBuildTagMap,
   recordBuildTag: () => recordBuildTag,
   reserveBuildOrdinalForProducts: () => reserveBuildOrdinalForProducts,
@@ -146,39 +147,56 @@ function shouldWrite(policy) {
 }
 
 // src/fs-action-cache.ts
-var CACHE_SCHEMA_VERSION = 2;
+var CACHE_SCHEMA_VERSION = 3;
+var CACHE_ENTRY_FILE_RE = /^[0-9a-f]{16}\.json$/;
 var FsActionCache = class {
   root;
   policy;
+  skipped = /* @__PURE__ */ new Set();
   constructor(config) {
     this.root = config.root;
     this.policy = config.policy;
   }
   async get(key) {
-    if (this.policy.disableCache) return null;
+    if (this.policy.disableCache || this.skipped.has(key)) return null;
     const onCurrent = await this.readEntry(this.policy.currentBranch, key);
-    if (onCurrent !== null) return onCurrent;
+    if (onCurrent?.kind === "suppressed") return null;
+    if (onCurrent?.kind === "value") return onCurrent.value;
     if (this.policy.currentBranch !== this.policy.defaultBranch) {
-      return this.readEntry(this.policy.defaultBranch, key);
+      const onDefault = await this.readEntry(this.policy.defaultBranch, key);
+      return onDefault?.kind === "value" ? onDefault.value : null;
     }
     return null;
   }
   async set(key, value) {
     if (!shouldWrite(this.policy)) return;
-    const dir = this.branchDir(this.policy.currentBranch);
-    await (0, import_promises.mkdir)(dir, { recursive: true });
-    const envelope = { schemaVersion: CACHE_SCHEMA_VERSION, value };
-    await (0, import_promises.writeFile)(this.entryPath(dir, key), `${JSON.stringify(envelope, null, 2)}
-`, "utf-8");
+    await this.writeEnvelope(key, { schemaVersion: CACHE_SCHEMA_VERSION, value });
   }
   async invalidate(key) {
-    const dir = this.branchDir(this.policy.currentBranch);
+    if (!shouldWrite(this.policy)) {
+      this.skipped.add(key);
+      return;
+    }
+    const { currentBranch, defaultBranch } = this.policy;
+    if (currentBranch !== defaultBranch) {
+      const onDefault = await this.readEntry(defaultBranch, key);
+      if (onDefault?.kind === "value") {
+        await this.writeEnvelope(key, { schemaVersion: CACHE_SCHEMA_VERSION, suppressed: true });
+        return;
+      }
+    }
     try {
-      await (0, import_promises.unlink)(this.entryPath(dir, key));
+      await (0, import_promises.unlink)(this.entryPath(this.branchDir(currentBranch), key));
     } catch (err) {
-      if (err.code !== "ENOENT") throw err;
+      if (!isMissing(err)) throw err;
     }
   }
+  async writeEnvelope(key, envelope) {
+    const dir = this.branchDir(this.policy.currentBranch);
+    await (0, import_promises.mkdir)(dir, { recursive: true });
+    await (0, import_promises.writeFile)(this.entryPath(dir, key), `${JSON.stringify(envelope, null, 2)}
+`, "utf-8");
+  }
   branchDir(branch) {
     return (0, import_node_path2.join)(this.root, (0, import_agent_qa_schema.slugBranch)(branch));
   }
@@ -190,7 +208,7 @@ var FsActionCache = class {
     try {
       raw = await (0, import_promises.readFile)(this.entryPath(this.branchDir(branch), key), "utf-8");
     } catch (err) {
-      if (err.code === "ENOENT") return null;
+      if (isMissing(err)) return null;
       throw err;
     }
     let envelope;
@@ -200,9 +218,44 @@ var FsActionCache = class {
       return null;
     }
     if (envelope?.schemaVersion !== CACHE_SCHEMA_VERSION) return null;
-    return envelope.value ?? null;
+    if (envelope.suppressed === true) return { kind: "suppressed" };
+    return envelope.value ? { kind: "value", value: envelope.value } : null;
   }
 };
+function isMissing(err) {
+  return err.code === "ENOENT";
+}
+async function purgeActionCache(root) {
+  let branches;
+  try {
+    branches = await (0, import_promises.readdir)(root, { withFileTypes: true });
+  } catch (err) {
+    if (isMissing(err)) return 0;
+    throw err;
+  }
+  let purged = 0;
+  for (const branch of branches) {
+    if (!branch.isDirectory() || (0, import_agent_qa_schema.slugBranch)(branch.name) !== branch.name) continue;
+    const dir = (0, import_node_path2.join)(root, branch.name);
+    const entries = await (0, import_promises.readdir)(dir, { withFileTypes: true });
+    let removed = 0;
+    for (const entry of entries) {
+      if (!entry.isFile() || !CACHE_ENTRY_FILE_RE.test(entry.name)) continue;
+      try {
+        await (0, import_promises.unlink)((0, import_node_path2.join)(dir, entry.name));
+      } catch (err) {
+        if (!isMissing(err)) throw err;
+      }
+      removed++;
+    }
+    purged += removed;
+    if (removed > 0 && removed === entries.length) {
+      await (0, import_promises.rmdir)(dir).catch(() => {
+      });
+    }
+  }
+  return purged;
+}
 
 // src/storage-state-store.ts
 var import_promises2 = require("fs/promises");
@@ -898,9 +951,11 @@ var import_node_os = require("os");
 var import_agent_qa_runner = require("@vostride/agent-qa-runner");
 var DEFAULT_DOCKER_BIN = "docker";
 var DockerHookExecutor = class {
+  version;
   dockerBin;
   pullPolicy;
-  constructor(config = {}) {
+  constructor(config) {
+    this.version = config.version;
     this.dockerBin = config.dockerBin ?? DEFAULT_DOCKER_BIN;
     this.pullPolicy = config.pullPolicy ?? "if-not-present";
   }
@@ -908,15 +963,15 @@ var DockerHookExecutor = class {
     const dockerBin = this.dockerBin;
     const pullPolicy = this.pullPolicy;
     const start = Date.now();
-    const image = import_agent_qa_runner.RUNTIME_IMAGE_MAP[hook.runtime];
-    if (pullPolicy === "always" || pullPolicy === "if-not-present") {
-      const needsPull = pullPolicy === "always" || !await imageExists(dockerBin, image);
-      if (needsPull) {
-        await pullImage(dockerBin, image);
-      }
-    }
+    const image = (0, import_agent_qa_runner.hookRunnerImage)(hook.runtime, this.version);
     let workDir;
     try {
+      if (pullPolicy === "always" || pullPolicy === "if-not-present") {
+        const needsPull = pullPolicy === "always" || !await imageExists(dockerBin, image);
+        if (needsPull) {
+          await pullImage(dockerBin, image);
+        }
+      }
       workDir = await (0, import_promises3.mkdtemp)((0, import_node_path11.join)((0, import_node_os.tmpdir)(), "agent-qa-hook-"));
       await (0, import_promises3.mkdir)((0, import_node_path11.join)(workDir, "tmp"), { recursive: true });
       await (0, import_promises3.cp)(hook.file, (0, import_node_path11.join)(workDir, (0, import_node_path11.basename)(hook.file)));
@@ -1101,6 +1156,7 @@ function execDocker(dockerBin, args, timeout) {
   latestBuildOrdinal,
   loadCompletedRunHandoff,
   nextBuildOrdinal,
+  purgeActionCache,
   readBuildTagMap,
   recordBuildTag,
   reserveBuildOrdinalForProducts,