@qatonic_innovations/qaios 0.4.1 → 0.5.1

package/dist/index.js

@@ -905,7 +905,19 @@ var QaiosConfig = z.object({
     // Playwright defaults to `data-testid`; apps using `data-test`,
     // `data-qa`, etc. must declare it here so generated specs emit a
     // `test.use({ testIdAttribute })` and actually find their elements.
-    testIdAttribute: z.string().default("data-testid")
+    testIdAttribute: z.string().default("data-testid"),
+    // 0.5 DOM grounding: when true (default), `qaios test` captures the
+    // live page at `app.baseUrl` before writing and feeds the real DOM
+    // to write.web so it generates locators against actual elements
+    // instead of guessing from the DesignSpec. Needs the app reachable
+    // at generation time; when it isn't, capture fails fast (see
+    // groundCaptureTimeoutMs) and falls back to blind generation. Set
+    // false to skip capture entirely (also: `qaios test --no-ground`).
+    groundLocators: z.boolean().default(true),
+    // Hard cap on the headless page-capture before grounding gives up
+    // and falls back. Kept tight so a not-yet-running dev server doesn't
+    // stall generation — a real capture against a live page is fast.
+    groundCaptureTimeoutMs: z.number().int().positive().default(15e3)
   }).default({}),
   visual: z.object({
     viewports: z.array(
@@ -1103,6 +1115,97 @@ var ulidImpl = monotonicFactory();
 function ulid() {
   return ulidImpl();
 }
+var GET_BY_TESTID_RE = /getByTestId\(\s*['"`]([^'"`]+)['"`]\s*\)/g;
+var GET_BY_ROLE_RE = /getByRole\(\s*['"`]([^'"`]+)['"`]\s*(?:,\s*\{[^}]*?name\s*:\s*['"`]([^'"`]+)['"`][^}]*\})?\s*\)/g;
+var GET_BY_TEXT_RE = /getByText\(\s*['"`]([^'"`]+)['"`]/g;
+function countOccurrences(hay, needle) {
+  if (needle.length === 0) return 0;
+  let count = 0;
+  let from = 0;
+  for (; ; ) {
+    const idx = hay.indexOf(needle, from);
+    if (idx < 0) break;
+    count += 1;
+    from = idx + needle.length;
+  }
+  return count;
+}
+function collectA11yNodes(tree, out) {
+  if (tree === null || typeof tree !== "object") return;
+  const node = tree;
+  if (typeof node.role === "string") {
+    out.push({ role: node.role, name: typeof node.name === "string" ? node.name : "" });
+  }
+  if (Array.isArray(node.children)) {
+    for (const child of node.children) collectA11yNodes(child, out);
+  }
+}
+function verifyLocatorsAgainstSnapshot(specContents, snapshot) {
+  if (snapshot === void 0) {
+    return { okCount: 0, problems: [], uncheckedCount: 0 };
+  }
+  const html = snapshot.html ?? "";
+  const truncated = html.includes("<!-- truncated -->");
+  const a11yNodes = [];
+  collectA11yNodes(snapshot.accessibilityTree, a11yNodes);
+  const problems = [];
+  let okCount = 0;
+  let uncheckedCount = 0;
+  for (const m of specContents.matchAll(GET_BY_TESTID_RE)) {
+    const id = m[1];
+    const valueHits = countOccurrences(html, `"${id}"`) + countOccurrences(html, `'${id}'`);
+    if (valueHits === 0) {
+      if (truncated) uncheckedCount += 1;
+      else problems.push({ expression: m[0], problem: "missing", count: 0, kind: "testid" });
+    } else if (valueHits > 1) {
+      problems.push({ expression: m[0], problem: "ambiguous", count: valueHits, kind: "testid" });
+    } else {
+      okCount += 1;
+    }
+  }
+  if (a11yNodes.length > 0) {
+    for (const m of specContents.matchAll(GET_BY_ROLE_RE)) {
+      const role = m[1];
+      const name = m[2];
+      const roleMatches = a11yNodes.filter((n) => n.role === role);
+      if (roleMatches.length === 0) {
+        problems.push({ expression: m[0], problem: "missing", count: 0, kind: "role" });
+      } else if (name !== void 0) {
+        const exact = roleMatches.filter((n) => n.name === name);
+        if (exact.length > 1) {
+          problems.push({
+            expression: m[0],
+            problem: "ambiguous",
+            count: exact.length,
+            kind: "role"
+          });
+        } else {
+          okCount += 1;
+        }
+      } else {
+        uncheckedCount += 1;
+      }
+    }
+  } else {
+    for (const _m of specContents.matchAll(GET_BY_ROLE_RE)) uncheckedCount += 1;
+  }
+  for (const m of specContents.matchAll(GET_BY_TEXT_RE)) {
+    const text = m[1];
+    const hits = countOccurrences(html, text);
+    if (hits === 0) {
+      if (truncated) uncheckedCount += 1;
+      else problems.push({ expression: m[0], problem: "missing", count: 0, kind: "text" });
+    } else {
+      okCount += 1;
+    }
+  }
+  return { okCount, problems, uncheckedCount };
+}
+function summarizeLocatorProblems(v) {
+  return v.problems.map(
+    (p) => p.problem === "missing" ? `${p.expression} \u2014 matched no element on the captured page` : `${p.expression} \u2014 matched ${p.count} elements (ambiguous; narrow it instead of using .nth/.first)`
+  );
+}
 var QaiosError = class extends Error {
   code;
   detail;
@@ -4197,6 +4300,8 @@ function tempForTier(tier) {
   }
 }
 var DEFAULT_LLM_TIMEOUT_MS = 24e4;
+var DEFAULT_MAX_OUTPUT_TOKENS = 16384;
+var MAX_OUTPUT_TOKENS_CEILING = 65536;
 function llmTimeoutMs() {
   const raw = process.env["QAIOS_LLM_TIMEOUT_MS"];
   if (raw === void 0) return DEFAULT_LLM_TIMEOUT_MS;
@@ -4257,6 +4362,7 @@ var SkillRunner = class {
     const maxAttempts = maxRetries + 1;
     let attempt = 0;
     let lastValidationError = null;
+    let currentMaxTokens = DEFAULT_MAX_OUTPUT_TOKENS;
     while (attempt < maxAttempts) {
       attempt++;
       const userPromptBase = skill.userPrompt(effectiveCtx, input);
@@ -4274,9 +4380,9 @@ var SkillRunner = class {
           // but Sonnet 4.6 routinely hits stopReason=max_tokens at
           // that limit — surfacing as a Zod validation error from
           // truncated tool input. Sonnet supports up to 128k output;
-          // 16k is plenty of headroom for v0.1 skill outputs without
-          // burning a runaway loop's worth of cost.
-          maxTokens: 16384,
+          // 16k is the default headroom, escalated on truncation-retry
+          // (see currentMaxTokens) for genuinely large suites.
+          maxTokens: currentMaxTokens,
           temperature: tempForTier(skill.modelTier)
         };
         const signal = buildCallSignal(ctx.cancelSignal);
@@ -4329,14 +4435,21 @@ var SkillRunner = class {
       });
       if (response.stopReason === "max_tokens") {
         if (attempt < maxAttempts) {
+          const escalated = Math.min(currentMaxTokens * 2, MAX_OUTPUT_TOKENS_CEILING);
           ctx.auditLogger.append({
             workflowId: ctx.workflowId,
             phase: skill.phase,
             skillId: skill.id,
             event: "skill.retry",
-            payload: { attempt, reason: "output_truncated_max_tokens" },
+            payload: {
+              attempt,
+              reason: "output_truncated_max_tokens",
+              maxTokensWas: currentMaxTokens,
+              maxTokensNext: escalated
+            },
             actor: "agent"
           });
+          currentMaxTokens = escalated;
           continue;
         }
         ctx.auditLogger.append({
@@ -5237,7 +5350,7 @@ function extractLocatorToken(expr) {
   }
   return null;
 }
-function countOccurrences(haystack, needle) {
+function countOccurrences2(haystack, needle) {
   if (needle.length === 0) return 0;
   let n = 0;
   let i = haystack.indexOf(needle);
@@ -5281,7 +5394,7 @@ function applyLocatorFix(opts) {
   const cwd = opts.cwd ?? process.cwd();
   const absPath = path15.isAbsolute(opts.testFile) ? opts.testFile : path15.resolve(cwd, opts.testFile);
   const oldContents = readFileSync(absPath, "utf-8");
-  const occurrences = countOccurrences(oldContents, opts.fix.oldLocator);
+  const occurrences = countOccurrences2(oldContents, opts.fix.oldLocator);
   if (occurrences === 0) {
     return {
       applied: false,
@@ -6059,12 +6172,65 @@ var writing = async (deps) => {
   const appContext = buildAppContext(deps.args);
   appContext["pomMode"] = typeof deps.args["pomMode"] === "string" && (deps.args["pomMode"] === "off" || deps.args["pomMode"] === "auto" || deps.args["pomMode"] === "on") ? deps.args["pomMode"] : "auto";
   appContext["framework"] = "playwright";
+  const groundUrl = baseUrlFromArgs(deps.args);
+  const groundDisabled = deps.args["noGround"] === true;
+  const groundTimeoutMs = typeof deps.args["groundCaptureTimeoutMs"] === "number" ? deps.args["groundCaptureTimeoutMs"] : 15e3;
+  if (!groundDisabled && groundUrl.length > 0 && deps.capturePageSnapshot !== void 0) {
+    const cwd = typeof deps.args["cwd"] === "string" ? deps.args["cwd"] : process.cwd();
+    try {
+      const snapshot2 = await deps.capturePageSnapshot({
+        url: groundUrl,
+        cwd,
+        timeoutMs: groundTimeoutMs
+      });
+      if (snapshot2 !== null) {
+        appContext["pageSnapshot"] = snapshot2;
+        auditGrounding(deps, "grounding_captured", { url: groundUrl });
+      } else {
+        auditGrounding(deps, "grounding_skipped", {
+          reason: "page capture returned no snapshot",
+          url: groundUrl
+        });
+      }
+    } catch (err) {
+      auditGrounding(deps, "grounding_skipped", {
+        reason: `page capture failed: ${err.message}`,
+        url: groundUrl
+      });
+    }
+  }
   const input = {
     designSpec: designOutput?.["designSpec"] ?? null,
     appContext,
     testDir: typeof deps.args["testDir"] === "string" ? deps.args["testDir"] : "tests"
   };
-  return runSkill(deps, skillId, input, "writing_gate");
+  const result = await runSkill(deps, skillId, input, "writing_gate");
+  const snapshot = appContext["pageSnapshot"];
+  if (snapshot !== void 0) {
+    try {
+      const out = result.skillExecution?.output;
+      const files = Array.isArray(out?.files) ? out.files : [];
+      const specText = files.filter((f) => f.kind === "spec" && typeof f.contents === "string").map((f) => f.contents).join("\n");
+      const verification = verifyLocatorsAgainstSnapshot(
+        specText,
+        snapshot
+      );
+      if (verification.problems.length > 0) {
+        const summary = summarizeLocatorProblems(verification);
+        auditGrounding(deps, "grounding_warning", {
+          problems: summary,
+          okCount: verification.okCount,
+          uncheckedCount: verification.uncheckedCount
+        });
+        result.warnings = [
+          `${summary.length} generated locator(s) don't cleanly match the live page:`,
+          ...summary.map((s) => `  \u2022 ${s}`)
+        ];
+      }
+    } catch {
+    }
+  }
+  return result;
 };
 var writingGate = async (deps) => {
   const writeKey2 = Object.keys(deps.prior).find((k) => k.startsWith("write."));
@@ -6441,6 +6607,16 @@ function auditHeal(deps, kind, payload) {
     actor: "system"
   });
 }
+function auditGrounding(deps, kind, payload) {
+  deps.auditLogger.append({
+    workflowId: deps.workflow.id,
+    phase: "WRITING",
+    skillId: "write.web",
+    event: "skill.completed",
+    payload: { kind: `write.${kind}`, ...payload },
+    actor: "system"
+  });
+}
 function auditDefect(deps, kind, payload) {
   deps.auditLogger.append({
     workflowId: deps.workflow.id,
@@ -6583,7 +6759,11 @@ var Orchestrator = class {
     this.costTracker = new CostTracker(deps.storage.db);
     this.gatesRepo = new GatesRepository(deps.storage.db);
     this.costCap = deps.costCap === null ? null : deps.costCap ?? DEFAULT_COST_CAP;
-    this.capturePageSnapshot = deps.capturePageSnapshot ?? ((o) => capturePageSnapshot({ url: o.url, cwd: o.cwd }));
+    this.capturePageSnapshot = deps.capturePageSnapshot ?? ((o) => capturePageSnapshot({
+      url: o.url,
+      cwd: o.cwd,
+      ...o.timeoutMs !== void 0 ? { timeoutMs: o.timeoutMs } : {}
+    }));
   }
   deps;
   workflowsRepo;
@@ -6977,7 +7157,8 @@ var Orchestrator = class {
           workflowId,
           skillId: result.skillExecution.skillId,
           durationMs: Date.now() - skillStart,
-          output: result.skillExecution.output
+          output: result.skillExecution.output,
+          ...result.warnings !== void 0 && result.warnings.length > 0 ? { warnings: result.warnings } : {}
         });
       }
       if (this.costCap !== null) {
@@ -10045,10 +10226,23 @@ var WriteWebInput = z.object({
     // Defaults to Playwright's own default (`data-testid`); when the app
     // uses something else (e.g. `data-test`), the generated spec must set
     // it via `test.use({ testIdAttribute })` or getByTestId finds nothing.
-    testIdAttribute: z.string().default("data-testid")
+    testIdAttribute: z.string().default("data-testid"),
+    // 0.5: DOM grounding. When the orchestrator has a baseUrl, it
+    // captures the live page (same headless snapshot heal.locator uses)
+    // and threads it in here so the writer generates locators against
+    // REAL elements instead of guessing from the DesignSpec's text hints.
+    // Optional: API tests and no-baseUrl runs omit it and the writer
+    // behaves byte-for-byte as before. Same shape as HealLocatorInput's
+    // pageSnapshot so both paths share the capture infrastructure.
+    pageSnapshot: z.object({
+      url: z.string(),
+      html: z.string().optional(),
+      accessibilityTree: z.unknown().optional()
+    }).optional()
   }),
   testDir: z.string().default("tests")
 });
+var WRITE_WEB_HTML_MAX_CHARS = 5e4;
 var HEADER_PLACEHOLDERS = {
   workflowId: "<ulid will be filled by runtime>",
   generatedAt: "<iso-8601 will be filled by runtime>",
@@ -10090,7 +10284,39 @@ function detectDisallowedImports(contents) {
   }
   return [...specifiers].sort();
 }
-function buildSystemPrompt12(projectMemory, locatorPreference, knownSelectors, testIdAttribute) {
+function truncateHtmlForWriting(html, maxChars = WRITE_WEB_HTML_MAX_CHARS) {
+  if (html.length <= maxChars) return html;
+  return html.slice(0, maxChars) + "\n<!-- truncated -->";
+}
+function buildPageGroundingBlock(snapshot) {
+  if (snapshot === void 0) return "";
+  const html = snapshot.html ? truncateHtmlForWriting(snapshot.html) : "(no HTML captured)";
+  const a11y = snapshot.accessibilityTree !== void 0 ? JSON.stringify(snapshot.accessibilityTree, null, 2) : "(no accessibility tree captured)";
+  return `
+
+REAL PAGE ELEMENTS (captured live from ${snapshot.url}):
+The HTML and accessibility tree below are the ACTUAL target page. Treat them
+as ground truth, overriding any locator hint in the DesignSpec that conflicts.
+- Generate every locator against an element you can SEE here. Do NOT invent
+  test-ids, roles, or text that do not appear below.
+- Prefer a locator that matches EXACTLY ONE element. If the obvious locator
+  matches several (e.g. several elements share the same role+name), narrow it
+  with a containing region (\`getByRole('navigation').getByRole('link', \u2026)\`),
+  a more specific accessible name, or \`.filter({ hasText })\` \u2014 do NOT reach
+  for \`.nth(n)\` or \`.first()\` to paper over an ambiguous match.
+- If a DesignSpec step needs an element that genuinely is NOT on this page,
+  emit \`test.fixme(...)\` with a comment and add the gap to \`qaiosMdSuggestions\`
+  \u2014 do not guess a locator.
+
+Page HTML (truncated):
+\`\`\`html
+${html}
+\`\`\`
+
+Accessibility tree:
+${a11y}`;
+}
+function buildSystemPrompt12(projectMemory, locatorPreference, knownSelectors, testIdAttribute, pageSnapshot) {
   const knownSelectorsRendered = knownSelectors.length === 0 ? "(none)" : knownSelectors.join(", ");
   const testIdBlock = !testIdAttribute || testIdAttribute === "data-testid" ? "" : `
 - TEST-ID ATTRIBUTE: this project's test ids use the \`${testIdAttribute}\` attribute,
@@ -10147,7 +10373,7 @@ Project context:
 ${projectMemory}
 
 Locator preference order: ${locatorPreference.join(", ")}
-Known selectors (use these where applicable): ${knownSelectorsRendered}
+Known selectors (use these where applicable): ${knownSelectorsRendered}${buildPageGroundingBlock(pageSnapshot)}
 
 Output via the \`submit\` tool.`;
 }
@@ -10175,7 +10401,8 @@ var writeWebSkill = {
     ctx.projectMemory,
     input.appContext.locatorPreference,
     input.appContext.knownSelectors,
-    input.appContext.testIdAttribute
+    input.appContext.testIdAttribute,
+    input.appContext.pageSnapshot
   ),
   userPrompt: (_ctx, input) => buildUserPrompt12(input),
   confidenceFn: (output, ctx) => {
@@ -10187,7 +10414,21 @@ var writeWebSkill = {
     const scan = scanGeneratedTests(output.files, designedIds);
     const assertionsPresent = scan.specFileCount === 0 || scan.specFilesWithoutAssertions.length === 0 ? 1 : 0.6;
     const idsInContents = scan.unreferencedScenarioIds.length === 0 ? 1 : 0.7;
-    return Math.max(0, Math.min(1, reported * noSilentDrops * assertionsPresent * idsInContents));
+    const snapshot = ctx.input.appContext.pageSnapshot;
+    let locatorsVerified = 1;
+    if (snapshot !== void 0) {
+      const allSpecText = output.files.filter((f) => f.kind === "spec").map((f) => f.contents).join("\n");
+      const v = verifyLocatorsAgainstSnapshot(allSpecText, snapshot);
+      const checked = v.okCount + v.problems.length;
+      if (checked > 0 && v.problems.length > 0) {
+        const badRatio = v.problems.length / checked;
+        locatorsVerified = Math.max(0.5, 1 - badRatio);
+      }
+    }
+    return Math.max(
+      0,
+      Math.min(1, reported * noSilentDrops * assertionsPresent * idsInContents * locatorsVerified)
+    );
   }
 };
 var WriteApiInput = z.object({
@@ -14014,7 +14255,9 @@ function riskHotspotsFromMemory2(mem) {
 var GLYPH6 = {
   ok: "\u2713",
   failed: "\u2717",
-  blocked: "\u23F8"};
+  blocked: "\u23F8",
+  warn: "\u26A0"
+};
 function renderTextLine4(event) {
   switch (event.kind) {
     case "workflow_started":
@@ -14024,8 +14267,13 @@ function renderTextLine4(event) {
       const seconds = (event.durationMs / 1e3).toFixed(1);
       const conf = readConfidence2(event.output);
       const confSuffix = conf !== null ? ` (confidence ${conf.toFixed(2)})` : "";
-      return `\u25B8 ${event.skillId.padEnd(22)} ${GLYPH6.ok} ${seconds}s${confSuffix}
+      let line = `\u25B8 ${event.skillId.padEnd(22)} ${GLYPH6.ok} ${seconds}s${confSuffix}
+`;
+      if (Array.isArray(event.warnings) && event.warnings.length > 0) {
+        line += `  ${GLYPH6.warn} ${event.warnings.join("\n  ")}
 `;
+      }
+      return line;
     }
     case "gate_evaluated":
       return event.decision === "pass" ? `\u25B8 gate.${event.gateType.padEnd(16)} ${GLYPH6.ok} passed
@@ -14172,6 +14420,11 @@ ${epSummary}`;
     riskHotspots: riskHotspotsFromMemory2(memory),
     knownSelectors: []
   };
+  if (testType === "web") {
+    const groundEnabled = opts.noGround === true ? false : testCfg?.groundLocators ?? true;
+    if (!groundEnabled) args["noGround"] = true;
+    args["groundCaptureTimeoutMs"] = testCfg?.groundCaptureTimeoutMs ?? 15e3;
+  }
   if (testType === "api" && parsedSpec !== null) {
     args["apiContext"] = {
       baseUrl: parsedSpec.baseUrl,
@@ -14465,7 +14718,10 @@ function buildProgram() {
     "--scope <scope>",
     "design scope: minimal (~3 scenarios) | standard (default) | exhaustive (~15-25 scenarios)",
     "standard"
-  ).option("--run", "execute tests immediately after generating").option("--out <path>", "override default test output directory").option("--feature-name <name>", "override auto-detected feature name").option("--yes", "auto-approve any blocking gate for this run (non-interactive)").option(
+  ).option("--run", "execute tests immediately after generating").option(
+    "--no-ground",
+    "skip DOM grounding (do not capture the live page before writing locators)"
+  ).option("--out <path>", "override default test output directory").option("--feature-name <name>", "override auto-detected feature name").option("--yes", "auto-approve any blocking gate for this run (non-interactive)").option(
     "--workflow <id>",
     "resume a prior workflow interrupted mid-run instead of starting fresh"
   ).action(async (description, cmdOpts, command) => {
@@ -14481,6 +14737,7 @@ function buildProgram() {
     };
     if (typeof cmdOpts["apiSpec"] === "string") opts.apiSpec = cmdOpts["apiSpec"];
     if (typeof cmdOpts["baseUrl"] === "string") opts.baseUrl = cmdOpts["baseUrl"];
+    if (cmdOpts["ground"] === false) opts.noGround = true;
     if (typeof cmdOpts["out"] === "string") opts.out = cmdOpts["out"];
     if (typeof cmdOpts["featureName"] === "string") opts.featureName = cmdOpts["featureName"];
     if (cmdOpts["yes"] === true) opts.yes = true;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@qatonic_innovations/qaios",
-  "version": "0.4.1",
+  "version": "0.5.1",
   "type": "module",
   "description": "AI QA engineer in your terminal — designs, writes, runs, heals, and explores tests for web UI and APIs with audit-grade traceability.",
   "license": "MIT",