@keber/qa-framework 1.10.0 → 1.12.0

package/integrations/playwright/README.md

@@ -1,4 +1,4 @@
-# Integration — Playwright
+# Integration - Playwright
 
 This directory documents how `@playwright/test` is configured within the
 `keber/qa-framework` opinionated setup.
@@ -40,9 +40,9 @@ suite execution.
 
 ## Debugging checklist
 
-1. `PWDEBUG=1 npx playwright test` — step through in inspector
-2. `--headed` — watch the browser
-3. `--trace on` — record full trace; open with `npx playwright show-trace`
+1. `PWDEBUG=1 npx playwright test` - step through in inspector
+2. `--headed` - watch the browser
+3. `--trace on` - record full trace; open with `npx playwright show-trace`
 4. Increase `actionTimeout` if the app has slow server-side rendering
 5. Add `await page.waitForLoadState('networkidle')` before assertions on
    dynamically loaded content
@@ -66,3 +66,127 @@ test.describe('[TC-M-S-NNN] Module > Submodule @P0', () => {
   });
 });
 ```
+
+## Visual AI helper (optional advanced pattern)
+
+> **Advanced / optional.** This pattern is not part of the default scaffold and requires
+> an AI vision API key. Use it when a selector-based assertion is too brittle or the
+> element structure changes frequently (e.g. PDF previews, dashboard charts, export outputs).
+
+Instead of asserting specific selectors, take a screenshot and ask an AI vision model to
+verify that the expected content is visually present. This decouples the assertion from DOM
+structure and is useful for generated documents or complex layouts.
+
+```typescript
+// fixtures/vision-helper.ts
+import * as https from 'https';
+
+export interface VisionResult {
+  passes: boolean;
+  reason: string;
+  raw?: string;
+}
+
+/**
+ * Sends a screenshot to an AI vision model and asks whether the described
+ * content is visually present. Degrades gracefully when the API key is absent.
+ *
+ * Requires an API key in the environment (e.g. OPENAI_API_KEY for GPT-4o,
+ * or adapt the request body for any other vision-capable model).
+ */
+export async function assertScreenshotShowsContent(
+  screenshotBuffer: Buffer,
+  contentDescription: string,
+  options: { model?: string; degradeOnMissingKey?: boolean } = {}
+): Promise<VisionResult> {
+  const apiKey = process.env.AI_VISION_API_KEY;
+  const degrade = options.degradeOnMissingKey ?? true;
+
+  if (!apiKey) {
+    if (degrade) {
+      console.warn('[vision-helper] AI_VISION_API_KEY not set. Skipping visual assertion.');
+      return { passes: true, reason: 'SKIPPED: AI_VISION_API_KEY not configured' };
+    }
+    return { passes: false, reason: 'AI_VISION_API_KEY not set in environment' };
+  }
+
+  const base64Image = screenshotBuffer.toString('base64');
+  const model = options.model ?? 'gpt-4o';
+
+  const prompt = `You are a QA assistant verifying screenshots of a web application.
+Answer YES or NO: Does the screenshot clearly show ${contentDescription}?
+Reply in exactly this format:
+VERDICT: YES
+REASON: <one short sentence>`;
+
+  const body = JSON.stringify({
+    model,
+    max_tokens: 150,
+    messages: [{
+      role: 'user',
+      content: [
+        { type: 'text', text: prompt },
+        { type: 'image_url', image_url: { url: `data:image/png;base64,${base64Image}`, detail: 'low' } },
+      ],
+    }],
+  });
+
+  const raw = await new Promise<string>((resolve, reject) => {
+    const req = https.request({
+      hostname: 'api.openai.com',
+      path: '/v1/chat/completions',
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${apiKey}`,
+        'Content-Length': Buffer.byteLength(body),
+      },
+    }, (res) => {
+      let data = '';
+      res.on('data', (chunk) => { data += chunk; });
+      res.on('end', () => resolve(data));
+    });
+    req.on('error', reject);
+    req.write(body);
+    req.end();
+  });
+
+  const parsed = JSON.parse(raw);
+  const text: string = parsed.choices?.[0]?.message?.content ?? '';
+  const verdict = text.match(/VERDICT:\s*(YES|NO)/i);
+  const reason  = text.match(/REASON:\s*(.+)/i);
+
+  if (!verdict) return { passes: false, reason: `Unexpected response: ${text}`, raw: text };
+  return {
+    passes: verdict[1].toUpperCase() === 'YES',
+    reason: reason?.[1]?.trim() ?? text,
+    raw: text,
+  };
+}
+```
+
+**Usage in a test:**
+
+```typescript
+import { assertScreenshotShowsContent } from '../fixtures/vision-helper';
+
+test('[TC-MOD-001] Export PDF contains expected content', async ({ page }) => {
+  // ... trigger export ...
+  const screenshot = await page.screenshot();
+  const result = await assertScreenshotShowsContent(
+    screenshot,
+    'a completed export confirmation message with a download link'
+  );
+  expect(result.passes, result.reason).toBe(true);
+});
+```
+
+**Setup notes:**
+
+- Copy `vision-helper.ts` into your `fixtures/` directory.
+- Set `AI_VISION_API_KEY` in your `.env` file (not committed). The example uses the OpenAI
+  API; adapt the request body for any other vision model.
+- By default, the helper degrades to a skip (passes: true) when the key is absent, so CI
+  runs without the key do not fail on this assertion.
+- `pdf-parse` (v2+) exports a `PDFParse` class rather than a function. Use
+  `new PDFParse({ data: buffer })` and call `.getText()` on the instance.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@keber/qa-framework",
-  "version": "1.10.0",
+  "version": "1.12.0",
   "description": "Reusable spec-driven QA framework for IDE-agent-assisted automated testing. Installable as an npm package. Provides structure, templates, agent instructions, and optional integrations for Playwright and Azure DevOps.",
   "keywords": [
     "qa",
@@ -21,7 +21,11 @@
     "postinstall": "node scripts/init.js --skip-if-exists",
     "init": "node scripts/cli.js init",
     "generate": "node scripts/cli.js generate",
-    "validate": "node scripts/cli.js validate"
+    "validate": "node scripts/cli.js validate",
+    "check-accents": "node scripts/check-spanish-accents.js",
+    "check-chars": "node scripts/check-forbidden-chars.js",
+    "test": "node --test test/*.test.js",
+    "version": "node scripts/sync-version.js && git add README.md .github/copilot-instructions.md"
   },
   "files": [
     "agent-instructions/",
@@ -36,12 +40,11 @@
     "qa-framework.config.json"
   ],
   "engines": {
-    "node": ">=18"
+    "node": ">=20"
   },
-  "devDependencies": {},
   "peerDependencies": {
-    "@playwright/test": ">=1.40.0",
-    "@keber/ado-qa": ">=1.0.0"
+    "@keber/ado-qa": ">=1.0.0",
+    "@playwright/test": ">=1.40.0"
   },
   "peerDependenciesMeta": {
     "@playwright/test": {

package/qa-framework.config.json

@@ -1,6 +1,6 @@
 {
   "$schema": "./qa-framework.config.schema.json",
-  "frameworkVersion": "1.1.3",
+  "frameworkVersion": "1.10.0",
 
   "project": {
     "name": "{{PROJECT_NAME}}",
@@ -27,7 +27,7 @@
     "testCaseIdPattern": "TC-{MODULE}-{SUBMODULE}-{NUM}",
     "businessRuleIdPattern": "RN-{MODULE}-{NUM}",
     "workflowIdPattern": "FL-{MODULE}-{NUM}",
-    "defectIdPattern": "DEF-{NUM}",
+    "defectIdPattern": "DEF-{PREFIX}-{NUM}",
     "testNamingPattern": "[{ID}] {title} @{priority}",
     "timestampFormat": "YYYY-MM-DD_HH-MM-SS",
     "tcPriorityLevels": ["P0", "P1", "P2", "P3"]
@@ -42,6 +42,20 @@
     }
   ],
 
+  "parallelLanes": {
+    "enabled": false,
+    "lanes": [
+      {
+        "id": "1",
+        "account": "QA_USER",
+        "storageState": ".auth/user-default.json",
+        "mcpNamespace": null,
+        "runnerProject": "chromium",
+        "capabilities": []
+      }
+    ]
+  },
+
   "integrations": {
     "playwright": {
       "enabled": false,
@@ -69,7 +83,14 @@
       "variableGroup": "{{ADO_VARIABLE_GROUP}}",
       "pipelineFile": "qa/08-azure-integration/pipelines/azure-pipeline-qa.yml",
       "moduleRegistry": "qa/08-azure-integration/module-registry.json",
-      "reporterPackage": "@alex_neo/playwright-azure-reporter"
+      "reporterPackage": "@alex_neo/playwright-azure-reporter",
+      "sprintCycle": {
+        "enabled": false,
+        "sprintDurationDays": 8,
+        "manualTestingTimeboxDays": 2,
+        "dateFormat": "dd-mm-aaaa",
+        "timezone": "America/Santiago"
+      }
     }
   },
 

package/scripts/check-forbidden-chars.js

@@ -0,0 +1,187 @@
+#!/usr/bin/env node
+/**
+ * check-forbidden-chars.js
+ *
+ * Enforces the BLOCKING character-safety rule declared in
+ * .github/copilot-instructions.md: generated content must never carry typographic
+ * characters that survive a copy-paste but break on the way to a terminal, a CSV,
+ * a PowerShell script or an Azure DevOps field.
+ *
+ * Forbidden: em-dash (U+2014), en-dash (U+2013), horizontal ellipsis (U+2026),
+ * smart quotes (U+201C, U+201D, U+2018, U+2019) and arrows (U+2190-U+21FF).
+ * Each has an ASCII equivalent that renders identically everywhere.
+ *
+ * Latin Extended (U+00C0-U+024F) is explicitly NOT flagged: Spanish accents,
+ * n with tilde and u/o with umlaut are required orthography, not decoration.
+ * Emoji are not flagged either - they are outside the forbidden set.
+ *
+ * Unlike check-spanish-accents.js this scans .js and .ts as well as .md, because
+ * the generators under scripts/ embed these characters in template literals and
+ * console output that propagate into every consuming project.
+ *
+ * Usage:
+ *   node scripts/check-forbidden-chars.js <file|dir> [...]   check paths
+ *   node scripts/check-forbidden-chars.js --staged           check staged files
+ *   node scripts/check-forbidden-chars.js --report <paths>   print findings, never fail
+ *
+ * Exit codes: 0 = clean, 1 = at least one forbidden character, 2 = usage error.
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { execFileSync } = require('child_process');
+
+// '.example' covers .env.example, which ships to consuming projects like any other
+// template. A BOM or an em-dash there is not cosmetic: some .env parsers read the
+// first key as QA_BASE_URL and silently fail to find it.
+const EXTENSIONS = ['.md', '.js', '.ts', '.ps1', '.example'];
+
+// Code point -> { name, replacement }. Arrows are handled by range below.
+const FORBIDDEN = {
+  0x2014: { name: 'em dash', replacement: ' - ' },
+  0x2013: { name: 'en dash', replacement: ' - ' },
+  0x2026: { name: 'horizontal ellipsis', replacement: '...' },
+  0x201c: { name: 'left double quotation mark', replacement: '"' },
+  0x201d: { name: 'right double quotation mark', replacement: '"' },
+  0x2018: { name: 'left single quotation mark', replacement: "'" },
+  0x2019: { name: 'right single quotation mark', replacement: "'" },
+};
+
+const ARROW_START = 0x2190;
+const ARROW_END = 0x21ff;
+
+// The two arrows with an obvious ASCII spelling; every other arrow in the block
+// has no single equivalent, so the report asks for a rewrite instead of guessing.
+const ARROW_REPLACEMENTS = {
+  0x2190: '<-',
+  0x2192: '->',
+};
+
+function describe(codePoint) {
+  if (FORBIDDEN[codePoint]) return FORBIDDEN[codePoint];
+  if (codePoint >= ARROW_START && codePoint <= ARROW_END) {
+    return {
+      name: 'arrow',
+      replacement: ARROW_REPLACEMENTS[codePoint] || 'an ASCII arrow such as -> or <-',
+    };
+  }
+  return null;
+}
+
+function hex(codePoint) {
+  return 'U+' + codePoint.toString(16).toUpperCase().padStart(4, '0');
+}
+
+/**
+ * Scan one file's text. Returns an array of findings; column is 1-based and
+ * counted in code points, so an astral character earlier on the line does not
+ * shift the reported position.
+ */
+function scanText(text, file) {
+  const findings = [];
+  const lines = text.split(/\r?\n/);
+
+  lines.forEach((line, lineIndex) => {
+    let column = 0;
+    for (const char of line) {
+      column += 1;
+      const codePoint = char.codePointAt(0);
+      const match = describe(codePoint);
+      if (!match) continue;
+      findings.push({
+        file,
+        line: lineIndex + 1,
+        column,
+        char,
+        codePoint,
+        name: match.name,
+        replacement: match.replacement,
+        text: line.trim(),
+      });
+    }
+  });
+
+  return findings;
+}
+
+function scanFile(file) {
+  return scanText(fs.readFileSync(file, 'utf8'), file);
+}
+
+// Archived copies of content this framework shipped in an earlier release. Their bytes
+// are the point: test/upgrade.test.js matches them against SHIPPED_SCAFFOLD_HASHES to
+// prove a real legacy installation is recognised as pristine. Rewriting a character in
+// one of them changes its hash and silently voids that proof, so they are read-only
+// history rather than authored content and the rule does not apply to them.
+const SKIP_DIRS = new Set(['node_modules', '.git', 'legacy-v1.11.3']);
+
+function collect(target) {
+  const stat = fs.statSync(target);
+  if (stat.isFile()) return EXTENSIONS.includes(path.extname(target)) ? [target] : [];
+  return fs.readdirSync(target).flatMap((entry) => {
+    if (SKIP_DIRS.has(entry)) return [];
+    return collect(path.join(target, entry));
+  });
+}
+
+function stagedFiles() {
+  const out = execFileSync(
+    'git',
+    ['diff', '--cached', '--name-only', '--diff-filter=ACM'],
+    { encoding: 'utf8' }
+  );
+  return out
+    .split('\n')
+    .map((s) => s.trim())
+    .filter((s) => s && EXTENSIONS.includes(path.extname(s)) && fs.existsSync(s));
+}
+
+function main(argv) {
+  const reportOnly = argv.includes('--report');
+  const args = argv.filter((a) => a !== '--report');
+
+  let files;
+  if (args.includes('--staged')) {
+    files = stagedFiles();
+  } else if (args.length) {
+    files = args.flatMap(collect);
+  } else {
+    console.error('usage: check-forbidden-chars.js <file|dir>... | --staged [--report]');
+    return 2;
+  }
+
+  const findings = files.flatMap(scanFile);
+
+  if (findings.length) {
+    const stream = reportOnly ? console.log : console.error;
+    stream(`\nForbidden characters (${findings.length} occurrence(s)):\n`);
+    for (const f of findings) {
+      stream(`  ${f.file}:${f.line}:${f.column}  "${f.char}" ${hex(f.codePoint)} ${f.name}`);
+      stream(`    use ${f.replacement}`);
+      stream(`    ${f.text}`);
+    }
+    if (!reportOnly) {
+      stream(
+        '\nThese characters break on the way to a terminal, a CSV export or an Azure' +
+        '\nDevOps field. Replace them with their ASCII equivalents rather than' +
+        '\nbypassing this check. Spanish accents are permitted and are never flagged.\n'
+      );
+      return 1;
+    }
+    return 0;
+  }
+
+  if (reportOnly) {
+    console.log(`check-forbidden-chars: ${files.length} file(s) checked, no findings.`);
+    return 0;
+  }
+
+  console.log(`check-forbidden-chars: ${files.length} file(s) checked, no violations.`);
+  return 0;
+}
+
+if (require.main === module) process.exit(main(process.argv.slice(2)));
+
+module.exports = { scanText, scanFile, main, EXTENSIONS, FORBIDDEN, ARROW_START, ARROW_END };

package/scripts/check-spanish-accents.js

@@ -0,0 +1,159 @@
+#!/usr/bin/env node
+/**
+ * check-spanish-accents.js
+ *
+ * Fails when a Markdown file whose prose is Spanish carries no accented characters.
+ *
+ * Why this exists: a spec written as "Codigo" while the UI renders "Código" produces a
+ * page-object selector that never matches. In one project that cost 24 of 39 failing smoke
+ * tests and two wrong mitigations (raising a timeout, forcing HTTP/1.1) before the real
+ * cause was found, because the failure presents as a timeout, not as a text mismatch.
+ * Accent stripping is a defect, not a house style - and its measured incidence in
+ * agent-generated artifacts is high enough that honor-system compliance does not hold.
+ *
+ * The hard part is not detecting missing accents; it is deciding which files are Spanish.
+ * A naive "every .md must contain accents" rule fails every legitimately English artifact
+ * the framework ships. Counting Spanish marker words in raw text does not separate them
+ * either: an English document about a Spanish-language project quotes enough Spanish paths,
+ * identifiers and snippets to cross any absolute threshold that still catches short
+ * Spanish files.
+ *
+ * So this classifies on the density of Spanish function words in PROSE ONLY, after
+ * stripping fenced code, inline code, link targets, HTML tags and table pipes. English
+ * prose that merely cites Spanish identifiers scores low because those citations live in
+ * the stripped regions; real Spanish prose scores high because its function words are
+ * spread through the sentences themselves.
+ *
+ * Usage:
+ *   node scripts/check-spanish-accents.js <file|dir> [...]   check paths
+ *   node scripts/check-spanish-accents.js --staged           check staged .md files
+ *   node scripts/check-spanish-accents.js --report <paths>   print scores, never fail
+ *
+ * Exit codes: 0 = clean, 1 = at least one Spanish file has no accents, 2 = usage error.
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { execFileSync } = require('child_process');
+
+// Function words that are unavoidable in real Spanish prose and rare in English text.
+// Deliberately excludes "no", "a", "en", "es" and other tokens that collide with English
+// or with code, and excludes accented forms so a stripped file is not penalised twice.
+const MARKERS = [
+  'que', 'para', 'con', 'por', 'los', 'las', 'del', 'una', 'como', 'pero',
+  'este', 'esta', 'estos', 'estas', 'sin', 'sobre', 'desde', 'cuando', 'donde',
+  'cada', 'todo', 'todos', 'toda', 'todas', 'entre', 'hasta', 'porque', 'segun',
+  'debe', 'deben', 'ser', 'son', 'hay', 'muy', 'ya', 'si', 'lo', 'le', 'se',
+];
+
+const ACCENTED = /[áéíóúÁÉÍÓÚñÑüÜ]/;
+
+// A file is Spanish when at least this share of its prose words are Spanish markers.
+// Calibrated against this package's own corpus: Spanish agent templates land near 0.13,
+// English skill files and the FRAMEWORK-FIXES reports stay below 0.03.
+const DENSITY_THRESHOLD = 0.07;
+// Below this many prose words the density estimate is too noisy to act on.
+const MIN_PROSE_WORDS = 60;
+
+/** Remove every region where Spanish tokens are citations rather than prose. */
+function extractProse(markdown) {
+  return markdown
+    .replace(/^---\n[\s\S]*?\n---/, ' ')      // YAML frontmatter
+    .replace(/```[\s\S]*?```/g, ' ')          // fenced code
+    .replace(/~~~[\s\S]*?~~~/g, ' ')          // fenced code, alt syntax
+    .replace(/`[^`\n]*`/g, ' ')               // inline code
+    .replace(/\]\([^)]*\)/g, '] ')            // link targets, keep link text
+    .replace(/<[^>\n]*>/g, ' ')               // HTML tags and <placeholders>
+    .replace(/\{\{[^}]*\}\}/g, ' ')           // {{TEMPLATE_PLACEHOLDERS}}
+    .replace(/\|/g, ' ')                      // table pipes
+    .replace(/^\s{4,}\S.*$/gm, ' ')           // indented code blocks
+    .replace(/[A-Za-z0-9_.\-]+\/[A-Za-z0-9_./\-]+/g, ' ') // bare paths
+    .replace(/[#*_>[\]]/g, ' ');              // remaining markdown punctuation
+}
+
+function score(markdown) {
+  const prose = extractProse(markdown);
+  const words = prose.toLowerCase().match(/[a-zà-ÿ]+/g) || [];
+  const markers = words.filter((w) => MARKERS.includes(w)).length;
+  const density = words.length ? markers / words.length : 0;
+  return {
+    words: words.length,
+    markers,
+    density,
+    isSpanish: words.length >= MIN_PROSE_WORDS && density >= DENSITY_THRESHOLD,
+    hasAccents: ACCENTED.test(markdown),
+  };
+}
+
+function collect(target) {
+  const stat = fs.statSync(target);
+  if (stat.isFile()) return target.endsWith('.md') ? [target] : [];
+  return fs.readdirSync(target).flatMap((entry) => {
+    if (entry === 'node_modules' || entry === '.git') return [];
+    return collect(path.join(target, entry));
+  });
+}
+
+function stagedMarkdown() {
+  const out = execFileSync(
+    'git',
+    ['diff', '--cached', '--name-only', '--diff-filter=ACM', '--', '*.md'],
+    { encoding: 'utf8' }
+  );
+  return out.split('\n').map((s) => s.trim()).filter((s) => s && fs.existsSync(s));
+}
+
+function main(argv) {
+  const reportOnly = argv.includes('--report');
+  const args = argv.filter((a) => a !== '--report');
+
+  let files;
+  if (args.includes('--staged')) {
+    files = stagedMarkdown();
+  } else if (args.length) {
+    files = args.flatMap(collect);
+  } else {
+    console.error('usage: check-spanish-accents.js <file|dir>... | --staged [--report]');
+    return 2;
+  }
+
+  const failures = [];
+  for (const file of files) {
+    const result = score(fs.readFileSync(file, 'utf8'));
+    if (reportOnly) {
+      console.log(
+        `${result.isSpanish ? 'ES' : 'en'}  density=${result.density.toFixed(3)}  ` +
+        `words=${String(result.words).padStart(5)}  accents=${result.hasAccents ? 'yes' : 'NO '}  ${file}`
+      );
+    }
+    if (result.isSpanish && !result.hasAccents) failures.push({ file, ...result });
+  }
+
+  if (reportOnly) return 0;
+
+  if (failures.length) {
+    console.error('\nSpanish Markdown without accented characters:\n');
+    for (const f of failures) {
+      console.error(`  ${f.file}`);
+      console.error(
+        `    ${f.markers} Spanish markers in ${f.words} prose words ` +
+        `(density ${f.density.toFixed(3)}), 0 accented characters.`
+      );
+    }
+    console.error(
+      '\nSpanish prose must carry its accents: a spec saying "Codigo" does not match a UI' +
+      '\nthat renders "Código", and the resulting selector failure looks like a timeout.' +
+      '\nFix the orthography rather than bypassing this check.\n'
+    );
+    return 1;
+  }
+
+  console.log(`check-spanish-accents: ${files.length} file(s) checked, no violations.`);
+  return 0;
+}
+
+if (require.main === module) process.exit(main(process.argv.slice(2)));
+
+module.exports = { score, extractProse, MARKERS, DENSITY_THRESHOLD };

package/scripts/cli.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 /**
- * scripts/cli.js — keber/qa-framework CLI entry point
+ * scripts/cli.js - keber/qa-framework CLI entry point
  *
  * Usage:
  *   npx keber/qa-framework <command> [options]

package/scripts/generate.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 /**
- * scripts/generate.js — Generate a qa artifact from a template
+ * scripts/generate.js - Generate a qa artifact from a template
  *
  * Usage:
  *   qa-framework generate <artifact> [options]
@@ -39,7 +39,7 @@ const outputDir     = getArg('--output')    ? path.resolve(process.cwd(), getArg
 const templateDir = path.resolve(__dirname, '..', 'templates');
 
 const ARTIFACTS = {
-  spec:             null,    // special — copies all 6 files
+  spec:             null,    // special - copies all 6 files
   'test-plan':      path.join(templateDir, 'test-plan.md'),
   'test-case':      path.join(templateDir, 'test-case.md'),
   'execution-report': path.join(templateDir, 'execution-report.md'),