@keber/qa-framework 1.10.0 → 1.11.3

package/templates/integration-scaffold/.env.example

@@ -0,0 +1,40 @@
+# -----------------------------------------------------------------------
+# Integration test environment variables
+# -----------------------------------------------------------------------
+# Copy this file to .env and fill in real values.
+# NEVER commit .env to source control.
+# -----------------------------------------------------------------------
+
+# -----------------------------------------------------------------------
+# Application under test
+# -----------------------------------------------------------------------
+QA_BASE_URL=https://your-app.qa.example.com
+
+# -----------------------------------------------------------------------
+# Auth - choose one strategy
+# -----------------------------------------------------------------------
+
+# Strategy 1: Static bearer token (preferred for pure API services)
+# QA_API_TOKEN=your-api-token-here
+
+# Strategy 2: Browser login (for apps with cookie/session auth)
+# QA_USER_EMAIL=qa-user@example.com
+# QA_USER_PASSWORD=CHANGE_ME
+
+# -----------------------------------------------------------------------
+# Login form selectors (used only when QA_API_TOKEN is not set)
+# -----------------------------------------------------------------------
+# QA_LOGIN_PATH=/login
+# QA_LOGIN_EMAIL_SELECTOR=input[type="email"]
+# QA_LOGIN_PASSWORD_SELECTOR=input[type="password"]
+# QA_LOGIN_SUBMIT_SELECTOR=button[type="submit"]
+# QA_LOGIN_SUCCESS_SELECTOR=.dashboard
+
+# -----------------------------------------------------------------------
+# Azure DevOps integration (optional)
+# -----------------------------------------------------------------------
+# ADO_ORG=your-org
+# ADO_PROJECT=YourProject
+# ADO_PLAN_ID=00000
+# ADO_PAT=your-pat-here
+# SYSTEM_ACCESSTOKEN=   # Set automatically in Azure Pipelines

package/templates/integration-scaffold/README.md

@@ -0,0 +1,77 @@
+# API Integration Tests
+
+Playwright-based API test project for `qa/07-automation/integration/`.
+
+Tests run without a browser. All tests use Playwright's `APIRequestContext`
+to call HTTP endpoints directly. Auth state is established once by
+`global-setup.ts` and reused across the test run.
+
+## Requirements
+
+- Node.js 18+
+- `@playwright/test` (installed via `npm install` in this directory)
+
+## Setup
+
+```bash
+cd qa/07-automation/integration
+npm install
+cp .env.example .env
+# Edit .env and fill in QA_BASE_URL and auth credentials
+```
+
+Add the following to your root `.gitignore`:
+
+```
+qa/07-automation/integration/.env
+qa/07-automation/integration/.auth/
+qa/07-automation/integration/test-results/
+qa/07-automation/integration/playwright-report/
+```
+
+## Environment variables
+
+| Variable | Required | Description |
+|---|---|---|
+| `QA_BASE_URL` | Yes | Base URL of the API under test |
+| `QA_API_TOKEN` | One of | Static bearer token for token-based APIs |
+| `QA_USER_EMAIL` | One of | Email/username for browser-login APIs |
+| `QA_USER_PASSWORD` | One of | Password for browser-login APIs |
+
+See `.env.example` for all supported variables.
+
+## Running tests
+
+```bash
+# All tests
+npm test
+
+# CI mode (retries enabled)
+npm run test:ci
+
+# Open HTML report
+npm run report
+```
+
+## Writing tests
+
+Import from `@playwright/test` and use the `request` fixture:
+
+```typescript
+import { test, expect } from '@playwright/test';
+
+test('GET /users returns 200', async ({ request }) => {
+  const response = await request.get('/users');
+  expect(response.status()).toBe(200);
+});
+```
+
+For token-based APIs that require an `Authorization` header, pass it per request:
+
+```typescript
+const response = await request.get('/api/protected', {
+  headers: { Authorization: `Bearer ${process.env.QA_API_TOKEN}` },
+});
+```
+
+See `tests/example.spec.ts` for a starter test to replace.

package/templates/integration-scaffold/global-setup.ts

@@ -0,0 +1,99 @@
+/**
+ * global-setup.ts - Integration (API-only) project
+ *
+ * Runs once before all test files to establish auth state.
+ *
+ * Two strategies, tried in order:
+ *   1. Token-based: QA_API_TOKEN is set -> write a minimal storageState with the
+ *      token in localStorage so the `request` fixture picks it up automatically.
+ *   2. Browser login: QA_USER_EMAIL + QA_USER_PASSWORD are set -> launch Chromium,
+ *      perform the login flow, and save the resulting storageState to
+ *      .auth/api-session.json.
+ *
+ * Playwright's `storageState` mechanism handles cookies and localStorage for
+ * APIRequestContext automatically when `use.storageState` is configured.
+ * For token-only APIs that require an Authorization header, set the header
+ * directly in each test or extend the `request` fixture in fixtures/base.ts.
+ */
+
+import * as fs   from 'fs';
+import * as path from 'path';
+import { chromium } from '@playwright/test';
+import * as dotenv  from 'dotenv';
+
+dotenv.config();
+
+const AUTH_FILE = path.join(__dirname, '.auth', 'api-session.json');
+
+export default async function globalSetup(): Promise<void> {
+  fs.mkdirSync(path.dirname(AUTH_FILE), { recursive: true });
+
+  if (process.env.QA_API_TOKEN) {
+    await setupWithToken(process.env.QA_API_TOKEN, process.env.QA_BASE_URL!);
+    return;
+  }
+
+  await setupWithBrowserLogin();
+}
+
+/**
+ * Write a minimal storageState that carries the bearer token in localStorage.
+ * Replace the localStorage key/value format if your app uses a different scheme
+ * (e.g. a cookie, or a different localStorage key name).
+ */
+async function setupWithToken(token: string, baseURL: string): Promise<void> {
+  const origin = baseURL.replace(/\/$/, '');
+
+  const state = {
+    cookies: [],
+    origins: [
+      {
+        origin,
+        localStorage: [
+          { name: 'qa_api_token', value: token },
+        ],
+      },
+    ],
+  };
+
+  fs.writeFileSync(AUTH_FILE, JSON.stringify(state, null, 2), 'utf-8');
+  console.log('[integration-setup] Token-based auth written to api-session.json');
+}
+
+/**
+ * Perform a browser login and save the resulting cookies + localStorage to disk.
+ *
+ * Configure login form selectors via env vars (see .env.example) or update the
+ * defaults below to match your application's login page.
+ */
+async function setupWithBrowserLogin(): Promise<void> {
+  const baseURL          = process.env.QA_BASE_URL!.replace(/\/$/, '');
+  const email            = process.env.QA_USER_EMAIL!;
+  const password         = process.env.QA_USER_PASSWORD!;
+  const loginPath        = process.env.QA_LOGIN_PATH              ?? '/login';
+  const emailSelector    = process.env.QA_LOGIN_EMAIL_SELECTOR    ?? 'input[type="email"]';
+  const passwordSelector = process.env.QA_LOGIN_PASSWORD_SELECTOR ?? 'input[type="password"]';
+  const submitSelector   = process.env.QA_LOGIN_SUBMIT_SELECTOR   ?? 'button[type="submit"]';
+  const successSelector  = process.env.QA_LOGIN_SUCCESS_SELECTOR  ?? '.dashboard, .main-content, [data-testid="app-shell"]';
+
+  console.log('[integration-setup] Performing browser login...');
+
+  const browser = await chromium.launch();
+  const context = await browser.newContext();
+  const page    = await context.newPage();
+
+  try {
+    await page.goto(`${baseURL}${loginPath}`, { waitUntil: 'load', timeout: 60_000 });
+
+    await page.locator(emailSelector).waitFor({ state: 'visible', timeout: 30_000 });
+    await page.locator(emailSelector).fill(email);
+    await page.locator(passwordSelector).fill(password);
+    await page.locator(submitSelector).click();
+    await page.waitForSelector(successSelector, { timeout: 60_000 });
+
+    await context.storageState({ path: AUTH_FILE });
+    console.log(`[integration-setup] Auth saved: ${AUTH_FILE}`);
+  } finally {
+    await browser.close();
+  }
+}

package/templates/integration-scaffold/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "qa-integration-tests",
+  "version": "1.0.0",
+  "description": "API integration tests - no browser required",
+  "private": true,
+  "scripts": {
+    "test":    "playwright test",
+    "test:ci": "CI=true playwright test",
+    "report":  "playwright show-report"
+  },
+  "devDependencies": {
+    "@playwright/test": "^1.44.0",
+    "dotenv":           "^16.4.0"
+  }
+}

package/templates/integration-scaffold/playwright.config.ts

@@ -0,0 +1,78 @@
+/**
+ * playwright.config.ts - API integration test project (no browser)
+ *
+ * All tests use APIRequestContext directly. No browser projects are defined.
+ * Auth is handled by global-setup.ts, which writes .auth/api-session.json.
+ *
+ * Required env vars:
+ *   QA_BASE_URL     - Base URL of the API under test
+ *
+ * Auth (at least one of the following):
+ *   QA_API_TOKEN    - Bearer token (skips browser login in global-setup)
+ *   QA_USER_EMAIL + QA_USER_PASSWORD  - Credentials for browser-based login
+ */
+
+import { defineConfig } from '@playwright/test';
+import * as dotenv from 'dotenv';
+
+dotenv.config();
+
+const required = ['QA_BASE_URL'];
+for (const key of required) {
+  if (!process.env[key]) {
+    throw new Error(`[integration] Missing required env var: ${key}. Check your .env file.`);
+  }
+}
+
+const hasToken = !!process.env.QA_API_TOKEN;
+const hasCredentials = !!(process.env.QA_USER_EMAIL && process.env.QA_USER_PASSWORD);
+
+if (!hasToken && !hasCredentials) {
+  throw new Error(
+    '[integration] No auth configured. Set QA_API_TOKEN, or both QA_USER_EMAIL and QA_USER_PASSWORD.',
+  );
+}
+
+// Optional: Azure DevOps reporter configuration
+// To enable: set env vars ADO_ORG, ADO_PROJECT, ADO_PAT, ADO_PLAN_ID and CI=true,
+// then uncomment the reporter entry in the `reporter` array below.
+// Install:  npm install @alex_neo/playwright-azure-reporter --save-dev
+// const adoReporterConfig = {
+//   orgUrl:                  `https://dev.azure.com/${process.env.ADO_ORG}`,
+//   token:                   process.env.SYSTEM_ACCESSTOKEN ?? process.env.ADO_PAT,
+//   planId:                  Number(process.env.ADO_PLAN_ID),
+//   projectName:             process.env.ADO_PROJECT!,
+//   testRunTitle:            `[Auto API] ${new Date().toISOString().slice(0, 10)}`,
+//   publishTestResultsMode:  'testRun' as const,
+//   uploadAttachments:       false,
+//   isDisabled:              !process.env.CI,
+//   autoMarkTestCasesAsAutomated: {
+//     enabled:                    true,
+//     updateAutomatedTestName:    true,
+//     updateAutomatedTestStorage: true,
+//   },
+// };
+
+export default defineConfig({
+  testDir:       './tests',
+  fullyParallel: false,
+  workers:       1,
+  timeout:       30_000,
+  retries:       process.env.CI ? 1 : 0,
+
+  globalSetup:  './global-setup.ts',
+
+  reporter: [
+    ['list'],
+    ['html', { open: 'never', outputFolder: 'playwright-report' }],
+    // Uncomment for ADO:
+    // ['@alex_neo/playwright-azure-reporter', adoReporterConfig],
+  ],
+
+  use: {
+    baseURL:      process.env.QA_BASE_URL,
+    storageState: '.auth/api-session.json',
+  },
+
+  outputDir: 'test-results/',
+});

package/templates/integration-scaffold/tests/example.spec.ts

@@ -0,0 +1,43 @@
+/**
+ * example.spec.ts - API integration test example
+ *
+ * This file shows the basic pattern for API tests using Playwright's
+ * APIRequestContext. Replace this with your real tests.
+ *
+ * The `request` fixture is pre-configured with:
+ *   - baseURL from QA_BASE_URL
+ *   - storageState from .auth/api-session.json (set by global-setup.ts)
+ *
+ * For token-based APIs that require an Authorization header, add it
+ * explicitly per request or extend the fixture in a fixtures/base.ts file.
+ */
+
+import { test, expect } from '@playwright/test';
+
+// Replace with the actual health/status endpoint of your API.
+const HEALTH_ENDPOINT = '/health';
+
+test.describe('API smoke tests', () => {
+
+  test('GET health endpoint returns 200', async ({ request }) => {
+    const response = await request.get(HEALTH_ENDPOINT);
+
+    expect(response.status()).toBe(200);
+  });
+
+  // Example of a test with an Authorization header when QA_API_TOKEN is set.
+  // Uncomment and adapt when your API uses Bearer token auth.
+  //
+  // test('GET protected resource returns 200 with bearer token', async ({ request }) => {
+  //   const response = await request.get('/api/me', {
+  //     headers: {
+  //       Authorization: `Bearer ${process.env.QA_API_TOKEN}`,
+  //     },
+  //   });
+  //
+  //   expect(response.status()).toBe(200);
+  //   const body = await response.json();
+  //   expect(body).toHaveProperty('id');
+  // });
+
+});

package/templates/memory/ci-pipeline-findings.md

@@ -0,0 +1,36 @@
+# CI Pipeline Findings
+
+> Document CI pipeline issues, root causes, and applied fixes as they are discovered.
+> This file is loaded by the agent when diagnosing CI failures or reviewing pipeline changes.
+
+## Applied Fixes
+
+Document each fix that was applied to the pipeline or test setup. Use a table or prose - whatever fits the change.
+
+| Problem | Root cause | Fix applied |
+|---|---|---|
+| _Add pipeline issue here_ | _Root cause_ | _What was changed and where_ |
+
+> Examples: auth setup defaults, missing reporter packages, artifact name collisions,
+> browser binary not installed in a CI stage, grep flag escaping issues.
+
+## Known Limitations
+
+Document constraints that are accepted but not fixed (timeout budgets, worker counts, retry behavior).
+
+> Examples:
+> - Full suite exceeds the CI timeout with 1 worker. Workaround: run only P0/P1 tags by default.
+> - `retries: 2` counts each attempt separately, so reported execution count exceeds test count.
+> - Attachment upload warnings on `PublishTestResults` when multiple failures share filenames.
+
+## Failure Triage Categories
+
+Use these categories when classifying CI failures. Add project-specific categories as needed.
+
+| Category | Description | Action |
+|---|---|---|
+| TIMEOUT | Test exceeded its time budget | Adjust `timeout` in the spec or in `playwright.config.ts` |
+| SESSION | Auth session expired mid-run | Implement session-refresh fixture or reduce suite duration |
+| FLAKY | Intermittent failure, passes on retry | Acceptable if retries cover it; investigate if failure rate > 20% |
+| APP-DEFECT | Application does not meet the spec | Open defect file; mark test with `test.fail()` |
+| TEST-CODE | Selector or assertion is wrong in CI | Fix the spec file |

package/templates/memory/data-volatility-strategies.md

@@ -0,0 +1,70 @@
+# Data Volatility Strategies
+
+> The QA environment is typically shared between automated tests and manual testers.
+> Data (record counts, available filter values, document states) changes constantly,
+> which invalidates hardcoded values in specs. Document the chosen strategy here.
+
+## Problem
+
+List the types of data that are volatile in this project's QA environment.
+
+| Data type | Example of fragile assertion | Tests affected |
+|---|---|---|
+| Filter option values | `hasText: 'specific-value'` | _Add count here_ |
+| Exact row counts | `toBe(33)` | _Add count here_ |
+| Record names used as search inputs | `fill('specific-record-name')` | _Add count here_ |
+| Process or transaction IDs | `TEST_ID = '608'` | _Add count here_ |
+
+## Strategy 1: Shared Constants with Dynamic Discovery
+
+Maintain a `fixtures/test-data-constants.ts` file with well-named constants. Populate them
+from a discovery step (global-setup or a `beforeAll` fixture) that reads the actual environment.
+
+```typescript
+// fixtures/test-data-constants.ts
+export const QA_ITEM_CURRENT = 'current-item-name';  // most recent available
+export const QA_ITEM_OLDEST  = 'oldest-item-name';   // oldest available
+export const QA_ITEM_INVALID = 'nonexistent-item';   // guaranteed absent
+```
+
+**Tradeoff:** eliminates hardcoded filter values centrally, but discovery adds startup time
+and blocks the suite if it fails.
+
+## Strategy 2: Resilient Assertions (Patterns and Ranges)
+
+Replace exact-value assertions with pattern or range checks.
+
+```typescript
+// Instead of:
+expect(count).toBe(33);
+await expect(counter).toHaveText('1-5 of 5');
+
+// Use:
+expect(count).toBeGreaterThan(0);
+await expect(counter).toHaveText(/\d+-\d+ of \d+/);
+```
+
+**Tradeoff:** no extra infrastructure, each test is self-contained, but precision is lower.
+Useful for count assertions; less useful for filter-value selection.
+
+## Strategy 3: Snapshot Delta Validation
+
+Capture the initial state before applying a filter, then assert that the filtered state
+differs from the initial state in the expected direction (fewer rows, etc.).
+
+**Tradeoff:** maximum resilience, but tests become longer and harder to trace. Use only for
+critical filter tests where behavior (not data) must be validated.
+
+## Chosen Approach
+
+> Document which strategy (or combination) was chosen for this project, and which test
+> categories each strategy applies to.
+
+| Test category | Strategy | Status |
+|---|---|---|
+| Filter option values | Strategy 1 or 2 | _Pending / Implemented_ |
+| Row count assertions | Strategy 2 | _Pending / Implemented_ |
+| Search input values | Strategy 2 or 3 | _Pending / Implemented_ |
+
+> Tests that are blocked on a data strategy can be marked `test.skip()` with a
+> `PENDING-DATA-STRATEGY` annotation until the strategy is implemented.

package/templates/memory/e2e-stabilization-patterns.md

@@ -0,0 +1,54 @@
+# E2E Stabilization Patterns
+
+> A running log of flakiness patterns found in this project and the techniques used to fix them.
+> Load this file when investigating intermittent test failures or refactoring fragile specs.
+
+## Crash-Recovery Pattern
+
+When the application server crashes intermittently (e.g. unhandled error page with a Reload link),
+a bare `waitFor` will timeout. Use `Promise.race` to detect the error state and recover:
+
+```typescript
+const outcome = await Promise.race([
+  targetElement.waitFor({ state: 'visible', timeout: 60_000 }).then(() => 'ready' as const),
+  page.getByText('An unhandled error has occurred.').waitFor({ state: 'visible', timeout: 60_000 }).then(() => 'crash' as const),
+]).catch(() => 'timeout' as const);
+
+if (outcome === 'crash') {
+  await page.getByRole('link', { name: 'Reload' }).click();
+  await targetElement.waitFor({ state: 'visible', timeout: 60_000 });
+}
+```
+
+> Document which spec files use this pattern and under what conditions it fires.
+
+## Strict-Mode Selector Conflicts
+
+`getByText()` and similar matchers can match multiple elements, triggering strict-mode violations.
+Scope selectors to the most specific container available.
+
+> Document conflicts found in this project here.
+> Example: a pagination counter regex that also matches a card element.
+> Fix: use a scoped `locator('.specific-class')` instead of a broad text match.
+
+## Known Flaky Failure Types
+
+Document categories of instability specific to this project. A row per pattern is enough.
+
+| ID | Root cause | Fix | Affected tests |
+|---|---|---|---|
+| _Add ID_ | _Describe the underlying cause_ | _Technique used to fix_ | _Spec files or TC IDs_ |
+
+> Examples: framework hydration delays, input binding issues requiring `pressSequentially`,
+> elements that are in the DOM but not yet rendered, session expiry during long runs.
+
+## Reference Timeouts
+
+Fill in the timeout values used in this project once they are stabilized.
+
+| Setting | Value | Location |
+|---|---|---|
+| Default test timeout | - | `playwright.config.ts` |
+| `actionTimeout` | - | `playwright.config.ts` |
+| `navigationTimeout` | - | `playwright.config.ts` |
+| Long-running spec timeout | - | Per-spec `describe.configure` |

package/templates/qa-framework.instructions.md

@@ -24,7 +24,15 @@ This project uses `@keber/qa-framework` v{{VERSION}} for spec-driven automated t
 1. Generate the output in `UTF-8` encoding without BOM (`UTF-8`, no signature).
 2. Ensure the raw output is `UTF-8` encoded with no `EF BB BF` bytes at the beginning.
 
-{{ADO_SECTION}}
+## Azure DevOps integration
+
+Before any ADO operation, check if `.github/skills/ado-qa/` exists or if
+`integrations.ado.enabled` is `true` in `qa/qa-framework.config.json`. If either
+condition is met, load `.github/skills/qa-ado-integration/SKILL.md` and use it for
+all ADO interactions (work items, test plans, test cases, bugs, etc.).
+
+> To enable Azure DevOps integration run: `npm install github:keber/ado-qa`
+
 ## QA Pipeline
 
 Stages must run in order. Never start a stage unless its prerequisite is met.

package/templates/qa-readme.md

@@ -2,6 +2,7 @@
 
 > Framework: `@keber/qa-framework` v{VERSION}
 > Language: {LANGUAGE} | Base URL: `{QA_BASE_URL}`
+> ADO Project: `{ADO_ORG} / {ADO_PROJECT}` _(optional — remove line if not using Azure DevOps)_
 
 ---
 
@@ -47,12 +48,32 @@ npx playwright test --last-failed
 
 ---
 
+## Last Run
+
+| Suite | Date | Pass | Fail | Skip | CI Link |
+|---|---|---|---|---|---|
+| E2E | — | — | — | — | — |
+| API | — | — | — | — | — |
+
+> Update this table after each significant run. The CI Link can be an ADO TestRun URL, a
+> GitHub Actions run URL, or a path to a local report file.
+
 ## Last Execution
 
 | Date | Suite | Pass | Skip | Fail | Duration | Report |
 |---|---|---|---|---|---|---|
 | — | — | — | — | — | — | — |
 
+## Flaky Tests
+
+> Mark known flaky tests with this inline annotation directly in the table or spec file.
+> Format: `[flaky] <test-id-or-name> - <root cause> - last seen <YYYY-MM-DD>`
+>
+> Example:
+> `[flaky] TC-MOD-012 - timing issue on slow CI agents - last seen 2026-01-15`
+>
+> Remove the annotation once the test has been stable for 2+ consecutive CI runs.
+
 ---
 
 ## Sprint History