@1aboveio/skills 0.10.0

package/skills/engineering/e2e-test/assets/ci-gates.cloudbuild.yaml

@@ -0,0 +1,272 @@
+# Coverage gates — Cloud Build template (satisfies ensure-coverage references/enforcement/ci-contract.md).
+#
+# Each gate is BLOCKING (a failing Cloud Build step fails the build), none standing in
+# for another. Two of the three lanes are in this file (release lives in resolve-release):
+# the PR-lane floor (inventory, accounting, render sweep,
+# no-over-mock, Review-Contract checklist validate, quarantine auto-resurface, and the
+# curated E2E SMOKE golden-path liveness gate #8) runs every PR; the broad functional +
+# journey E2E suite (gate 4) is DEMOTED to the integration/nightly lane (branch-guarded
+# below) — verified before release, not on the per-PR critical path. Plus the meta-gate
+# that runs ci-audit against this file so the gates can't be silently dropped. Copy into
+# your cloudbuild.*.yaml or wire as a dedicated PR trigger. Replace <…> placeholders.
+# Verify it stays wired: `node <ensure-coverage>/scripts/ci-audit.mjs --files cloudbuild.gates.yaml`.
+#
+# Conventions this template encodes:
+#   1. DOCKER-PREFERRED: every browser step already runs in the official pinned
+#      Playwright image (`name:` below) — Cloud Build steps are containers by
+#      construction. Keep every pin IDENTICAL and matching the repo's
+#      @playwright/test version (mismatch → a network `playwright install` per run,
+#      or protocol failures). Canonical pin: mcr.microsoft.com/playwright:v1.61.0-noble.
+#   2. TIME BUDGET: the PR lane targets ≤5 min wall clock — browser steps carry
+#      `timeout: 300s`, the build carries an overall `timeout`, and golden paths run
+#      in parallel inside the smoke step (E2E_WORKERS). Steps run serially by default;
+#      use `waitFor: ['-']` on independent static gates to overlap them if the total
+#      creeps past budget.
+#   3. UMBRELLA: Cloud Build needs no umbrella job — the build's single pass/fail
+#      status IS the one check. Point GitHub branch protection (or your merge gate)
+#      at this trigger's status check and nothing else.
+
+steps:
+  # Static test-lint gates — fails if this push ADDS a banned pattern to a test
+  # file: a skip/focus/xfail (no-skip), a fixed sleep (no-fixed-timeout), an empty
+  # test (no-empty-test), or a mock of a first-party internal seam (mock-internal-seam,
+  # default-on). Diff mode leaves inherited violations alone; a change may not add a new one.
+  - id: lint-tests
+    name: node:20
+    entrypoint: bash
+    args: ['-c', 'node <ensure-coverage>/scripts/lint-tests.mjs --diff "$_BASE_REF"']
+
+  # Gate 5 — no over-mock (depth axis), WHOLE-TREE lane: inherited first-party-seam
+  # mocks block until removed (the --diff run above already covers newly-added ones).
+  # Zero-tolerance, no waiver; ci-audit looks for `mock-internal-seam` by name.
+  - id: over-mock-whole-tree
+    name: node:20
+    entrypoint: bash
+    args: ['-c', 'node <ensure-coverage>/scripts/lint-tests.mjs --rules mock-internal-seam']
+
+  # Gate 1 — inventory freshness: regenerate the surface inventory + route manifest,
+  # fail on drift so a new route can't be silently uncovered in either manifest.
+  - id: inventory-freshness
+    name: node:20
+    entrypoint: bash
+    args:
+      - -c
+      - |
+        node scripts/gen-inventory.mjs              # your INVENTORY-CONTRACT adapter → surface-inventory.json
+        node <e2e-test>/scripts/project-route-manifest.mjs \
+          --inventory docs/tests/_generated/surface-inventory.json \
+          --overlay e2e/route-manifest.overlay.json --out e2e/route-manifest.json
+        git diff --exit-code docs/tests/_generated/surface-inventory.json e2e/route-manifest.json
+
+  # Gate 2 — accounting: the ledger. Fails on Missing / Over-mocked / blocking
+  # Unverified. Static + fast; no browser. Proves a qualifying test was AUTHORED.
+  - id: coverage-accounting
+    name: node:20
+    entrypoint: bash
+    args: ['-c', 'node <ensure-coverage>/scripts/coverage-ledger.mjs validate']
+
+  # Gate 6 — Review-Contract checklist validate. Static anti-ritual guard: every
+  # PASS/FAIL checklist row carries an evidence locator (file:line / `command` / #issue)
+  # and no gate row is TODO. Doctrine: ensure-coverage hooks.md (CI tier). Point at the
+  # committed Review-Contract checklist doc (or the file the PR body's checklist extracts to).
+  - id: coverage-checklist
+    name: node:20
+    entrypoint: bash
+    args: ['-c', 'node <ensure-coverage>/scripts/coverage-checklist.mjs validate <docs/tests/CONTRACT-CHECKLIST.md>']
+
+  # Gate 6b — Review-Contract evidence-block freshness. The block is stamped with the
+  # HEAD it describes; validate fails when the stamp isn't that HEAD, so a fix-round
+  # commit landing after collection can't ship stale. Cloud Build checks out the
+  # triggering commit directly, so HEAD is already the right SHA — no ref override.
+  # Needs FULL history: triggers clone shallow by default, and the block's base..head
+  # derivations (including the production-diff claim) can't resolve in a shallow clone.
+  # Set the trigger's fetch depth to 0, or unshallow before this step.
+  - id: evidence-block
+    name: node:20
+    entrypoint: bash
+    args: ['-c', 'node <ensure-coverage>/scripts/evidence-block.mjs validate <docs/tests/evidence.json> --source <src,app,packages>']
+
+  # Gate 7 — quarantine auto-resurface. BLOCKING (distinct from the non-blocking
+  # quarantine *lane* below): fails once any `@quarantine(until=…)` / `until=` exit date
+  # is in the past, so a deferred suppression can't rot green into a permanent skip.
+  # ci-audit looks for `check-quarantine-expiry` by name.
+  - id: quarantine-expiry
+    name: node:20
+    entrypoint: bash
+    args: ['-c', 'node <ensure-coverage>/scripts/check-quarantine-expiry.mjs']   # exit 1 on any expired dated exit
+
+  # Gate 3 — render sweep: opens EVERY manifest route in a real browser and asserts
+  # the presentation floor (render-health spine first). SEPARATE job — needs a
+  # browser + seeded DB + auth bypass. This is the gate that catches a crashing
+  # route a green functional suite sails past. NOT the functional suite.
+  - id: presentation-sweep
+    name: mcr.microsoft.com/playwright:v1.61.0-noble
+    timeout: 300s   # PR-lane budget: the sweep must stay inside 5 min
+    entrypoint: bash
+    env: ['E2E_AUTH_BYPASS=1', 'DATABASE_URL=${_DATABASE_URL}']
+    args:
+      - -c
+      - |
+        <seed + migrate the test DB>
+        <boot the app on $BASE_URL>   # or point --base-url at a preview deploy
+        node <e2e-test>/scripts/presentation-sweep.mjs \
+          --manifest e2e/route-manifest.json --base-url "$$BASE_URL" \
+          --fixtures e2e/sweep-fixtures.mjs \
+          --out docs/tests/_generated/presentation-results.json
+
+  # Gate 8 — curated E2E smoke (golden-path liveness). PR-LANE FLOOR, blocking.
+  # Boots the REAL app and drives only the do-not-ship journeys (sign in → core
+  # action → pay), writes smoke-results.json, then reduces those observations through
+  # smoke.mjs verdict. Any fail/blocked/missing reds the build. This — NOT the broad
+  # functional suite below — is the per-PR liveness gate; it catches the integrated
+  # render-time bug a green unit/functional suite sails past (#237). ci-audit looks
+  # for `smoke … verdict` by name.
+  - id: smoke
+    name: mcr.microsoft.com/playwright:v1.61.0-noble
+    timeout: 300s   # PR-lane budget: golden paths run in parallel (E2E_WORKERS), wall clock ≈ slowest path
+    entrypoint: bash
+    env: ['E2E_AUTH_BYPASS=1', 'E2E_WORKERS=4', 'DATABASE_URL=${_DATABASE_URL}']
+    args:
+      - -c
+      - |
+        <seed + migrate the test DB>
+        <boot the app on $BASE_URL>
+        # Validate + reconcile before gating: --journeys verifies every browser path
+        # references a real @smoke-tagged journey AND every @smoke journey is in the
+        # manifest — a drift either way (esp. an @smoke journey missing from the
+        # manifest, a silent false-ALIVE) reds the build instead of passing vacuously.
+        node <smoke>/scripts/smoke.mjs validate --manifest smoke.manifest.json --journeys <e2e-journeys-dir>
+        node <smoke>/scripts/smoke.mjs plan --manifest smoke.manifest.json --target local --json > smoke-plan.json
+        <drive smoke-plan.json through Playwright/API/CLI and write smoke-results.json>
+        node <smoke>/scripts/smoke.mjs verdict --manifest smoke.manifest.json --target local --results smoke-results.json --out smoke-verdict.json
+
+  # Gate 4 — broad functional + journey E2E (distinct from gate 3 and gate 8).
+  # HEAVY/INTEGRATION LANE, blocking — but DEMOTED off the per-PR critical path: it runs
+  # only on the integration branch (REQUIRED before release), never on a PR. An OPTIONAL
+  # nightly cadence is opt-in via `_LANE=nightly` — not required; the integration-branch
+  # run is the release gate. The per-PR floor is the smoke gate above; the full
+  # behavioral suite is verified before release. Excludes @quarantine-tagged tests.
+  - id: e2e
+    name: mcr.microsoft.com/playwright:v1.61.0-noble
+    timeout: 1800s   # integration lane — the one lane allowed past the 5-min PR budget (no-op on PRs)
+    entrypoint: bash
+    env: ['E2E_AUTH_BYPASS=1', 'E2E_WORKERS=4', 'DATABASE_URL=${_DATABASE_URL}']
+    args:
+      - -c
+      - |
+        if [ "$BRANCH_NAME" = "integration" ] || [ "${_LANE}" = "nightly" ]; then
+          <seed + migrate the test DB>
+          pnpm playwright test --grep-invert "@quarantine"
+        else
+          echo "skip: broad functional+journey E2E runs on the integration/nightly lane, not per-PR (gate 8 smoke is the per-PR floor)"
+        fi
+
+  # Quarantine lane — the ONLY sanctioned non-blocking path (tests are never skipped).
+  # Un-sandboxable externals or a genuine flake under active fix go HERE: the test
+  # still RUNS and REPORTS, it just doesn't block (allowFailure). Keep the surface
+  # graded `Unverified` in the ledger (never `verified: true`), with a dated exit.
+  - id: quarantine
+    name: mcr.microsoft.com/playwright:v1.61.0-noble
+    entrypoint: bash
+    allowFailure: true   # the whole point — must never hard-block the build
+    env: ['E2E_AUTH_BYPASS=1', 'DATABASE_URL=${_DATABASE_URL}']
+    args:
+      - -c
+      - |
+        <seed + migrate the test DB>
+        pnpm playwright test --grep "@quarantine" --pass-with-no-tests   # provider test keys via secrets, never live keys; green when nothing is quarantined
+
+  # Visual regression GATE (presentation:visual) — OPT-IN, BLOCKING when present.
+  # Same PINNED Playwright image as the regen step so baselines and this comparison
+  # share one rendering environment (a baseline minted on another OS reds on every
+  # glyph's anti-aliasing). Compares against the COMMITTED baselines; never writes
+  # them. Only needed when a surface declares `requiredLevels: [presentation:visual]`.
+  - id: visual-gate
+    name: mcr.microsoft.com/playwright:v1.61.0-noble   # PIN — must match visual-baseline-regen
+    entrypoint: bash
+    env: ['E2E_AUTH_BYPASS=1', 'DATABASE_URL=${_DATABASE_URL}']
+    args:
+      - -c
+      - |
+        <seed + migrate the test DB>
+        <boot the app on $BASE_URL>
+        # --pass-with-no-tests: a repo with no @visual specs must not red the whole build.
+        pnpm playwright test --grep "@visual" --pass-with-no-tests   # toHaveScreenshot specs; fails on drift from committed baselines
+
+  # Visual BASELINE-REGEN — ON-DEMAND (gated on the _UPDATE_VISUAL_BASELINES
+  # substitution; wire it to a manual/labelled trigger), the ONLY sanctioned writer
+  # of baselines. Same pinned image. Regenerates the PNGs, commits them back, then
+  # posts the durable baseline link + ephemeral diff URL to the issue AND PR and
+  # RETURNS both as a value (`--evidence-out`, plus stdout) for the resolve-issues
+  # orchestrator to record on the unit. This lane never writes the run manifest itself —
+  # the orchestrator is its only writer (ADR 0001 decision 5). Never blocks; an explicit
+  # human/agent action.
+  #
+  # `--evidence-out` is not optional here: it is what gives this lane returned-value
+  # parity with `ci-gates.github.yml`. A lane that only comments on the issue leaves the
+  # orchestrator nothing to record, and the unit's visual proof-of-execution is silently
+  # absent — the gate looks green because the evidence was never asked for.
+  - id: visual-baseline-regen
+    name: mcr.microsoft.com/playwright:v1.61.0-noble   # MUST match visual-gate
+    entrypoint: bash
+    env: ['E2E_AUTH_BYPASS=1', 'DATABASE_URL=${_DATABASE_URL}']
+    args:
+      - -c
+      - |
+        [ "$_UPDATE_VISUAL_BASELINES" = "true" ] || { echo "skip: regen not requested"; exit 0; }
+        <seed + migrate the test DB>
+        <boot the app on $BASE_URL>
+        pnpm playwright test --grep "@visual" --update-snapshots
+        git add '**/*-snapshots/**' && (git diff --staged --quiet || git commit -m "chore(visual): regenerate baselines") && git push
+        node <e2e-test>/scripts/post-visual-evidence.mjs \
+          --unit "$_PR_NUMBER" --repo "$_REPO" --sha "$(git rev-parse HEAD)" \
+          --issue "$_LINKED_ISSUE" --pr "$_PR_NUMBER" \
+          --diff-artifact "$_DIFF_ARTIFACT_URL" \
+          --results docs/tests/_generated/presentation-results.json \
+          --evidence-out /workspace/visual-evidence.json
+
+  # Publish the RETURNED evidence so the run can read it. `/workspace` persists across
+  # steps but not past the build, which is the Cloud Build counterpart of GitHub's
+  # `upload-artifact`. Guarded twice — on the regen substitution and on the file
+  # existing — because a top-level `artifacts:` block uploads unconditionally and reds
+  # every ordinary build, where this lane did not run and the file is absent.
+  - id: visual-evidence-publish
+    name: gcr.io/cloud-builders/gsutil
+    entrypoint: bash
+    args:
+      - -c
+      - |
+        [ "$_UPDATE_VISUAL_BASELINES" = "true" ] || { echo "skip: regen not requested"; exit 0; }
+        [ -n "$_EVIDENCE_BUCKET" ] || { echo "skip: _EVIDENCE_BUCKET unset — evidence stays on the regen step's stdout"; exit 0; }
+        [ -f /workspace/visual-evidence.json ] || { echo "skip: no evidence file"; exit 0; }
+        gsutil cp /workspace/visual-evidence.json "gs://$_EVIDENCE_BUCKET/$BUILD_ID/visual-evidence.json"
+
+  # Meta-gate (bootstrap) — run ci-audit against THIS gate file so the gates above
+  # can't be silently deleted or weakened later. ci-audit's own META_GATE is exactly
+  # "is ci-audit run by CI?". Keep --files in sync with the gate file(s) you ship.
+  - id: gate-audit
+    name: node:20
+    entrypoint: bash
+    args: ['-c', 'node <ensure-coverage>/scripts/ci-audit.mjs --files cloudbuild.gates.yaml']
+
+# DO NOT add `allowFailure: true` to gates 2/3/4 — that turns a gate into theatre.
+# (The quarantine lane above is the one sanctioned exception: it RUNS + REPORTS the
+#  test non-blocking, with the surface graded `Unverified`; it is not a skip.)
+# Phase-in (warn-only) is acceptable only temporarily and must be reported by the
+# CI audit as `warn-only` with a dated exit plan, never treated as `present`.
+# Overall ceiling. Steps run serially, so this must exceed the sum of the step
+# timeouts that can actually run together — the e2e step alone may use its full
+# 1800s on the integration lane. The PR lane (where e2e no-ops) finishes well
+# inside 5 min; this ceiling is for the integration lane.
+timeout: 3600s
+options:
+  env: ['BASE_URL=http://127.0.0.1:3000']
+substitutions:
+  _DATABASE_URL: ''  # injected from Secret Manager in your trigger
+  _BASE_REF: 'origin/main'  # the promotion/PR base the no-skip diff lints against
+  _UPDATE_VISUAL_BASELINES: 'false'  # set true on a manual/labelled trigger to regenerate visual baselines
+  _PR_NUMBER: ''       # PR number for the visual evidence comment
+  _LINKED_ISSUE: ''    # the source issue # the evidence comment is also posted to
+  _REPO: ''            # owner/repo for the durable baseline link
+  _DIFF_ARTIFACT_URL: ''  # ephemeral diff/preview artifact URL (with expiry), if uploaded
+  _EVIDENCE_BUCKET: ''    # GCS bucket for the RETURNED visual-evidence value; empty = stdout only