@momentiq/dark-factory-cli 2.4.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +83 -39
- package/dist/cli.js.map +1 -1
- package/dist/commands/verify.d.ts +6 -0
- package/dist/commands/verify.d.ts.map +1 -0
- package/dist/commands/verify.js +223 -0
- package/dist/commands/verify.js.map +1 -0
- package/dist/evidence/route-runner.d.ts +1 -0
- package/dist/evidence/route-runner.d.ts.map +1 -1
- package/dist/evidence/route-runner.js +23 -2
- package/dist/evidence/route-runner.js.map +1 -1
- package/dist/handoff/handoff-verb.d.ts +6 -0
- package/dist/handoff/handoff-verb.d.ts.map +1 -1
- package/dist/handoff/handoff-verb.js +58 -0
- package/dist/handoff/handoff-verb.js.map +1 -1
- package/dist/mcp/tools/handoff.d.ts.map +1 -1
- package/dist/mcp/tools/handoff.js +7 -0
- package/dist/mcp/tools/handoff.js.map +1 -1
- package/dist/policy/gate.d.ts.map +1 -1
- package/dist/policy/gate.js +30 -18
- package/dist/policy/gate.js.map +1 -1
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +11 -6
- package/dist/runner.js.map +1 -1
- package/dist/skills/install.d.ts.map +1 -1
- package/dist/skills/install.js +1 -0
- package/dist/skills/install.js.map +1 -1
- package/package.json +2 -2
- package/skills/chief-engineer-blitz/SKILL.md.tmpl +7 -6
- package/skills/chief-engineer-blitz/skill.json +1 -1
- package/skills/verify/SKILL.md.tmpl +100 -0
- package/skills/verify/producer/README.md +73 -0
- package/skills/verify/producer/coverage.ts +121 -0
- package/skills/verify/producer/playwright-route.sh +161 -0
- package/skills/verify/producer/playwright.ui-route.config.ts +111 -0
- package/skills/verify/producer/ui-route.producer.spec.ts +281 -0
- package/skills/verify/skill.json +22 -0
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# Reusable Playwright (UI) verification-route producer
|
|
2
|
+
|
|
3
|
+
These are **reference files** for the Dark Factory `playwright` verification
|
|
4
|
+
route — the UI-layer half of evidence-gated validation (`web/**` / `*.tsx`
|
|
5
|
+
changes). They ship inside `@momentiq/dark-factory-cli` (so `df skills install
|
|
6
|
+
verify` and the consumer guide can point you at them), but unlike a rendered
|
|
7
|
+
skill they are **copy-once-and-own**: you adapt `SURFACES[]` and your auth to
|
|
8
|
+
your app, and a future `df skills install verify` will **not** overwrite them.
|
|
9
|
+
|
|
10
|
+
Generalized from the proven `dark-factory-dashboard` dogfood producer (Cycle 21
|
|
11
|
+
EC2/EC8). The Clerk/Doppler auth coupling has been removed — auth is now an
|
|
12
|
+
optional, consumer-supplied Playwright **storage-state**.
|
|
13
|
+
|
|
14
|
+
## Files
|
|
15
|
+
|
|
16
|
+
| File | Role | You edit? |
|
|
17
|
+
|---|---|---|
|
|
18
|
+
| `playwright-route.sh` | Route `command`: SHA + clean-tree guards, computes changed UI paths, runs Playwright, maps the 0/1/2 exit. | Env knobs only |
|
|
19
|
+
| `playwright.ui-route.config.ts` | Dedicated Playwright config: one chromium project, optional storage-state, optional local-preview server. | `testDir` / dev command |
|
|
20
|
+
| `ui-route.producer.spec.ts` | The capture: ARIA snapshot + after (+ optional before) screenshots, fail-closed coverage, optional VLM hook. | **`SURFACES[]`** (required) |
|
|
21
|
+
| `coverage.ts` | Pure fail-closed coverage logic (a changed UI path with no surface blocks). Unit-tested upstream. | No |
|
|
22
|
+
|
|
23
|
+
## Setup (one-time)
|
|
24
|
+
|
|
25
|
+
1. **Copy** the four files into your repo. Convention: the script under
|
|
26
|
+
`scripts/verify/`, and `playwright.ui-route.config.ts` + `tests/e2e/
|
|
27
|
+
ui-route.producer.spec.ts` + `tests/e2e/coverage.ts` under your web app dir
|
|
28
|
+
(`web/` by default). Adjust `DF_UI_ROUTE_WEB_DIR` / `DF_UI_ROUTE_CONFIG` (see
|
|
29
|
+
the script header) if your layout differs.
|
|
30
|
+
2. **Edit `SURFACES[]`** in `ui-route.producer.spec.ts` — your routes, each with
|
|
31
|
+
a structural `requiredHeading` (an accessible-name regex) and the `covers`
|
|
32
|
+
source globs it is the evidence for. v1 ships one public `/` surface so it
|
|
33
|
+
works zero-auth out of the box.
|
|
34
|
+
3. **(Protected surfaces only)** Mint an authenticated Playwright `storageState`
|
|
35
|
+
JSON with your own auth and point `DF_UI_ROUTE_STORAGE_STATE` at it. Public
|
|
36
|
+
surfaces need none. Gitignore your storage-state (`**/.auth/`).
|
|
37
|
+
4. **Arm the route** in `.agent-review/config.json` — override the `playwright`
|
|
38
|
+
route's placeholder `command` with `bash scripts/verify/playwright-route.sh`:
|
|
39
|
+
|
|
40
|
+
```jsonc
|
|
41
|
+
{
|
|
42
|
+
"id": "playwright",
|
|
43
|
+
"trigger": ["web/**", "**/*.tsx"],
|
|
44
|
+
"command": "bash scripts/verify/playwright-route.sh",
|
|
45
|
+
"evidencePath": "agent-reviews/quality-gates/${sha}.json",
|
|
46
|
+
"category": "ui",
|
|
47
|
+
"evidenceKind": "playwright"
|
|
48
|
+
}
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
5. **Produce evidence**: `df verify --route playwright` (or it runs automatically
|
|
52
|
+
when `df verify` / `df gate-push` arm the route for a `web/**` / `*.tsx`
|
|
53
|
+
change). See `docs/guides/ui-verification-route.md` for the full walkthrough.
|
|
54
|
+
|
|
55
|
+
## Environment knobs
|
|
56
|
+
|
|
57
|
+
| Var | Default | Purpose |
|
|
58
|
+
|---|---|---|
|
|
59
|
+
| `DF_UI_ROUTE_WEB_DIR` | `web` | Where your app + `@playwright/test` live. |
|
|
60
|
+
| `DF_UI_ROUTE_CONFIG` | `playwright.ui-route.config.ts` | Producer config, relative to the web dir. |
|
|
61
|
+
| `DF_UI_ROUTE_STORAGE_STATE` | _(unset)_ | Path to an authenticated storage-state (protected surfaces). |
|
|
62
|
+
| `PLAYWRIGHT_TEST_BASE_URL` | local preview | Point at a deployed preview of the SHA instead of serving locally. |
|
|
63
|
+
| `DF_UI_ROUTE_DEV_CMD` | `npm run dev` | Local-preview server command. |
|
|
64
|
+
| `DF_UI_ROUTE_PORT` | `3217` | Local-preview port. |
|
|
65
|
+
| `DF_UI_ROUTE_BEFORE_BASE_URL` | _(unset)_ | Origin for the optional before/after baseline. |
|
|
66
|
+
| `DF_UI_ROUTE_VLM` | _(unset)_ | `1` arms the optional VLM hook (deferred/no-op in v1). |
|
|
67
|
+
|
|
68
|
+
## What this is NOT
|
|
69
|
+
|
|
70
|
+
It does not mint your auth session and it does not guarantee that gitignored
|
|
71
|
+
content absent from the commit cannot influence a capture (the v1 trust-boundary
|
|
72
|
+
limitation — content-isolated re-execution is the v2 fix). It validates the
|
|
73
|
+
**HEAD** working tree; producing evidence for a non-HEAD SHA fails closed.
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fail-closed coverage logic for the reusable Evidence-Gated Validation
|
|
3
|
+
* **playwright (UI) route** producer (momentiq-ai/dark-factory#193).
|
|
4
|
+
*
|
|
5
|
+
* Generalized from the dark-factory-dashboard dogfood producer (Cycle 21 PR-4).
|
|
6
|
+
* This module is pure, importable, and unit-tested without a browser, so the
|
|
7
|
+
* load-bearing rule — *a changed UI path that maps to no capture surface BLOCKS
|
|
8
|
+
* the route* — cannot regress.
|
|
9
|
+
*
|
|
10
|
+
* A `web/**`/`*.tsx` change arms the route; this module decides, per changed
|
|
11
|
+
* path, whether a capture surface exists. If a changed *product-UI* path has no
|
|
12
|
+
* surface, it lands in `uncovered` and the producer fails closed — so the gate
|
|
13
|
+
* cannot be satisfied by rendering only an unaffected surface.
|
|
14
|
+
*
|
|
15
|
+
* This is a CONSUMER REFERENCE FILE: copy it into your repo alongside the
|
|
16
|
+
* producer spec and own it. It has no Dark Factory / vendor dependency.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
export interface UiSurface {
|
|
20
|
+
/** URL path to navigate (relative to baseURL). */
|
|
21
|
+
path: string;
|
|
22
|
+
/** Filesystem-safe slug for the evidence subdir. */
|
|
23
|
+
slug: string;
|
|
24
|
+
/** A role+name the surface MUST expose (structural floor assertion). */
|
|
25
|
+
requiredHeading: RegExp;
|
|
26
|
+
/**
|
|
27
|
+
* Source-path globs this surface is the evidence for. A changed path
|
|
28
|
+
* matching one of these is "covered" by this surface's capture.
|
|
29
|
+
*/
|
|
30
|
+
covers: readonly string[];
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface CoveragePartition {
|
|
34
|
+
/** Surfaces armed because a changed path matches their `covers`. */
|
|
35
|
+
armed: UiSurface[];
|
|
36
|
+
/** Changed UI paths with no mapped surface (the fail-closed set). */
|
|
37
|
+
uncovered: string[];
|
|
38
|
+
/**
|
|
39
|
+
* True when the route is armed (the change touched the trigger surface) but
|
|
40
|
+
* NO surface was armed AND nothing is uncovered — i.e. the change is
|
|
41
|
+
* harness/non-UI only (e.g. an edit to the gate logic or the producer config
|
|
42
|
+
* itself). In that case the producer captures ALL surfaces as a regression
|
|
43
|
+
* smoke so the route still produces real ARIA + after evidence and cannot be
|
|
44
|
+
* passed with zero captures (the harness-only bypass — symmetric to the
|
|
45
|
+
* false-positive hole).
|
|
46
|
+
*/
|
|
47
|
+
smokeAllSurfaces: boolean;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Translate a glob (`**`, `*`, literal route-group `(group)` dirs) to a RegExp.
|
|
52
|
+
* `**` matches across `/`; `*` matches within a path segment. Anchored.
|
|
53
|
+
*
|
|
54
|
+
* Built by scanning the glob (no placeholder char) so `**` and `*` are expanded
|
|
55
|
+
* cleanly; everything else is regex-escaped — including the parentheses in
|
|
56
|
+
* Next.js route groups, which we want treated as literals.
|
|
57
|
+
*/
|
|
58
|
+
export function globToRegExp(glob: string): RegExp {
|
|
59
|
+
let pattern = "";
|
|
60
|
+
for (let i = 0; i < glob.length; i++) {
|
|
61
|
+
const ch = glob[i] as string;
|
|
62
|
+
if (ch === "*") {
|
|
63
|
+
if (glob[i + 1] === "*") {
|
|
64
|
+
pattern += ".*"; // ** → any chars including /
|
|
65
|
+
i++;
|
|
66
|
+
} else {
|
|
67
|
+
pattern += "[^/]*"; // * → any chars except /
|
|
68
|
+
}
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
pattern += ch.replace(/[.+^${}()|[\]\\]/, "\\$&"); // escape one metachar
|
|
72
|
+
}
|
|
73
|
+
return new RegExp(`^${pattern}$`);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function matchesAny(path: string, globs: readonly string[]): boolean {
|
|
77
|
+
return globs.some((g) => globToRegExp(g).test(path));
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Partition changed UI paths into the surfaces they arm and the paths that map
|
|
82
|
+
* to NO surface (the fail-closed set). `nonSurfaceGlobs` are harness / non-UI
|
|
83
|
+
* files that are exempt from the coverage requirement.
|
|
84
|
+
*
|
|
85
|
+
* Empty `changed` (e.g. a manual run with the env unset) arms every surface so
|
|
86
|
+
* the producer still captures evidence; the gate's real armed-by-diff semantics
|
|
87
|
+
* come from the changed-paths the route script computes.
|
|
88
|
+
*/
|
|
89
|
+
export function partitionChangedPaths(
|
|
90
|
+
changed: readonly string[],
|
|
91
|
+
surfaces: readonly UiSurface[],
|
|
92
|
+
nonSurfaceGlobs: readonly string[],
|
|
93
|
+
): CoveragePartition {
|
|
94
|
+
if (changed.length === 0) {
|
|
95
|
+
// No changed-path signal (e.g. a manual run with the env unset) — arm all
|
|
96
|
+
// surfaces so the producer still captures evidence.
|
|
97
|
+
return { armed: [...surfaces], uncovered: [], smokeAllSurfaces: false };
|
|
98
|
+
}
|
|
99
|
+
const armed = new Set<UiSurface>();
|
|
100
|
+
const uncovered: string[] = [];
|
|
101
|
+
for (const path of changed) {
|
|
102
|
+
if (matchesAny(path, nonSurfaceGlobs)) continue; // harness/non-UI — skip.
|
|
103
|
+
const matched = surfaces.filter((s) => matchesAny(path, s.covers));
|
|
104
|
+
if (matched.length === 0) {
|
|
105
|
+
uncovered.push(path);
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
for (const s of matched) armed.add(s);
|
|
109
|
+
}
|
|
110
|
+
// Harness-only change (the route is armed by a web/** edit, but every changed
|
|
111
|
+
// path is a non-surface harness/config/gate-logic file): capture ALL surfaces
|
|
112
|
+
// as a regression smoke so the route NEVER passes with zero evidence. This
|
|
113
|
+
// closes the harness-only bypass — a change to the gate logic itself still
|
|
114
|
+
// produces real ARIA + after captures.
|
|
115
|
+
const smokeAllSurfaces = armed.size === 0 && uncovered.length === 0;
|
|
116
|
+
return {
|
|
117
|
+
armed: smokeAllSurfaces ? [...surfaces] : [...armed],
|
|
118
|
+
uncovered,
|
|
119
|
+
smokeAllSurfaces,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Reusable Evidence-Gated Validation **playwright (UI) route** producer command
|
|
3
|
+
# (momentiq-ai/dark-factory#193). Generalized from the dark-factory-dashboard
|
|
4
|
+
# dogfood (Cycle 21 PR-4).
|
|
5
|
+
#
|
|
6
|
+
# This is the `command` of the `playwright` route in
|
|
7
|
+
# `.agent-review/config.json:validation.verificationRoutes`. The
|
|
8
|
+
# @momentiq/dark-factory-cli route-runner (`df verify` / `df gate-push`) spawns
|
|
9
|
+
# it with NO shell — the route command is `command.trim().split(/\s+/)` then
|
|
10
|
+
# `spawn(argv[0], argv.slice(1))` — which is why the route command is the single
|
|
11
|
+
# token-pair `bash scripts/verify/playwright-route.sh` and ALL the real work
|
|
12
|
+
# (guards, Playwright, exit mapping) lives here.
|
|
13
|
+
#
|
|
14
|
+
# Exit-code contract (the 0/1/2 route contract):
|
|
15
|
+
# 0 — green: evidence captured, the routed UI surface(s) rendered + asserted.
|
|
16
|
+
# 1 — block: a routed UI surface failed to render/assert, OR a real producer
|
|
17
|
+
# error (Playwright crashed, a guarded file is dirty). Fail closed.
|
|
18
|
+
# 2 — soft-skip: the producer cannot run HERE (Playwright not installed, or a
|
|
19
|
+
# required auth storage-state is configured but absent). Mapped by the CLI
|
|
20
|
+
# to `requiresHumanJudgment` — NOT a pass. We NEVER fabricate evidence.
|
|
21
|
+
#
|
|
22
|
+
# ── CONSUMER REFERENCE FILE — copy into your repo (e.g. scripts/verify/) and
|
|
23
|
+
# OWN it. Adjust the env knobs at the top to your repo layout. Then arm the
|
|
24
|
+
# `playwright` route's `command` to `bash <path-to-this-script>` in
|
|
25
|
+
# .agent-review/config.json. `df skills install verify` does NOT overwrite it.
|
|
26
|
+
#
|
|
27
|
+
# Usage (normally invoked by the route-runner via `df verify --route playwright`):
|
|
28
|
+
# bash scripts/verify/playwright-route.sh [--commit <ref>]
|
|
29
|
+
|
|
30
|
+
set -uo pipefail
|
|
31
|
+
|
|
32
|
+
EXIT_GREEN=0
|
|
33
|
+
EXIT_BLOCK=1
|
|
34
|
+
EXIT_SOFT_SKIP=2
|
|
35
|
+
|
|
36
|
+
# --- consumer-tunable knobs (env overrides) --------------------------------
|
|
37
|
+
# Directory where your web app + @playwright/test live (Playwright runs there).
|
|
38
|
+
WEB_DIR="${DF_UI_ROUTE_WEB_DIR:-web}"
|
|
39
|
+
# The producer config, relative to WEB_DIR.
|
|
40
|
+
ROUTE_CONFIG="${DF_UI_ROUTE_CONFIG:-playwright.ui-route.config.ts}"
|
|
41
|
+
|
|
42
|
+
REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
|
43
|
+
cd "${REPO_ROOT}"
|
|
44
|
+
|
|
45
|
+
# Path of THIS script relative to the repo root — added to the clean-tree guard
|
|
46
|
+
# so a dirty producer (which would run DIFFERENT logic than the committed SHA)
|
|
47
|
+
# fails closed too. Computed dynamically so it works wherever you copy it.
|
|
48
|
+
SCRIPT_ABS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")"
|
|
49
|
+
SCRIPT_REL="${SCRIPT_ABS#"${REPO_ROOT}/"}"
|
|
50
|
+
|
|
51
|
+
# --- resolve the SHA under review ------------------------------------------
|
|
52
|
+
# Prefer DF_VERIFY_COMMIT (exported by a route orchestrator that pins the gated
|
|
53
|
+
# commit), then an explicit --commit, then HEAD.
|
|
54
|
+
SHA_REF="${DF_VERIFY_COMMIT:-HEAD}"
|
|
55
|
+
while [[ $# -gt 0 ]]; do
|
|
56
|
+
case "$1" in
|
|
57
|
+
--commit) SHA_REF="${2:-HEAD}"; shift 2 ;;
|
|
58
|
+
--commit=*) SHA_REF="${1#*=}"; shift ;;
|
|
59
|
+
*) shift ;;
|
|
60
|
+
esac
|
|
61
|
+
done
|
|
62
|
+
SHA="$(git rev-parse "${SHA_REF}" 2>/dev/null || echo "")"
|
|
63
|
+
if [[ -z "${SHA}" ]]; then
|
|
64
|
+
echo "[playwright-route] FAIL: could not resolve commit '${SHA_REF}'." >&2
|
|
65
|
+
exit "${EXIT_BLOCK}"
|
|
66
|
+
fi
|
|
67
|
+
|
|
68
|
+
# SHA-binding guard: the producer drives a browser against the CHECKED-OUT
|
|
69
|
+
# working tree (HEAD), so it can only honestly attest evidence for HEAD. If
|
|
70
|
+
# asked to produce evidence for a non-HEAD SHA, FAIL CLOSED rather than stamp a
|
|
71
|
+
# passing record whose screenshots actually reflect HEAD.
|
|
72
|
+
HEAD_SHA="$(git rev-parse HEAD 2>/dev/null || echo "")"
|
|
73
|
+
if [[ "${SHA}" != "${HEAD_SHA}" ]]; then
|
|
74
|
+
echo "[playwright-route] BLOCK: asked to produce evidence for ${SHA:0:12} but HEAD is ${HEAD_SHA:0:12}." >&2
|
|
75
|
+
echo "[playwright-route] the producer validates the checked-out tree (HEAD) only; refusing to mis-bind evidence. Check out the target SHA and re-run." >&2
|
|
76
|
+
exit "${EXIT_BLOCK}"
|
|
77
|
+
fi
|
|
78
|
+
|
|
79
|
+
# Clean-tree guard: the producer screenshots the WORKING TREE, but evidence
|
|
80
|
+
# binds to the committed SHA. If any file that affects either the rendered UI OR
|
|
81
|
+
# the route's execution/evidence contract is dirty or untracked, the capture
|
|
82
|
+
# would attest code that is NOT in ${SHA}. FAIL CLOSED. The guarded set is the
|
|
83
|
+
# UI surface (web/** + *.tsx) AND the route machinery itself (this script + the
|
|
84
|
+
# route config) — a dirty producer runs different logic than the committed SHA.
|
|
85
|
+
GUARDED_PATHS=(
|
|
86
|
+
"${WEB_DIR}/**" '*.tsx'
|
|
87
|
+
"${SCRIPT_REL}"
|
|
88
|
+
'.agent-review/config.json'
|
|
89
|
+
)
|
|
90
|
+
if ! git diff-index --quiet HEAD -- "${GUARDED_PATHS[@]}" 2>/dev/null; then
|
|
91
|
+
echo "[playwright-route] BLOCK: tracked UI or route-machinery files are dirty (uncommitted changes)." >&2
|
|
92
|
+
echo "[playwright-route] the producer captures the working tree but evidence binds to ${SHA:0:12}; commit or stash changes under ${WEB_DIR}/**, *.tsx, ${SCRIPT_REL}, or .agent-review/config.json, then re-run." >&2
|
|
93
|
+
exit "${EXIT_BLOCK}"
|
|
94
|
+
fi
|
|
95
|
+
UNTRACKED_GUARDED="$(git ls-files --others --exclude-standard -- "${GUARDED_PATHS[@]}" 2>/dev/null | head -1 || true)"
|
|
96
|
+
if [[ -n "${UNTRACKED_GUARDED}" ]]; then
|
|
97
|
+
echo "[playwright-route] BLOCK: untracked UI/route-machinery file(s) present (e.g. ${UNTRACKED_GUARDED}) not in ${SHA:0:12}." >&2
|
|
98
|
+
echo "[playwright-route] commit or remove untracked files under the guarded set so the capture matches the gated commit, then re-run." >&2
|
|
99
|
+
exit "${EXIT_BLOCK}"
|
|
100
|
+
fi
|
|
101
|
+
|
|
102
|
+
echo "[playwright-route] producing UI evidence for ${SHA:0:12} (ref=${SHA_REF}, =HEAD, clean tree)" >&2
|
|
103
|
+
|
|
104
|
+
# Compute the UI paths this commit actually changed (parent..SHA), restricted to
|
|
105
|
+
# the route's trigger surface (web/** + *.tsx). Handed to the producer so it can
|
|
106
|
+
# FAIL CLOSED when a changed UI path has no mapped capture surface. Excludes
|
|
107
|
+
# deletions (D) — a removed file has no surface to render.
|
|
108
|
+
PARENT="$(git rev-parse "${SHA}^" 2>/dev/null || echo "")"
|
|
109
|
+
if [[ -n "${PARENT}" ]]; then
|
|
110
|
+
CHANGED_UI_PATHS="$(git diff --name-only --diff-filter=ACMRT "${PARENT}" "${SHA}" -- "${WEB_DIR}/**" '*.tsx' 2>/dev/null || true)"
|
|
111
|
+
else
|
|
112
|
+
# Root commit — list all tracked UI paths.
|
|
113
|
+
CHANGED_UI_PATHS="$(git ls-files -- "${WEB_DIR}/**" '*.tsx' 2>/dev/null || true)"
|
|
114
|
+
fi
|
|
115
|
+
export DF_UI_ROUTE_CHANGED_PATHS="${CHANGED_UI_PATHS}"
|
|
116
|
+
|
|
117
|
+
# Anchor the evidence dir at the REPO ROOT (absolute), because Playwright runs
|
|
118
|
+
# with cwd=${WEB_DIR} — a relative path would land under ${WEB_DIR}.
|
|
119
|
+
OUT_DIR="${REPO_ROOT}/agent-reviews/quality-gates/ui/${SHA}"
|
|
120
|
+
mkdir -p "${OUT_DIR}"
|
|
121
|
+
export DF_UI_ROUTE_SHA="${SHA}"
|
|
122
|
+
export DF_UI_ROUTE_OUT="${OUT_DIR}"
|
|
123
|
+
|
|
124
|
+
# --- preflight: web tier present + Playwright installed --------------------
|
|
125
|
+
if [[ ! -d "${WEB_DIR}/node_modules/@playwright" ]]; then
|
|
126
|
+
echo "[playwright-route] SOFT-SKIP: ${WEB_DIR}/ Playwright not installed (run \`cd ${WEB_DIR} && npm ci\`)." >&2
|
|
127
|
+
echo "[playwright-route] not fabricating evidence — requiresHumanJudgment." >&2
|
|
128
|
+
exit "${EXIT_SOFT_SKIP}"
|
|
129
|
+
fi
|
|
130
|
+
|
|
131
|
+
# --- auth storage-state gate (optional) ------------------------------------
|
|
132
|
+
# Public surfaces need no auth. If you configured an auth storage-state
|
|
133
|
+
# (DF_UI_ROUTE_STORAGE_STATE) but the file is absent (e.g. a cloud sandbox where
|
|
134
|
+
# your auth harness could not mint it), SOFT-SKIP rather than pass — we will not
|
|
135
|
+
# produce a protected-surface artifact we cannot legitimately authenticate.
|
|
136
|
+
if [[ -n "${DF_UI_ROUTE_STORAGE_STATE:-}" && ! -f "${DF_UI_ROUTE_STORAGE_STATE}" ]]; then
|
|
137
|
+
echo "[playwright-route] SOFT-SKIP: DF_UI_ROUTE_STORAGE_STATE=${DF_UI_ROUTE_STORAGE_STATE} is set but the file is absent." >&2
|
|
138
|
+
echo "[playwright-route] cannot authenticate the configured session; not fabricating evidence (requiresHumanJudgment)." >&2
|
|
139
|
+
exit "${EXIT_SOFT_SKIP}"
|
|
140
|
+
fi
|
|
141
|
+
|
|
142
|
+
# --- run the producer -------------------------------------------------------
|
|
143
|
+
LOG_FILE="$(mktemp -t df-playwright-route.XXXXXX)"
|
|
144
|
+
trap 'rm -f "${LOG_FILE}"' EXIT
|
|
145
|
+
|
|
146
|
+
set +e
|
|
147
|
+
( cd "${WEB_DIR}" && npx playwright test --config "${ROUTE_CONFIG}" ) \
|
|
148
|
+
>"${LOG_FILE}" 2>&1
|
|
149
|
+
RC=$?
|
|
150
|
+
set -e 2>/dev/null || true
|
|
151
|
+
|
|
152
|
+
# Surface the producer output (bounded) for the gate log / critic.
|
|
153
|
+
tail -n 120 "${LOG_FILE}" >&2 || true
|
|
154
|
+
|
|
155
|
+
if [[ ${RC} -eq 0 ]]; then
|
|
156
|
+
echo "[playwright-route] OK — evidence in ${OUT_DIR}" >&2
|
|
157
|
+
exit "${EXIT_GREEN}"
|
|
158
|
+
fi
|
|
159
|
+
|
|
160
|
+
echo "[playwright-route] BLOCK: producer failed (exit ${RC}) — see output above." >&2
|
|
161
|
+
exit "${EXIT_BLOCK}"
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { defineConfig, devices } from "@playwright/test";
|
|
2
|
+
import { existsSync } from "node:fs";
|
|
3
|
+
import { resolve } from "node:path";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Reusable Playwright config for the Evidence-Gated Validation **playwright
|
|
7
|
+
* (UI) route producer** (momentiq-ai/dark-factory#193). Generalized from the
|
|
8
|
+
* dark-factory-dashboard dogfood (Cycle 21 PR-4; ADR 2026-06).
|
|
9
|
+
*
|
|
10
|
+
* Kept SEPARATE from your broad e2e config so the route producer is a single,
|
|
11
|
+
* fast, deterministic capture:
|
|
12
|
+
* - one chromium project, the producer spec only.
|
|
13
|
+
* - no retries (a flaky retry could mask a genuinely-failing UI surface).
|
|
14
|
+
* - an OPTIONAL authenticated storage-state (public surfaces need none).
|
|
15
|
+
* - an OPTIONAL local preview `webServer` (the SHA under review) for the
|
|
16
|
+
* no-auth dogfood path; set `PLAYWRIGHT_TEST_BASE_URL` to point at a
|
|
17
|
+
* deployed preview of the SHA instead.
|
|
18
|
+
*
|
|
19
|
+
* This config is invoked by `playwright-route.sh`, which is the route's
|
|
20
|
+
* `command` in `.agent-review/config.json`.
|
|
21
|
+
*
|
|
22
|
+
* ── CONSUMER REFERENCE FILE — copy into your repo (next to the producer spec)
|
|
23
|
+
* and OWN it. Adjust `testDir`, the `webServer.command`, and the env knobs
|
|
24
|
+
* to your repo's layout. `df skills install verify` does NOT overwrite it.
|
|
25
|
+
*
|
|
26
|
+
* NEVER set a routine pre-push bypass to get around a failing producer — a
|
|
27
|
+
* failing/absent capture MUST block the route (that is the feature).
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
// Auth is OPTIONAL and consumer-supplied. Mint an authenticated Playwright
|
|
31
|
+
// storageState JSON with YOUR auth (Clerk/Auth0/bot login/etc.) and point
|
|
32
|
+
// DF_UI_ROUTE_STORAGE_STATE at it for protected surfaces. Public surfaces need
|
|
33
|
+
// none — when the file is absent we run unauthenticated rather than fail.
|
|
34
|
+
//
|
|
35
|
+
// The default path is anchored to `process.cwd()` — NOT `__dirname` or
|
|
36
|
+
// `import.meta.url`. Playwright may load this config as CJS or ESM depending on
|
|
37
|
+
// the consumer's `"type"` + Playwright version: `__dirname` is undefined under
|
|
38
|
+
// ESM and `import.meta` is invalid under CJS, so either module-global would
|
|
39
|
+
// crash the producer for half of consumers before it could even soft-skip. The
|
|
40
|
+
// route script runs Playwright with `cwd` = the web dir (where this config
|
|
41
|
+
// lives), so cwd-relative resolves to `<web>/.auth/storage-state.json` and
|
|
42
|
+
// works in both. Gitignore **/.auth/.
|
|
43
|
+
const STORAGE_STATE_PATH =
|
|
44
|
+
process.env.DF_UI_ROUTE_STORAGE_STATE ||
|
|
45
|
+
resolve(process.cwd(), ".auth/storage-state.json");
|
|
46
|
+
const storageState = existsSync(STORAGE_STATE_PATH) ? STORAGE_STATE_PATH : undefined;
|
|
47
|
+
|
|
48
|
+
// Local preview port for the SHA-under-review server. Overridable via
|
|
49
|
+
// DF_UI_ROUTE_PORT to avoid collisions with other local services.
|
|
50
|
+
const PREVIEW_PORT = process.env.DF_UI_ROUTE_PORT || "3217";
|
|
51
|
+
const BASE_URL =
|
|
52
|
+
process.env.PLAYWRIGHT_TEST_BASE_URL?.replace(/\/$/, "") ||
|
|
53
|
+
`http://localhost:${PREVIEW_PORT}`;
|
|
54
|
+
|
|
55
|
+
// Only stand up the local preview server when we are NOT pointed at a remote
|
|
56
|
+
// deployed surface. (A remote base URL means the SHA is already served.)
|
|
57
|
+
const usingLocalPreview =
|
|
58
|
+
!process.env.PLAYWRIGHT_TEST_BASE_URL ||
|
|
59
|
+
/^https?:\/\/localhost(:|\/|$)/.test(process.env.PLAYWRIGHT_TEST_BASE_URL);
|
|
60
|
+
|
|
61
|
+
// The command that serves the SHA under review for the local-preview path.
|
|
62
|
+
// Default `npm run dev`; override via DF_UI_ROUTE_DEV_CMD for your stack.
|
|
63
|
+
const DEV_CMD = process.env.DF_UI_ROUTE_DEV_CMD || "npm run dev";
|
|
64
|
+
|
|
65
|
+
export default defineConfig({
|
|
66
|
+
testDir: "./tests/e2e",
|
|
67
|
+
// Only the route producer — not your broad e2e suite.
|
|
68
|
+
testMatch: /ui-route\.producer\.spec\.ts$/,
|
|
69
|
+
fullyParallel: false,
|
|
70
|
+
forbidOnly: !!process.env.CI,
|
|
71
|
+
// The producer must be deterministic; no flaky retries that could mask a
|
|
72
|
+
// genuinely-failing UI surface behind a lucky re-run.
|
|
73
|
+
retries: 0,
|
|
74
|
+
workers: 1,
|
|
75
|
+
timeout: 90 * 1000,
|
|
76
|
+
expect: { timeout: 15 * 1000 },
|
|
77
|
+
reporter: [["list"]],
|
|
78
|
+
|
|
79
|
+
use: {
|
|
80
|
+
baseURL: BASE_URL,
|
|
81
|
+
// Authenticate ONLY when a storage-state file is present (protected
|
|
82
|
+
// surfaces). Public surfaces ignore it.
|
|
83
|
+
...(storageState ? { storageState } : {}),
|
|
84
|
+
trace: "retain-on-failure",
|
|
85
|
+
screenshot: "only-on-failure",
|
|
86
|
+
ignoreHTTPSErrors: true,
|
|
87
|
+
actionTimeout: 15 * 1000,
|
|
88
|
+
navigationTimeout: 30 * 1000,
|
|
89
|
+
},
|
|
90
|
+
|
|
91
|
+
projects: [
|
|
92
|
+
{
|
|
93
|
+
name: "ui-route-producer",
|
|
94
|
+
use: { ...devices["Desktop Chrome"] },
|
|
95
|
+
},
|
|
96
|
+
],
|
|
97
|
+
|
|
98
|
+
webServer: usingLocalPreview
|
|
99
|
+
? {
|
|
100
|
+
// Serve the SHA under review. Set the port via the PORT env var (the
|
|
101
|
+
// common convention); adjust DEV_CMD if your dev server differs.
|
|
102
|
+
command: DEV_CMD,
|
|
103
|
+
url: BASE_URL,
|
|
104
|
+
reuseExistingServer: !process.env.CI,
|
|
105
|
+
timeout: 120 * 1000,
|
|
106
|
+
env: {
|
|
107
|
+
PORT: PREVIEW_PORT,
|
|
108
|
+
},
|
|
109
|
+
}
|
|
110
|
+
: undefined,
|
|
111
|
+
});
|