@devfellowship/components 3.0.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/testing.cjs +171 -0
- package/dist/testing.d.cts +210 -0
- package/dist/testing.d.ts +210 -0
- package/dist/testing.js +140 -0
- package/package.json +8 -2
package/dist/testing.cjs
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
|
|
20
|
+
// src/testing/index.ts
|
|
21
|
+
var testing_exports = {};
|
|
22
|
+
__export(testing_exports, {
|
|
23
|
+
IAM_MEMBER_MIN_LEVEL: () => IAM_MEMBER_MIN_LEVEL,
|
|
24
|
+
VacuousVerificationError: () => VacuousVerificationError,
|
|
25
|
+
assertNotVacuouslyEmpty: () => assertNotVacuouslyEmpty,
|
|
26
|
+
classifyEmptiness: () => classifyEmptiness,
|
|
27
|
+
iamMemberProbe: () => iamMemberProbe
|
|
28
|
+
});
|
|
29
|
+
module.exports = __toCommonJS(testing_exports);
|
|
30
|
+
|
|
31
|
+
// src/testing/emptiness.ts
|
|
32
|
+
function classifyEmptiness(input) {
|
|
33
|
+
const { rowCount, deniedSignalPresent, viewerIsAuthorized } = input;
|
|
34
|
+
if (rowCount > 0) {
|
|
35
|
+
return {
|
|
36
|
+
verdict: "populated",
|
|
37
|
+
reason: `Surface rendered ${rowCount} row(s); the result is non-empty, so there is no emptiness to disambiguate.`
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
if (deniedSignalPresent) {
|
|
41
|
+
return {
|
|
42
|
+
verdict: "denied",
|
|
43
|
+
reason: "Surface rendered 0 rows AND an explicit permission-denied signal. The app failed loud, so the emptiness is attributable to a stated authorization failure rather than to a silent RLS filter \u2014 an assertion on this state is meaningful."
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
if (viewerIsAuthorized === true) {
|
|
47
|
+
return {
|
|
48
|
+
verdict: "genuinely-empty",
|
|
49
|
+
reason: "Surface rendered 0 rows with no permission-denied signal, and an independent authorization probe confirmed the viewer COULD have seen rows. The emptiness therefore reflects the data, not the viewer's access."
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
const authClause = viewerIsAuthorized === false ? "an independent probe reported the viewer is NOT authorized" : "the viewer's authorization could not be confirmed (unknown)";
|
|
53
|
+
return {
|
|
54
|
+
verdict: "vacuous",
|
|
55
|
+
reason: `Surface rendered 0 rows, the page rendered NO permission-denied signal, and ${authClause}. Supabase RLS hides unauthorized rows as HTTP 200 with an empty body, which is byte-identical to a genuinely empty result \u2014 so this emptiness is indistinguishable from lack of access and the check proves nothing. Fix by supplying an independent authorization signal (e.g. \`iamMemberProbe\`), by using an identity that can see rows, or by making the surface render an explicit permission-denied state.`
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// src/testing/assert-not-vacuously-empty.ts
|
|
60
|
+
var VacuousVerificationError = class extends Error {
|
|
61
|
+
constructor(details) {
|
|
62
|
+
super(
|
|
63
|
+
[
|
|
64
|
+
`Vacuous verification on "${details.surface}": this check proves nothing.`,
|
|
65
|
+
"",
|
|
66
|
+
` \u2022 the surface returned ${details.rowCount} rows (nothing to assert against)`,
|
|
67
|
+
" \u2022 the page rendered NO permission-denied signal, so the app did not fail loud",
|
|
68
|
+
details.viewerIsAuthorized === false ? " \u2022 the viewer is known NOT to be authorized" : " \u2022 the viewer's authorization could NOT be confirmed",
|
|
69
|
+
"",
|
|
70
|
+
details.reason
|
|
71
|
+
].join("\n")
|
|
72
|
+
);
|
|
73
|
+
this.verdict = "vacuous";
|
|
74
|
+
this.name = "VacuousVerificationError";
|
|
75
|
+
this.surface = details.surface;
|
|
76
|
+
this.rowCount = details.rowCount;
|
|
77
|
+
this.deniedSignalPresent = details.deniedSignalPresent;
|
|
78
|
+
this.viewerIsAuthorized = details.viewerIsAuthorized;
|
|
79
|
+
this.reason = details.reason;
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
async function resolveAuthorization(input) {
|
|
83
|
+
if (input === void 0) return "unknown";
|
|
84
|
+
if (typeof input !== "function") return input;
|
|
85
|
+
try {
|
|
86
|
+
const result = await input();
|
|
87
|
+
return result === true || result === false ? result : "unknown";
|
|
88
|
+
} catch {
|
|
89
|
+
return "unknown";
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
async function resolveRowCount(page, opts) {
|
|
93
|
+
if (opts.countText) {
|
|
94
|
+
const text = await page.locator(opts.countTextSelector ?? "body").innerText();
|
|
95
|
+
for (const rawLine of text.split("\n")) {
|
|
96
|
+
const match = opts.countText.exec(rawLine.trim());
|
|
97
|
+
if (match) {
|
|
98
|
+
const parsed = Number(match[1]);
|
|
99
|
+
if (Number.isFinite(parsed)) return parsed;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if (!opts.rowSelector) {
|
|
103
|
+
throw new Error(
|
|
104
|
+
`assertNotVacuouslyEmpty: countText ${String(opts.countText)} matched no line of "${opts.countTextSelector ?? "body"}". The row count is unreadable, so no verdict is possible \u2014 fix the pattern or pass rowSelector.`
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
if (opts.rowSelector) return page.locator(opts.rowSelector).count();
|
|
109
|
+
throw new Error(
|
|
110
|
+
"assertNotVacuouslyEmpty: pass countText and/or rowSelector \u2014 without one there is no row count to classify."
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
async function assertNotVacuouslyEmpty(page, opts = {}) {
|
|
114
|
+
const surface = opts.surface ?? "unnamed surface";
|
|
115
|
+
const rowCount = await resolveRowCount(page, opts);
|
|
116
|
+
const deniedSignalPresent = opts.deniedSelector ? await page.locator(opts.deniedSelector).count() > 0 : false;
|
|
117
|
+
const viewerIsAuthorized = await resolveAuthorization(opts.viewerIsAuthorized);
|
|
118
|
+
const { verdict, reason } = classifyEmptiness({
|
|
119
|
+
rowCount,
|
|
120
|
+
deniedSignalPresent,
|
|
121
|
+
viewerIsAuthorized
|
|
122
|
+
});
|
|
123
|
+
if (verdict === "vacuous") {
|
|
124
|
+
throw new VacuousVerificationError({
|
|
125
|
+
surface,
|
|
126
|
+
rowCount,
|
|
127
|
+
deniedSignalPresent,
|
|
128
|
+
viewerIsAuthorized,
|
|
129
|
+
reason
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
return verdict;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// src/testing/iam-member-probe.ts
|
|
136
|
+
var IAM_MEMBER_MIN_LEVEL = 50;
|
|
137
|
+
function readLevel(row) {
|
|
138
|
+
if (typeof row !== "object" || row === null) return null;
|
|
139
|
+
const level = row.level;
|
|
140
|
+
if (typeof level === "number" && Number.isFinite(level)) return level;
|
|
141
|
+
if (typeof level === "string" && level.trim() !== "") {
|
|
142
|
+
const parsed = Number(level);
|
|
143
|
+
if (Number.isFinite(parsed)) return parsed;
|
|
144
|
+
}
|
|
145
|
+
return null;
|
|
146
|
+
}
|
|
147
|
+
async function iamMemberProbe(supabaseClient, opts = {}) {
|
|
148
|
+
const minLevel = opts.minLevel ?? IAM_MEMBER_MIN_LEVEL;
|
|
149
|
+
const rpcName = opts.rpcName ?? "get_my_iam_role";
|
|
150
|
+
let data;
|
|
151
|
+
let error;
|
|
152
|
+
try {
|
|
153
|
+
({ data, error } = await supabaseClient.rpc(rpcName));
|
|
154
|
+
} catch {
|
|
155
|
+
return "unknown";
|
|
156
|
+
}
|
|
157
|
+
if (error) return "unknown";
|
|
158
|
+
const rows = Array.isArray(data) ? data : data == null ? [] : [data];
|
|
159
|
+
if (rows.length === 0) return "unknown";
|
|
160
|
+
const levels = rows.map(readLevel).filter((l) => l !== null);
|
|
161
|
+
if (levels.length === 0) return "unknown";
|
|
162
|
+
return levels.some((level) => level >= minLevel);
|
|
163
|
+
}
|
|
164
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
165
|
+
0 && (module.exports = {
|
|
166
|
+
IAM_MEMBER_MIN_LEVEL,
|
|
167
|
+
VacuousVerificationError,
|
|
168
|
+
assertNotVacuouslyEmpty,
|
|
169
|
+
classifyEmptiness,
|
|
170
|
+
iamMemberProbe
|
|
171
|
+
});
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Layer 1 — the pure decision. No I/O, no Playwright, no Supabase.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS EXISTS
|
|
5
|
+
* ---------------
|
|
6
|
+
* Agents verify UI through Playwright using a shared smoke identity that has
|
|
7
|
+
* IAM global level 0. Supabase RLS on `iam.is_member()`-gated tables returns
|
|
8
|
+
* ZERO ROWS WITH HTTP 200 — not 403. So the page renders "no results" and an
|
|
9
|
+
* assertion like `expect(rows).toHaveCount(0)` or "the page loaded without an
|
|
10
|
+
* error" passes VACUOUSLY: satisfied by absence of permission rather than by
|
|
11
|
+
* the state under test.
|
|
12
|
+
*
|
|
13
|
+
* Measured 2026-08-07: spec-builder `/history` showed "0 registros" to the
|
|
14
|
+
* smoke account while `work.ai_spec_inputs` held 17 rows. Every assertion on
|
|
15
|
+
* that page was green.
|
|
16
|
+
*
|
|
17
|
+
* THE HONEST PART
|
|
18
|
+
* ---------------
|
|
19
|
+
* PostgREST cannot tell "RLS hid it" from "genuinely empty" — both are
|
|
20
|
+
* `200 []`. There is no detector that can recover the difference from the
|
|
21
|
+
* response alone, and this module does not pretend otherwise. What it does
|
|
22
|
+
* instead is NAME the ambiguity and refuse to let a test claim a green it did
|
|
23
|
+
* not earn: an empty surface only proves something when we have an
|
|
24
|
+
* INDEPENDENT signal that the viewer was authorized to see rows (or that the
|
|
25
|
+
* app told us, loudly, that it was not).
|
|
26
|
+
*/
|
|
27
|
+
/**
|
|
28
|
+
* The four states an empty-looking surface can actually be in.
|
|
29
|
+
*
|
|
30
|
+
* - `populated` — rows are present. Nothing ambiguous.
|
|
31
|
+
* - `denied` — empty, and the app rendered a permission-denied
|
|
32
|
+
* signal. The app failed LOUD; that is correct
|
|
33
|
+
* behaviour, and asserting emptiness here is meaningful.
|
|
34
|
+
* - `genuinely-empty` — empty, no denied signal, and we independently know the
|
|
35
|
+
* viewer WAS authorized. Safe: the emptiness is real.
|
|
36
|
+
* - `vacuous` — empty, no denied signal, and the viewer's
|
|
37
|
+
* authorization is `false` or could not be confirmed.
|
|
38
|
+
* The check proves nothing. This is the failure.
|
|
39
|
+
*/
|
|
40
|
+
type EmptinessVerdict = "populated" | "denied" | "genuinely-empty" | "vacuous";
|
|
41
|
+
interface ClassifyEmptinessInput {
|
|
42
|
+
/** How many rows/items the surface actually rendered. */
|
|
43
|
+
rowCount: number;
|
|
44
|
+
/**
|
|
45
|
+
* Did the page render an explicit permission-denied affordance (banner,
|
|
46
|
+
* `data-testid="access-denied"`, 403 state)? An app that fails loud is
|
|
47
|
+
* verifiable; an app that renders a silent empty list is not.
|
|
48
|
+
*/
|
|
49
|
+
deniedSignalPresent: boolean;
|
|
50
|
+
/**
|
|
51
|
+
* INDEPENDENT authorization signal — must NOT be derived from the same
|
|
52
|
+
* response whose emptiness is in question, or the reasoning is circular.
|
|
53
|
+
* `'unknown'` is the correct value when no independent probe ran; it is
|
|
54
|
+
* deliberately treated exactly as harshly as `false`.
|
|
55
|
+
*/
|
|
56
|
+
viewerIsAuthorized: boolean | "unknown";
|
|
57
|
+
}
|
|
58
|
+
interface EmptinessClassification {
|
|
59
|
+
verdict: EmptinessVerdict;
|
|
60
|
+
/** Human-readable justification, safe to embed in a failure message. */
|
|
61
|
+
reason: string;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Classify what an (apparently) empty surface actually proves.
|
|
65
|
+
*
|
|
66
|
+
* Pure function: same input, same verdict, no clock, no network. Every I/O
|
|
67
|
+
* concern lives in Layer 2 (`assertNotVacuouslyEmpty`).
|
|
68
|
+
*/
|
|
69
|
+
declare function classifyEmptiness(input: ClassifyEmptinessInput): EmptinessClassification;
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Structural (duck-typed) subset of the Playwright surface this entry needs.
|
|
73
|
+
*
|
|
74
|
+
* WHY NOT `import type { Page } from "@playwright/test"`
|
|
75
|
+
* -----------------------------------------------------
|
|
76
|
+
* `@devfellowship/components` is a runtime dependency of every DFL app.
|
|
77
|
+
* Adding Playwright — even as a type-only import — would put a ~100MB test
|
|
78
|
+
* runner into the dependency graph of production bundles and would make this
|
|
79
|
+
* entry impossible to unit-test without a browser. So we declare the three
|
|
80
|
+
* methods we actually call and nothing else.
|
|
81
|
+
*
|
|
82
|
+
* A real Playwright `Page` / `Locator` satisfies these interfaces
|
|
83
|
+
* structurally: no adapter, no cast, no `@playwright/test` in package.json.
|
|
84
|
+
* A plain object literal satisfies them too, which is how the tests work.
|
|
85
|
+
*/
|
|
86
|
+
/** The slice of a Playwright `Locator` used here. */
|
|
87
|
+
interface LocatorLike {
|
|
88
|
+
/** Number of DOM elements the selector currently resolves to. */
|
|
89
|
+
count(): Promise<number>;
|
|
90
|
+
/** Rendered (visible) text of the resolved element. */
|
|
91
|
+
innerText(): Promise<string>;
|
|
92
|
+
}
|
|
93
|
+
/** The slice of a Playwright `Page` used here. */
|
|
94
|
+
interface PageLike {
|
|
95
|
+
locator(selector: string): LocatorLike;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Minimal shape of a `supabase-js` client — only `.rpc()`, only the fields
|
|
99
|
+
* `iamMemberProbe` reads. Keeps `@supabase/supabase-js` out of this entry's
|
|
100
|
+
* imports entirely (it is an OPTIONAL peer dependency of the package).
|
|
101
|
+
*/
|
|
102
|
+
interface SupabaseRpcClientLike {
|
|
103
|
+
rpc(fn: string, args?: Record<string, unknown>): PromiseLike<{
|
|
104
|
+
data: unknown;
|
|
105
|
+
error: unknown;
|
|
106
|
+
}>;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Thrown when a surface's emptiness is indistinguishable from lack of access.
|
|
111
|
+
*
|
|
112
|
+
* This is NOT "the page is broken" — it is "this check cannot tell you whether
|
|
113
|
+
* the page is broken", which is the more dangerous state, because the default
|
|
114
|
+
* assertion would have passed.
|
|
115
|
+
*/
|
|
116
|
+
declare class VacuousVerificationError extends Error {
|
|
117
|
+
readonly verdict: EmptinessVerdict;
|
|
118
|
+
readonly surface: string;
|
|
119
|
+
readonly rowCount: number;
|
|
120
|
+
readonly deniedSignalPresent: boolean;
|
|
121
|
+
readonly viewerIsAuthorized: boolean | "unknown";
|
|
122
|
+
readonly reason: string;
|
|
123
|
+
constructor(details: {
|
|
124
|
+
surface: string;
|
|
125
|
+
rowCount: number;
|
|
126
|
+
deniedSignalPresent: boolean;
|
|
127
|
+
viewerIsAuthorized: boolean | "unknown";
|
|
128
|
+
reason: string;
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
interface AssertNotVacuouslyEmptyOptions {
|
|
132
|
+
/**
|
|
133
|
+
* Regex matched against the surface's rendered text, LINE BY LINE (each line
|
|
134
|
+
* trimmed), so anchored patterns work — e.g. `/^(\d+) registros?$/`.
|
|
135
|
+
* Capture group 1 must be the count.
|
|
136
|
+
*/
|
|
137
|
+
countText?: RegExp;
|
|
138
|
+
/** CSS/testid selector whose element holds the count text. Default `body`. */
|
|
139
|
+
countTextSelector?: string;
|
|
140
|
+
/** Alternative to `countText`: count the elements matching this selector. */
|
|
141
|
+
rowSelector?: string;
|
|
142
|
+
/**
|
|
143
|
+
* Selector for the element/testid the app renders on permission denial.
|
|
144
|
+
* If the app has no such affordance, omit it — but then a denied surface is
|
|
145
|
+
* unverifiable by construction, which is exactly what `vacuous` reports.
|
|
146
|
+
*/
|
|
147
|
+
deniedSelector?: string;
|
|
148
|
+
/**
|
|
149
|
+
* INDEPENDENT authorization signal. A thunk (e.g. `() => iamMemberProbe(sb)`)
|
|
150
|
+
* is awaited; if it throws, the result is `'unknown'`.
|
|
151
|
+
* Omitted defaults to `'unknown'` — treated as harshly as `false`.
|
|
152
|
+
*/
|
|
153
|
+
viewerIsAuthorized?: boolean | "unknown" | (() => Promise<boolean | "unknown">);
|
|
154
|
+
/** Label used in the failure message. Default `unnamed surface`. */
|
|
155
|
+
surface?: string;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Layer 2 — the e2e adapter. Reads the three facts off a Playwright-ish page,
|
|
159
|
+
* hands them to the pure {@link classifyEmptiness}, and refuses to return a
|
|
160
|
+
* green it cannot justify.
|
|
161
|
+
*
|
|
162
|
+
* Returns the {@link EmptinessVerdict} for `populated`, `denied` and
|
|
163
|
+
* `genuinely-empty`. Throws {@link VacuousVerificationError} for `vacuous`.
|
|
164
|
+
*
|
|
165
|
+
* @example
|
|
166
|
+
* // Fails today against the level-0 smoke identity — by design.
|
|
167
|
+
* await assertNotVacuouslyEmpty(page, {
|
|
168
|
+
* surface: "spec-builder /history",
|
|
169
|
+
* countText: /^(\d+) registros?$/,
|
|
170
|
+
* deniedSelector: '[data-testid="access-denied"]',
|
|
171
|
+
* viewerIsAuthorized: () => iamMemberProbe(supabase),
|
|
172
|
+
* });
|
|
173
|
+
*/
|
|
174
|
+
declare function assertNotVacuouslyEmpty(page: PageLike, opts?: AssertNotVacuouslyEmptyOptions): Promise<EmptinessVerdict>;
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Default IAM level that counts as "member enough to see gated rows".
|
|
178
|
+
* Mirrors the threshold `iam.is_member()` enforces server-side.
|
|
179
|
+
*/
|
|
180
|
+
declare const IAM_MEMBER_MIN_LEVEL = 50;
|
|
181
|
+
/**
|
|
182
|
+
* Independent authorization probe: ask the database who the caller is,
|
|
183
|
+
* instead of inferring it from the very response whose emptiness is in doubt.
|
|
184
|
+
*
|
|
185
|
+
* Calls the existing `get_my_iam_role()` RPC (already granted to
|
|
186
|
+
* `authenticated`; returns rows of `{ role_id, level }`) and resolves whether
|
|
187
|
+
* any granted role clears `minLevel` (default {@link IAM_MEMBER_MIN_LEVEL}).
|
|
188
|
+
*
|
|
189
|
+
* FAILS TO `'unknown'`, NEVER TO `true`. An RPC error, a null payload, a
|
|
190
|
+
* non-array payload, or rows carrying no readable `level` all mean "we did not
|
|
191
|
+
* learn anything" — and `'unknown'` is treated by `classifyEmptiness` exactly
|
|
192
|
+
* as harshly as `false`. A probe that guessed optimistically would reintroduce
|
|
193
|
+
* the very false-green this module exists to prevent.
|
|
194
|
+
*
|
|
195
|
+
* Note that an empty row set is `'unknown'` rather than `false`: a level-0
|
|
196
|
+
* identity may legitimately produce zero rows, but so does a broken grant, and
|
|
197
|
+
* the two are not distinguishable from here. Both block a green.
|
|
198
|
+
*
|
|
199
|
+
* @example
|
|
200
|
+
* await assertNotVacuouslyEmpty(page, {
|
|
201
|
+
* countText: /^(\d+) registros?$/,
|
|
202
|
+
* viewerIsAuthorized: () => iamMemberProbe(supabase),
|
|
203
|
+
* });
|
|
204
|
+
*/
|
|
205
|
+
declare function iamMemberProbe(supabaseClient: SupabaseRpcClientLike, opts?: {
|
|
206
|
+
minLevel?: number;
|
|
207
|
+
rpcName?: string;
|
|
208
|
+
}): Promise<boolean | "unknown">;
|
|
209
|
+
|
|
210
|
+
export { type AssertNotVacuouslyEmptyOptions, type ClassifyEmptinessInput, type EmptinessClassification, type EmptinessVerdict, IAM_MEMBER_MIN_LEVEL, type LocatorLike, type PageLike, type SupabaseRpcClientLike, VacuousVerificationError, assertNotVacuouslyEmpty, classifyEmptiness, iamMemberProbe };
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Layer 1 — the pure decision. No I/O, no Playwright, no Supabase.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS EXISTS
|
|
5
|
+
* ---------------
|
|
6
|
+
* Agents verify UI through Playwright using a shared smoke identity that has
|
|
7
|
+
* IAM global level 0. Supabase RLS on `iam.is_member()`-gated tables returns
|
|
8
|
+
* ZERO ROWS WITH HTTP 200 — not 403. So the page renders "no results" and an
|
|
9
|
+
* assertion like `expect(rows).toHaveCount(0)` or "the page loaded without an
|
|
10
|
+
* error" passes VACUOUSLY: satisfied by absence of permission rather than by
|
|
11
|
+
* the state under test.
|
|
12
|
+
*
|
|
13
|
+
* Measured 2026-08-07: spec-builder `/history` showed "0 registros" to the
|
|
14
|
+
* smoke account while `work.ai_spec_inputs` held 17 rows. Every assertion on
|
|
15
|
+
* that page was green.
|
|
16
|
+
*
|
|
17
|
+
* THE HONEST PART
|
|
18
|
+
* ---------------
|
|
19
|
+
* PostgREST cannot tell "RLS hid it" from "genuinely empty" — both are
|
|
20
|
+
* `200 []`. There is no detector that can recover the difference from the
|
|
21
|
+
* response alone, and this module does not pretend otherwise. What it does
|
|
22
|
+
* instead is NAME the ambiguity and refuse to let a test claim a green it did
|
|
23
|
+
* not earn: an empty surface only proves something when we have an
|
|
24
|
+
* INDEPENDENT signal that the viewer was authorized to see rows (or that the
|
|
25
|
+
* app told us, loudly, that it was not).
|
|
26
|
+
*/
|
|
27
|
+
/**
|
|
28
|
+
* The four states an empty-looking surface can actually be in.
|
|
29
|
+
*
|
|
30
|
+
* - `populated` — rows are present. Nothing ambiguous.
|
|
31
|
+
* - `denied` — empty, and the app rendered a permission-denied
|
|
32
|
+
* signal. The app failed LOUD; that is correct
|
|
33
|
+
* behaviour, and asserting emptiness here is meaningful.
|
|
34
|
+
* - `genuinely-empty` — empty, no denied signal, and we independently know the
|
|
35
|
+
* viewer WAS authorized. Safe: the emptiness is real.
|
|
36
|
+
* - `vacuous` — empty, no denied signal, and the viewer's
|
|
37
|
+
* authorization is `false` or could not be confirmed.
|
|
38
|
+
* The check proves nothing. This is the failure.
|
|
39
|
+
*/
|
|
40
|
+
type EmptinessVerdict = "populated" | "denied" | "genuinely-empty" | "vacuous";
|
|
41
|
+
interface ClassifyEmptinessInput {
|
|
42
|
+
/** How many rows/items the surface actually rendered. */
|
|
43
|
+
rowCount: number;
|
|
44
|
+
/**
|
|
45
|
+
* Did the page render an explicit permission-denied affordance (banner,
|
|
46
|
+
* `data-testid="access-denied"`, 403 state)? An app that fails loud is
|
|
47
|
+
* verifiable; an app that renders a silent empty list is not.
|
|
48
|
+
*/
|
|
49
|
+
deniedSignalPresent: boolean;
|
|
50
|
+
/**
|
|
51
|
+
* INDEPENDENT authorization signal — must NOT be derived from the same
|
|
52
|
+
* response whose emptiness is in question, or the reasoning is circular.
|
|
53
|
+
* `'unknown'` is the correct value when no independent probe ran; it is
|
|
54
|
+
* deliberately treated exactly as harshly as `false`.
|
|
55
|
+
*/
|
|
56
|
+
viewerIsAuthorized: boolean | "unknown";
|
|
57
|
+
}
|
|
58
|
+
interface EmptinessClassification {
|
|
59
|
+
verdict: EmptinessVerdict;
|
|
60
|
+
/** Human-readable justification, safe to embed in a failure message. */
|
|
61
|
+
reason: string;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Classify what an (apparently) empty surface actually proves.
|
|
65
|
+
*
|
|
66
|
+
* Pure function: same input, same verdict, no clock, no network. Every I/O
|
|
67
|
+
* concern lives in Layer 2 (`assertNotVacuouslyEmpty`).
|
|
68
|
+
*/
|
|
69
|
+
declare function classifyEmptiness(input: ClassifyEmptinessInput): EmptinessClassification;
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Structural (duck-typed) subset of the Playwright surface this entry needs.
|
|
73
|
+
*
|
|
74
|
+
* WHY NOT `import type { Page } from "@playwright/test"`
|
|
75
|
+
* -----------------------------------------------------
|
|
76
|
+
* `@devfellowship/components` is a runtime dependency of every DFL app.
|
|
77
|
+
* Adding Playwright — even as a type-only import — would put a ~100MB test
|
|
78
|
+
* runner into the dependency graph of production bundles and would make this
|
|
79
|
+
* entry impossible to unit-test without a browser. So we declare the three
|
|
80
|
+
* methods we actually call and nothing else.
|
|
81
|
+
*
|
|
82
|
+
* A real Playwright `Page` / `Locator` satisfies these interfaces
|
|
83
|
+
* structurally: no adapter, no cast, no `@playwright/test` in package.json.
|
|
84
|
+
* A plain object literal satisfies them too, which is how the tests work.
|
|
85
|
+
*/
|
|
86
|
+
/** The slice of a Playwright `Locator` used here. */
|
|
87
|
+
interface LocatorLike {
|
|
88
|
+
/** Number of DOM elements the selector currently resolves to. */
|
|
89
|
+
count(): Promise<number>;
|
|
90
|
+
/** Rendered (visible) text of the resolved element. */
|
|
91
|
+
innerText(): Promise<string>;
|
|
92
|
+
}
|
|
93
|
+
/** The slice of a Playwright `Page` used here. */
|
|
94
|
+
interface PageLike {
|
|
95
|
+
locator(selector: string): LocatorLike;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Minimal shape of a `supabase-js` client — only `.rpc()`, only the fields
|
|
99
|
+
* `iamMemberProbe` reads. Keeps `@supabase/supabase-js` out of this entry's
|
|
100
|
+
* imports entirely (it is an OPTIONAL peer dependency of the package).
|
|
101
|
+
*/
|
|
102
|
+
interface SupabaseRpcClientLike {
|
|
103
|
+
rpc(fn: string, args?: Record<string, unknown>): PromiseLike<{
|
|
104
|
+
data: unknown;
|
|
105
|
+
error: unknown;
|
|
106
|
+
}>;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Thrown when a surface's emptiness is indistinguishable from lack of access.
|
|
111
|
+
*
|
|
112
|
+
* This is NOT "the page is broken" — it is "this check cannot tell you whether
|
|
113
|
+
* the page is broken", which is the more dangerous state, because the default
|
|
114
|
+
* assertion would have passed.
|
|
115
|
+
*/
|
|
116
|
+
declare class VacuousVerificationError extends Error {
|
|
117
|
+
readonly verdict: EmptinessVerdict;
|
|
118
|
+
readonly surface: string;
|
|
119
|
+
readonly rowCount: number;
|
|
120
|
+
readonly deniedSignalPresent: boolean;
|
|
121
|
+
readonly viewerIsAuthorized: boolean | "unknown";
|
|
122
|
+
readonly reason: string;
|
|
123
|
+
constructor(details: {
|
|
124
|
+
surface: string;
|
|
125
|
+
rowCount: number;
|
|
126
|
+
deniedSignalPresent: boolean;
|
|
127
|
+
viewerIsAuthorized: boolean | "unknown";
|
|
128
|
+
reason: string;
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
interface AssertNotVacuouslyEmptyOptions {
|
|
132
|
+
/**
|
|
133
|
+
* Regex matched against the surface's rendered text, LINE BY LINE (each line
|
|
134
|
+
* trimmed), so anchored patterns work — e.g. `/^(\d+) registros?$/`.
|
|
135
|
+
* Capture group 1 must be the count.
|
|
136
|
+
*/
|
|
137
|
+
countText?: RegExp;
|
|
138
|
+
/** CSS/testid selector whose element holds the count text. Default `body`. */
|
|
139
|
+
countTextSelector?: string;
|
|
140
|
+
/** Alternative to `countText`: count the elements matching this selector. */
|
|
141
|
+
rowSelector?: string;
|
|
142
|
+
/**
|
|
143
|
+
* Selector for the element/testid the app renders on permission denial.
|
|
144
|
+
* If the app has no such affordance, omit it — but then a denied surface is
|
|
145
|
+
* unverifiable by construction, which is exactly what `vacuous` reports.
|
|
146
|
+
*/
|
|
147
|
+
deniedSelector?: string;
|
|
148
|
+
/**
|
|
149
|
+
* INDEPENDENT authorization signal. A thunk (e.g. `() => iamMemberProbe(sb)`)
|
|
150
|
+
* is awaited; if it throws, the result is `'unknown'`.
|
|
151
|
+
* Omitted defaults to `'unknown'` — treated as harshly as `false`.
|
|
152
|
+
*/
|
|
153
|
+
viewerIsAuthorized?: boolean | "unknown" | (() => Promise<boolean | "unknown">);
|
|
154
|
+
/** Label used in the failure message. Default `unnamed surface`. */
|
|
155
|
+
surface?: string;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Layer 2 — the e2e adapter. Reads the three facts off a Playwright-ish page,
|
|
159
|
+
* hands them to the pure {@link classifyEmptiness}, and refuses to return a
|
|
160
|
+
* green it cannot justify.
|
|
161
|
+
*
|
|
162
|
+
* Returns the {@link EmptinessVerdict} for `populated`, `denied` and
|
|
163
|
+
* `genuinely-empty`. Throws {@link VacuousVerificationError} for `vacuous`.
|
|
164
|
+
*
|
|
165
|
+
* @example
|
|
166
|
+
* // Fails today against the level-0 smoke identity — by design.
|
|
167
|
+
* await assertNotVacuouslyEmpty(page, {
|
|
168
|
+
* surface: "spec-builder /history",
|
|
169
|
+
* countText: /^(\d+) registros?$/,
|
|
170
|
+
* deniedSelector: '[data-testid="access-denied"]',
|
|
171
|
+
* viewerIsAuthorized: () => iamMemberProbe(supabase),
|
|
172
|
+
* });
|
|
173
|
+
*/
|
|
174
|
+
declare function assertNotVacuouslyEmpty(page: PageLike, opts?: AssertNotVacuouslyEmptyOptions): Promise<EmptinessVerdict>;
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Default IAM level that counts as "member enough to see gated rows".
|
|
178
|
+
* Mirrors the threshold `iam.is_member()` enforces server-side.
|
|
179
|
+
*/
|
|
180
|
+
declare const IAM_MEMBER_MIN_LEVEL = 50;
|
|
181
|
+
/**
|
|
182
|
+
* Independent authorization probe: ask the database who the caller is,
|
|
183
|
+
* instead of inferring it from the very response whose emptiness is in doubt.
|
|
184
|
+
*
|
|
185
|
+
* Calls the existing `get_my_iam_role()` RPC (already granted to
|
|
186
|
+
* `authenticated`; returns rows of `{ role_id, level }`) and resolves whether
|
|
187
|
+
* any granted role clears `minLevel` (default {@link IAM_MEMBER_MIN_LEVEL}).
|
|
188
|
+
*
|
|
189
|
+
* FAILS TO `'unknown'`, NEVER TO `true`. An RPC error, a null payload, a
|
|
190
|
+
* non-array payload, or rows carrying no readable `level` all mean "we did not
|
|
191
|
+
* learn anything" — and `'unknown'` is treated by `classifyEmptiness` exactly
|
|
192
|
+
* as harshly as `false`. A probe that guessed optimistically would reintroduce
|
|
193
|
+
* the very false-green this module exists to prevent.
|
|
194
|
+
*
|
|
195
|
+
* Note that an empty row set is `'unknown'` rather than `false`: a level-0
|
|
196
|
+
* identity may legitimately produce zero rows, but so does a broken grant, and
|
|
197
|
+
* the two are not distinguishable from here. Both block a green.
|
|
198
|
+
*
|
|
199
|
+
* @example
|
|
200
|
+
* await assertNotVacuouslyEmpty(page, {
|
|
201
|
+
* countText: /^(\d+) registros?$/,
|
|
202
|
+
* viewerIsAuthorized: () => iamMemberProbe(supabase),
|
|
203
|
+
* });
|
|
204
|
+
*/
|
|
205
|
+
declare function iamMemberProbe(supabaseClient: SupabaseRpcClientLike, opts?: {
|
|
206
|
+
minLevel?: number;
|
|
207
|
+
rpcName?: string;
|
|
208
|
+
}): Promise<boolean | "unknown">;
|
|
209
|
+
|
|
210
|
+
export { type AssertNotVacuouslyEmptyOptions, type ClassifyEmptinessInput, type EmptinessClassification, type EmptinessVerdict, IAM_MEMBER_MIN_LEVEL, type LocatorLike, type PageLike, type SupabaseRpcClientLike, VacuousVerificationError, assertNotVacuouslyEmpty, classifyEmptiness, iamMemberProbe };
|
package/dist/testing.js
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
// src/testing/emptiness.ts
|
|
2
|
+
function classifyEmptiness(input) {
|
|
3
|
+
const { rowCount, deniedSignalPresent, viewerIsAuthorized } = input;
|
|
4
|
+
if (rowCount > 0) {
|
|
5
|
+
return {
|
|
6
|
+
verdict: "populated",
|
|
7
|
+
reason: `Surface rendered ${rowCount} row(s); the result is non-empty, so there is no emptiness to disambiguate.`
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
if (deniedSignalPresent) {
|
|
11
|
+
return {
|
|
12
|
+
verdict: "denied",
|
|
13
|
+
reason: "Surface rendered 0 rows AND an explicit permission-denied signal. The app failed loud, so the emptiness is attributable to a stated authorization failure rather than to a silent RLS filter \u2014 an assertion on this state is meaningful."
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
if (viewerIsAuthorized === true) {
|
|
17
|
+
return {
|
|
18
|
+
verdict: "genuinely-empty",
|
|
19
|
+
reason: "Surface rendered 0 rows with no permission-denied signal, and an independent authorization probe confirmed the viewer COULD have seen rows. The emptiness therefore reflects the data, not the viewer's access."
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
const authClause = viewerIsAuthorized === false ? "an independent probe reported the viewer is NOT authorized" : "the viewer's authorization could not be confirmed (unknown)";
|
|
23
|
+
return {
|
|
24
|
+
verdict: "vacuous",
|
|
25
|
+
reason: `Surface rendered 0 rows, the page rendered NO permission-denied signal, and ${authClause}. Supabase RLS hides unauthorized rows as HTTP 200 with an empty body, which is byte-identical to a genuinely empty result \u2014 so this emptiness is indistinguishable from lack of access and the check proves nothing. Fix by supplying an independent authorization signal (e.g. \`iamMemberProbe\`), by using an identity that can see rows, or by making the surface render an explicit permission-denied state.`
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// src/testing/assert-not-vacuously-empty.ts
|
|
30
|
+
var VacuousVerificationError = class extends Error {
|
|
31
|
+
constructor(details) {
|
|
32
|
+
super(
|
|
33
|
+
[
|
|
34
|
+
`Vacuous verification on "${details.surface}": this check proves nothing.`,
|
|
35
|
+
"",
|
|
36
|
+
` \u2022 the surface returned ${details.rowCount} rows (nothing to assert against)`,
|
|
37
|
+
" \u2022 the page rendered NO permission-denied signal, so the app did not fail loud",
|
|
38
|
+
details.viewerIsAuthorized === false ? " \u2022 the viewer is known NOT to be authorized" : " \u2022 the viewer's authorization could NOT be confirmed",
|
|
39
|
+
"",
|
|
40
|
+
details.reason
|
|
41
|
+
].join("\n")
|
|
42
|
+
);
|
|
43
|
+
this.verdict = "vacuous";
|
|
44
|
+
this.name = "VacuousVerificationError";
|
|
45
|
+
this.surface = details.surface;
|
|
46
|
+
this.rowCount = details.rowCount;
|
|
47
|
+
this.deniedSignalPresent = details.deniedSignalPresent;
|
|
48
|
+
this.viewerIsAuthorized = details.viewerIsAuthorized;
|
|
49
|
+
this.reason = details.reason;
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
async function resolveAuthorization(input) {
|
|
53
|
+
if (input === void 0) return "unknown";
|
|
54
|
+
if (typeof input !== "function") return input;
|
|
55
|
+
try {
|
|
56
|
+
const result = await input();
|
|
57
|
+
return result === true || result === false ? result : "unknown";
|
|
58
|
+
} catch {
|
|
59
|
+
return "unknown";
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
async function resolveRowCount(page, opts) {
|
|
63
|
+
if (opts.countText) {
|
|
64
|
+
const text = await page.locator(opts.countTextSelector ?? "body").innerText();
|
|
65
|
+
for (const rawLine of text.split("\n")) {
|
|
66
|
+
const match = opts.countText.exec(rawLine.trim());
|
|
67
|
+
if (match) {
|
|
68
|
+
const parsed = Number(match[1]);
|
|
69
|
+
if (Number.isFinite(parsed)) return parsed;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (!opts.rowSelector) {
|
|
73
|
+
throw new Error(
|
|
74
|
+
`assertNotVacuouslyEmpty: countText ${String(opts.countText)} matched no line of "${opts.countTextSelector ?? "body"}". The row count is unreadable, so no verdict is possible \u2014 fix the pattern or pass rowSelector.`
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
if (opts.rowSelector) return page.locator(opts.rowSelector).count();
|
|
79
|
+
throw new Error(
|
|
80
|
+
"assertNotVacuouslyEmpty: pass countText and/or rowSelector \u2014 without one there is no row count to classify."
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
async function assertNotVacuouslyEmpty(page, opts = {}) {
|
|
84
|
+
const surface = opts.surface ?? "unnamed surface";
|
|
85
|
+
const rowCount = await resolveRowCount(page, opts);
|
|
86
|
+
const deniedSignalPresent = opts.deniedSelector ? await page.locator(opts.deniedSelector).count() > 0 : false;
|
|
87
|
+
const viewerIsAuthorized = await resolveAuthorization(opts.viewerIsAuthorized);
|
|
88
|
+
const { verdict, reason } = classifyEmptiness({
|
|
89
|
+
rowCount,
|
|
90
|
+
deniedSignalPresent,
|
|
91
|
+
viewerIsAuthorized
|
|
92
|
+
});
|
|
93
|
+
if (verdict === "vacuous") {
|
|
94
|
+
throw new VacuousVerificationError({
|
|
95
|
+
surface,
|
|
96
|
+
rowCount,
|
|
97
|
+
deniedSignalPresent,
|
|
98
|
+
viewerIsAuthorized,
|
|
99
|
+
reason
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
return verdict;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// src/testing/iam-member-probe.ts
|
|
106
|
+
var IAM_MEMBER_MIN_LEVEL = 50;
|
|
107
|
+
function readLevel(row) {
|
|
108
|
+
if (typeof row !== "object" || row === null) return null;
|
|
109
|
+
const level = row.level;
|
|
110
|
+
if (typeof level === "number" && Number.isFinite(level)) return level;
|
|
111
|
+
if (typeof level === "string" && level.trim() !== "") {
|
|
112
|
+
const parsed = Number(level);
|
|
113
|
+
if (Number.isFinite(parsed)) return parsed;
|
|
114
|
+
}
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
117
|
+
async function iamMemberProbe(supabaseClient, opts = {}) {
|
|
118
|
+
const minLevel = opts.minLevel ?? IAM_MEMBER_MIN_LEVEL;
|
|
119
|
+
const rpcName = opts.rpcName ?? "get_my_iam_role";
|
|
120
|
+
let data;
|
|
121
|
+
let error;
|
|
122
|
+
try {
|
|
123
|
+
({ data, error } = await supabaseClient.rpc(rpcName));
|
|
124
|
+
} catch {
|
|
125
|
+
return "unknown";
|
|
126
|
+
}
|
|
127
|
+
if (error) return "unknown";
|
|
128
|
+
const rows = Array.isArray(data) ? data : data == null ? [] : [data];
|
|
129
|
+
if (rows.length === 0) return "unknown";
|
|
130
|
+
const levels = rows.map(readLevel).filter((l) => l !== null);
|
|
131
|
+
if (levels.length === 0) return "unknown";
|
|
132
|
+
return levels.some((level) => level >= minLevel);
|
|
133
|
+
}
|
|
134
|
+
export {
|
|
135
|
+
IAM_MEMBER_MIN_LEVEL,
|
|
136
|
+
VacuousVerificationError,
|
|
137
|
+
assertNotVacuouslyEmpty,
|
|
138
|
+
classifyEmptiness,
|
|
139
|
+
iamMemberProbe
|
|
140
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@devfellowship/components",
|
|
3
|
-
"version": "3.0
|
|
3
|
+
"version": "3.1.0",
|
|
4
4
|
"description": "DFL Design System — UI components, hooks, utils and providers",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"sideEffects": [
|
|
@@ -27,6 +27,11 @@
|
|
|
27
27
|
"import": "./dist/providers.js",
|
|
28
28
|
"require": "./dist/providers.cjs"
|
|
29
29
|
},
|
|
30
|
+
"./testing": {
|
|
31
|
+
"types": "./dist/testing.d.ts",
|
|
32
|
+
"import": "./dist/testing.js",
|
|
33
|
+
"require": "./dist/testing.cjs"
|
|
34
|
+
},
|
|
30
35
|
"./styles": "./dist/styles/theme.css",
|
|
31
36
|
"./shadcn": "./dist/styles/shadcn.css",
|
|
32
37
|
"./tokens": "./dist/styles/tokens.css",
|
|
@@ -52,7 +57,8 @@
|
|
|
52
57
|
"prepublishOnly": "npm run build",
|
|
53
58
|
"storybook": "storybook dev -p 6006",
|
|
54
59
|
"build-storybook": "storybook build",
|
|
55
|
-
"lint:no-docs": "node scripts/check-no-storybook-docs.mjs"
|
|
60
|
+
"lint:no-docs": "node scripts/check-no-storybook-docs.mjs",
|
|
61
|
+
"check:cli-offline": "node scripts/check-cli-bundle-offline.mjs"
|
|
56
62
|
},
|
|
57
63
|
"peerDependencies": {
|
|
58
64
|
"@supabase/supabase-js": ">=2.0.0",
|