@ultimat3/testing 29.4.0 → 30.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +56 -63
- package/README.md +281 -6
- package/package.json +27 -16
- package/src/ax-node.ts +96 -0
- package/src/cassette-errors.ts +91 -0
- package/src/cassette-format.ts +175 -0
- package/src/cassette-record.ts +120 -0
- package/src/cassette-secret-scan.ts +77 -0
- package/src/cassette.ts +166 -0
- package/src/cdp-browser.ts +5 -3
- package/src/cdp-e2e-page.ts +91 -3
- package/src/cdp-e2e-session.ts +18 -1
- package/src/define-exercise.ts +92 -0
- package/src/drill-budgets.ts +54 -0
- package/src/drill-errors.ts +45 -0
- package/src/drill-fleet.ts +251 -0
- package/src/drill-jobs.ts +79 -0
- package/src/drill-load.ts +166 -0
- package/src/drill-probe.ts +68 -0
- package/src/drill-run.ts +144 -0
- package/src/drill-services.ts +70 -0
- package/src/drill-sync.ts +108 -0
- package/src/drill-verdict.ts +256 -0
- package/src/drill-writes.ts +123 -0
- package/src/drills.ts +109 -0
- package/src/e2e-a11y-run.ts +189 -0
- package/src/e2e-a11y-tabs.ts +113 -0
- package/src/e2e-a11y.ts +153 -0
- package/src/e2e-driver.ts +46 -10
- package/src/e2e-page.ts +28 -3
- package/src/e2e-run.ts +1 -0
- package/src/e2e-spawn.ts +3 -2
- package/src/errors.ts +43 -0
- package/src/exercise-actor.ts +124 -0
- package/src/exercise-crash.ts +46 -0
- package/src/exercise-input.ts +182 -0
- package/src/exercise-override-input.ts +34 -0
- package/src/exercise-report.ts +282 -0
- package/src/exercise-request.ts +38 -0
- package/src/exercise-sweeps.ts +100 -0
- package/src/exercise.ts +451 -0
- package/src/factories.ts +10 -7
- package/src/factory-registry.ts +49 -15
- package/src/fake-identity-provider.ts +191 -0
- package/src/fault-proxy.ts +342 -0
- package/src/file-boundary.ts +4 -1
- package/src/fixture-jobs.ts +3 -1
- package/src/fixture-statements.ts +4 -1
- package/src/free-port.ts +59 -0
- package/src/index.ts +148 -3
- package/src/island-dom.ts +5 -0
- package/src/isolated-plugins.ts +1 -1
- package/src/job-crash-proof.ts +249 -0
- package/src/job-sim-driver.ts +164 -0
- package/src/job-sim-effects.ts +142 -0
- package/src/job-sim-errors.ts +72 -0
- package/src/job-sim-schedule.ts +154 -0
- package/src/job-sim-world.ts +198 -0
- package/src/job-simulate.ts +179 -0
- package/src/mock-route.ts +81 -0
- package/src/module-mock-boundary.ts +85 -0
- package/src/module-mock-error.ts +28 -0
- package/src/plan-oracle.ts +164 -0
- package/src/plan-sweep-fix.ts +123 -0
- package/src/plan-sweep-report.ts +84 -0
- package/src/plan-sweep.ts +148 -0
- package/src/preload.ts +24 -4
- package/src/registry-snapshot.ts +1 -1
- package/src/repro-stand-in.ts +67 -0
- package/src/repro.ts +158 -0
- package/src/request-log.ts +64 -0
- package/src/reusable-database.ts +1 -1
- package/src/scale-sweep-graph.ts +82 -0
- package/src/scale-sweep-run.ts +134 -0
- package/src/scale-sweep.ts +74 -0
- package/src/sealed-network.ts +63 -46
- package/src/seed-graph-error.ts +50 -0
- package/src/seed-graph-order.ts +98 -0
- package/src/seed-graph-pass.ts +85 -0
- package/src/seed-graph-plan.ts +81 -0
- package/src/seed-graph-row.ts +134 -0
- package/src/seed-graph-shape.ts +104 -0
- package/src/seed-graph-values.ts +135 -0
- package/src/seed-graph.ts +250 -0
- package/src/seed-writer.ts +180 -0
- package/src/subject-proof-error.ts +23 -0
- package/src/subject-proof.ts +180 -0
- package/src/template-db-roles.ts +51 -0
- package/src/template-db.ts +8 -2
- package/src/tenant-canary.ts +120 -0
- package/src/tenant-proof-report.ts +62 -0
- package/src/tenant-proof.ts +177 -0
- package/src/test-types.ts +12 -3
package/src/exercise.ts
ADDED
|
@@ -0,0 +1,451 @@
|
|
|
1
|
+
// `runExercisePass`: every registered action, query and job, run once with no test written. Each
|
|
2
|
+
// gets its own pair of committed tenants from the pass's one seed graph, an actor its policy allows,
|
|
3
|
+
// and the input 39's minimal draw gives with seeded ids bound in; the verdict is 39's loop.
|
|
4
|
+
//
|
|
5
|
+
// Committed, never rolled back. Around that call the tenant proof runs: the pair's other tenant
|
|
6
|
+
// calls with the first one's ids before and after it (`tenant-proof.ts`, plan 101, 40). A reached
|
|
7
|
+
// job is then killed at every step boundary (`exercise-crash.ts`).
|
|
8
|
+
|
|
9
|
+
import type { Actor, ExerciseAttempt, ExerciseVerdict } from '@ultimat3/core';
|
|
10
|
+
import { isUltimateError, renderThrowable, runExercise } from '@ultimat3/core';
|
|
11
|
+
import type { AnyJobHandle, JobExecution, JobRecord } from '@ultimat3/jobs';
|
|
12
|
+
import { registeredJobs } from '@ultimat3/jobs';
|
|
13
|
+
import type { Policy } from '@ultimat3/policy';
|
|
14
|
+
import type { StandardSchemaV1 } from '@ultimat3/schema';
|
|
15
|
+
import type { ExerciseOverride, ExerciseOverrides } from './define-exercise';
|
|
16
|
+
import { assertOverridesKnown, unknownDeclaration } from './define-exercise';
|
|
17
|
+
import type { RoleGrants } from './exercise-actor';
|
|
18
|
+
import { chooseActor } from './exercise-actor';
|
|
19
|
+
import { proveCrash } from './exercise-crash';
|
|
20
|
+
import type { IdLookup, Sampler } from './exercise-input';
|
|
21
|
+
import {
|
|
22
|
+
bindIds,
|
|
23
|
+
drawInput,
|
|
24
|
+
idStem,
|
|
25
|
+
loadSampler,
|
|
26
|
+
namesActorEntity,
|
|
27
|
+
seededIds,
|
|
28
|
+
} from './exercise-input';
|
|
29
|
+
import type { OverrideInput } from './exercise-override-input';
|
|
30
|
+
import { overrideInput } from './exercise-override-input';
|
|
31
|
+
import type {
|
|
32
|
+
EffectPayload,
|
|
33
|
+
ExerciseEffect,
|
|
34
|
+
ExerciseRecord,
|
|
35
|
+
ExerciseRoom,
|
|
36
|
+
Observed,
|
|
37
|
+
} from './exercise-report';
|
|
38
|
+
import { openExerciseRoom, overrideLine } from './exercise-report';
|
|
39
|
+
import { withExerciseRequest } from './exercise-request';
|
|
40
|
+
import { actorRowsOf, attachSweeps, seated, sizedCall } from './exercise-sweeps';
|
|
41
|
+
import type { RunJobs } from './fixture-jobs';
|
|
42
|
+
import { testJobs } from './fixture-jobs';
|
|
43
|
+
import { seedPass, tenantPair } from './seed-graph-pass';
|
|
44
|
+
import type { SeedGraph, SeedWriter } from './seed-graph-plan';
|
|
45
|
+
import type { SeedEntity } from './seed-graph-shape';
|
|
46
|
+
import { tenantTaint } from './tenant-canary';
|
|
47
|
+
import type { LegAnswer } from './tenant-proof';
|
|
48
|
+
import { proveTenant, publicReason, withoutCrossTenant } from './tenant-proof';
|
|
49
|
+
|
|
50
|
+
export interface ExercisePassOptions {
|
|
51
|
+
readonly entities: readonly SeedEntity[];
|
|
52
|
+
readonly writer: SeedWriter;
|
|
53
|
+
/**
|
|
54
|
+
* Seeds the pass's one graph: the shared rows and every primitive's tenants. Omitted, each pass
|
|
55
|
+
* draws a fresh one: the rows are COMMITTED, so a second pass on one seed against one database
|
|
56
|
+
* collides on its keys. Given, the pass replays byte for byte — on a database that has not seen it.
|
|
57
|
+
*/
|
|
58
|
+
readonly seed?: number;
|
|
59
|
+
readonly overrides?: ExerciseOverrides;
|
|
60
|
+
/** One primitive by name; unknown is `X_DECLARATION_UNKNOWN`. */
|
|
61
|
+
readonly only?: string;
|
|
62
|
+
readonly roles: RoleGrants;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** One primitive, as the pass drives it: the schema to draw from and the one call to make. */
|
|
66
|
+
interface Subject {
|
|
67
|
+
readonly kind: ExerciseRecord['kind'];
|
|
68
|
+
readonly name: string;
|
|
69
|
+
readonly input: StandardSchemaV1;
|
|
70
|
+
readonly policy: Policy | undefined;
|
|
71
|
+
/** Every permission the policy references — what `x policy explain` matches grants against. */
|
|
72
|
+
readonly permissions: readonly string[];
|
|
73
|
+
/** A job's handle: what the crash proof kills at every step boundary once the control reached it. */
|
|
74
|
+
readonly handle?: AnyJobHandle;
|
|
75
|
+
/** Resolves to the failure a worker recorded on the row (a job's), or what the call answered. */
|
|
76
|
+
call(input: unknown, actor: Actor | null, tenant: string, jobs: RunJobs): Promise<Answer>;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** `failure` set: the worker recorded one (a job's). Absent: answered, with `output`. */
|
|
80
|
+
interface Answer {
|
|
81
|
+
readonly failure?: unknown;
|
|
82
|
+
readonly output?: unknown;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const ANSWERED: ReadonlySet<string> = new Set(['completed', 'suspended']);
|
|
86
|
+
|
|
87
|
+
const statusOfJobFailure = (run: JobExecution | undefined): string =>
|
|
88
|
+
run === undefined ? 'the job never ran' : (run.error ?? `the run ended ${run.outcome}`);
|
|
89
|
+
|
|
90
|
+
async function subjectsOf(): Promise<readonly Subject[]> {
|
|
91
|
+
// On demand, as every fixture here loads them: the barrel stays off `action` and `query`.
|
|
92
|
+
const { invoke, listActions } = await import('@ultimat3/action');
|
|
93
|
+
const { listQueries, runQuery } = await import('@ultimat3/query');
|
|
94
|
+
const job = (handle: AnyJobHandle): Subject => ({
|
|
95
|
+
kind: 'job',
|
|
96
|
+
name: handle.name,
|
|
97
|
+
input: handle.input,
|
|
98
|
+
policy: undefined,
|
|
99
|
+
permissions: [],
|
|
100
|
+
handle,
|
|
101
|
+
call: async (input, actor, tenant, jobs) => {
|
|
102
|
+
const queued = await jobs.enqueue(handle, input, { tenantId: tenant });
|
|
103
|
+
const trace = await jobs.drain(actor === null ? {} : { actor });
|
|
104
|
+
const own = trace.executions.find((run) => run.jobId === queued.id);
|
|
105
|
+
// The worker's failure is a string on the row, never a throw: it is the attempt's error.
|
|
106
|
+
// Suspended is a body that ran and parked (`step.sleep`, `waitForEvent`): reached.
|
|
107
|
+
return own !== undefined && ANSWERED.has(own.outcome)
|
|
108
|
+
? { output: own.result }
|
|
109
|
+
: { failure: statusOfJobFailure(own) };
|
|
110
|
+
},
|
|
111
|
+
});
|
|
112
|
+
return [
|
|
113
|
+
...listActions().map(
|
|
114
|
+
(target): Subject => ({
|
|
115
|
+
kind: 'action',
|
|
116
|
+
name: target.name,
|
|
117
|
+
input: target.input,
|
|
118
|
+
policy: target.policy,
|
|
119
|
+
permissions: target.describe().permissions,
|
|
120
|
+
call: async (input, actor) => ({
|
|
121
|
+
output: await withExerciseRequest(target.name, actor, () =>
|
|
122
|
+
invoke(target, input, { surface: 'http' }),
|
|
123
|
+
),
|
|
124
|
+
}),
|
|
125
|
+
}),
|
|
126
|
+
),
|
|
127
|
+
...listQueries().map(
|
|
128
|
+
(target): Subject => ({
|
|
129
|
+
kind: 'query',
|
|
130
|
+
name: target.name,
|
|
131
|
+
input: target.input,
|
|
132
|
+
policy: target.policy,
|
|
133
|
+
permissions: target.describe().permissions,
|
|
134
|
+
call: async (input, actor) => ({
|
|
135
|
+
output: await withExerciseRequest(target.name, actor, () =>
|
|
136
|
+
runQuery(target, input, { surface: 'http' }),
|
|
137
|
+
),
|
|
138
|
+
}),
|
|
139
|
+
}),
|
|
140
|
+
),
|
|
141
|
+
...registeredJobs().map(job),
|
|
142
|
+
];
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const isOff = (overrides: ExerciseOverrides | undefined, name: string): boolean => {
|
|
146
|
+
const override = overrides?.entries.get(name);
|
|
147
|
+
return override !== undefined && 'off' in override;
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
const codeOf = (error: unknown): string | undefined => {
|
|
151
|
+
if (isUltimateError(error)) return error.code;
|
|
152
|
+
return typeof error === 'string' ? /^(X_[A-Z0-9_]+)/.exec(error)?.[1] : undefined;
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
const reasonOf = (error: unknown): string =>
|
|
156
|
+
isUltimateError(error) ? error.cause : typeof error === 'string' ? error : renderThrowable(error);
|
|
157
|
+
|
|
158
|
+
/** Jobs queued or run by the primitive, the primitive's own job excepted. */
|
|
159
|
+
async function enqueued(subject: Subject, jobs: RunJobs, runs: readonly JobExecution[]) {
|
|
160
|
+
const effects: ExerciseEffect[] = [];
|
|
161
|
+
for (const handle of registeredJobs()) {
|
|
162
|
+
if (subject.kind === 'job' && handle.name === subject.name) continue;
|
|
163
|
+
const ran = new Set(runs.filter((run) => run.job === handle.name).map((r) => r.jobId)).size;
|
|
164
|
+
const count = ran + (await jobs.depth(handle));
|
|
165
|
+
if (count > 0) effects.push({ kind: 'enqueue', job: handle.name, count });
|
|
166
|
+
}
|
|
167
|
+
return effects;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Every job a B leg's queue holds or ran, with its input — the subject's own job excepted, whose
|
|
172
|
+
* input is what B was handed. Read off the leg's own queue, page by page.
|
|
173
|
+
*/
|
|
174
|
+
async function queuedPayloads(subject: Subject): Promise<readonly EffectPayload[]> {
|
|
175
|
+
const jobs = await import('@ultimat3/jobs');
|
|
176
|
+
const introspect = jobs.jobDriver()?.introspect;
|
|
177
|
+
if (introspect === undefined) return [];
|
|
178
|
+
const records: JobRecord[] = [];
|
|
179
|
+
for (let after: string | undefined; ; ) {
|
|
180
|
+
const page = await introspect.list({
|
|
181
|
+
limit: jobs.MAX_JOB_PAGE,
|
|
182
|
+
...(after === undefined ? {} : { after }),
|
|
183
|
+
});
|
|
184
|
+
records.push(...page);
|
|
185
|
+
const last = page[page.length - 1];
|
|
186
|
+
if (last === undefined || page.length < jobs.MAX_JOB_PAGE) break;
|
|
187
|
+
after = jobs.jobCursor(last);
|
|
188
|
+
}
|
|
189
|
+
return records
|
|
190
|
+
.filter((record) => !(subject.kind === 'job' && record.name === subject.name))
|
|
191
|
+
.map((record, i) => ({
|
|
192
|
+
path: `effect.enqueue[${i}]`,
|
|
193
|
+
value: { job: record.name, input: record.input },
|
|
194
|
+
}));
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
interface Run {
|
|
198
|
+
readonly verdict: ExerciseVerdict;
|
|
199
|
+
readonly answered: boolean;
|
|
200
|
+
readonly lastError: unknown;
|
|
201
|
+
readonly lastInput: unknown;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function judged(subject: Subject, run: Run, observed: Observed) {
|
|
205
|
+
const idFields = (input: unknown): readonly string[] =>
|
|
206
|
+
typeof input === 'object' && input !== null
|
|
207
|
+
? Object.keys(input).filter((key) => idStem(key) !== undefined)
|
|
208
|
+
: [];
|
|
209
|
+
const unreached = (code: string | undefined, reason: string) => ({
|
|
210
|
+
outcome: 'unreached' as const,
|
|
211
|
+
...(code === undefined ? {} : { code }),
|
|
212
|
+
reason,
|
|
213
|
+
override: overrideLine(subject.name, idFields(run.lastInput)),
|
|
214
|
+
});
|
|
215
|
+
if (observed.refusedHosts.length > 0) {
|
|
216
|
+
// A call out is a refusal whatever the handler made of it: sealed, so never sent.
|
|
217
|
+
const hosts = [...new Set(observed.refusedHosts)].join(', ');
|
|
218
|
+
return unreached('X_TEST_NETWORK_SEALED', `called out to ${hosts}: sealed, never sent`);
|
|
219
|
+
}
|
|
220
|
+
if (run.verdict.kind === 'pass' && run.answered) return { outcome: 'reached' as const };
|
|
221
|
+
if (run.verdict.kind === 'drift') {
|
|
222
|
+
return unreached('X_CONTRACT_DRIFT', 'its input schema refused every generated value');
|
|
223
|
+
}
|
|
224
|
+
if (run.verdict.kind === 'defect') {
|
|
225
|
+
return unreached(codeOf(run.verdict.error) ?? 'X_CONTRACT_DEFECT', run.verdict.defect);
|
|
226
|
+
}
|
|
227
|
+
return unreached(codeOf(run.lastError), reasonOf(run.lastError));
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** Everything one primitive's calls share: who acts in each tenant, and how ids are bound. */
|
|
231
|
+
interface Prepared {
|
|
232
|
+
readonly subject: Subject;
|
|
233
|
+
readonly options: ExercisePassOptions;
|
|
234
|
+
readonly tenant: string;
|
|
235
|
+
readonly actor: Actor | null;
|
|
236
|
+
readonly runs: number;
|
|
237
|
+
readonly declaredRuns: number | undefined;
|
|
238
|
+
readonly lookups: readonly [IdLookup, IdLookup];
|
|
239
|
+
readonly bindInput: OverrideInput | undefined;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** A's positive control: 39's loop, as the pass has always run it. */
|
|
243
|
+
async function control(
|
|
244
|
+
run: Prepared,
|
|
245
|
+
room: ExerciseRoom,
|
|
246
|
+
sampler: Sampler,
|
|
247
|
+
): Promise<{ readonly record: ExerciseRecord; readonly output: unknown; readonly input: unknown }> {
|
|
248
|
+
const { subject, actor, tenant, declaredRuns, lookups, bindInput } = run;
|
|
249
|
+
const { kind, name } = subject;
|
|
250
|
+
let binds = 0;
|
|
251
|
+
await using jobs = await testJobs();
|
|
252
|
+
let answered = false;
|
|
253
|
+
let lastError: unknown;
|
|
254
|
+
let lastInput: unknown;
|
|
255
|
+
let output: unknown;
|
|
256
|
+
let executions: readonly JobExecution[] = [];
|
|
257
|
+
room.begin(name);
|
|
258
|
+
const started = performance.now();
|
|
259
|
+
const verdict = await runExercise({
|
|
260
|
+
name,
|
|
261
|
+
runs: run.runs,
|
|
262
|
+
stoppedLabel: 'stopped before the call answered',
|
|
263
|
+
// The minimal draw on every run the pass adds; 39's seeded draw only on runs an override asks.
|
|
264
|
+
draw: (at, n) => drawInput(sampler, subject.input, at, declaredRuns === undefined ? 0 : n),
|
|
265
|
+
bind: async (drawn) => {
|
|
266
|
+
const bound = bindIds(drawn, lookups[binds++ === 0 ? 0 : 1]);
|
|
267
|
+
lastInput = bindInput === undefined ? bound : await bindInput(bound, actor);
|
|
268
|
+
return lastInput;
|
|
269
|
+
},
|
|
270
|
+
accepts: async (input) =>
|
|
271
|
+
(await subject.input['~standard'].validate(input)).issues === undefined,
|
|
272
|
+
attempt: async (input): Promise<ExerciseAttempt> => {
|
|
273
|
+
try {
|
|
274
|
+
const answer = await subject.call(input, actor, tenant, jobs);
|
|
275
|
+
if ('failure' in answer) {
|
|
276
|
+
lastError = answer.failure;
|
|
277
|
+
return { ok: false, reached: true, error: answer.failure };
|
|
278
|
+
}
|
|
279
|
+
answered = true;
|
|
280
|
+
output = answer.output;
|
|
281
|
+
return { ok: true, reached: true };
|
|
282
|
+
} catch (error) {
|
|
283
|
+
lastError = error;
|
|
284
|
+
return { ok: false, reached: false, error };
|
|
285
|
+
}
|
|
286
|
+
},
|
|
287
|
+
statusOf: (await import('@ultimat3/http')).statusFor,
|
|
288
|
+
});
|
|
289
|
+
const ms = Math.round(performance.now() - started);
|
|
290
|
+
if (kind === 'job') executions = (await jobs.drain()).executions;
|
|
291
|
+
const observed = room.end();
|
|
292
|
+
const effects = [...observed.effects, ...(await enqueued(subject, jobs, executions))];
|
|
293
|
+
const verdictOf = judged(subject, { verdict, answered, lastError, lastInput }, observed);
|
|
294
|
+
const record = { kind, name, statements: observed.statements, effects, ms, ...verdictOf };
|
|
295
|
+
return { record, output, input: lastInput };
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/** One B leg: B's actor, A's ids, a queue of its own. What it answered and every effect it had. */
|
|
299
|
+
async function attack(
|
|
300
|
+
subject: Subject,
|
|
301
|
+
input: unknown,
|
|
302
|
+
actor: Actor | null,
|
|
303
|
+
tenant: string,
|
|
304
|
+
room: ExerciseRoom,
|
|
305
|
+
): Promise<LegAnswer> {
|
|
306
|
+
await using jobs = await testJobs();
|
|
307
|
+
room.begin(subject.name);
|
|
308
|
+
let output: unknown;
|
|
309
|
+
// Every `crossTenant()` B's call opened: a job's declared sweep is told from a leak by it alone.
|
|
310
|
+
// Loaded here, never at the top: the barrel stays entity-free (`seed-writer.ts`'s rule).
|
|
311
|
+
const { setCrossTenantObserver } = await import('@ultimat3/entity');
|
|
312
|
+
const crossings: string[] = [];
|
|
313
|
+
setCrossTenantObserver((reason) => crossings.push(reason));
|
|
314
|
+
try {
|
|
315
|
+
const answer = await subject.call(input, actor, tenant, jobs);
|
|
316
|
+
output = answer.output;
|
|
317
|
+
} catch {
|
|
318
|
+
// B's refusal is the answer a scoped primitive gives: only what B got back, and what B's call
|
|
319
|
+
// did to A's rows, is judged — never whether B was refused.
|
|
320
|
+
} finally {
|
|
321
|
+
setCrossTenantObserver(undefined);
|
|
322
|
+
}
|
|
323
|
+
const observed = room.end();
|
|
324
|
+
const payloads = [...observed.payloads, ...(await queuedPayloads(subject))];
|
|
325
|
+
return { output, payloads, crossings };
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
async function exerciseOne(
|
|
329
|
+
subject: Subject,
|
|
330
|
+
options: ExercisePassOptions,
|
|
331
|
+
[tenant, other]: readonly [string, string],
|
|
332
|
+
graph: SeedGraph,
|
|
333
|
+
room: ExerciseRoom,
|
|
334
|
+
sampler: Sampler,
|
|
335
|
+
): Promise<ExerciseRecord> {
|
|
336
|
+
const { kind, name } = subject;
|
|
337
|
+
const override: ExerciseOverride | undefined = options.overrides?.entries.get(name);
|
|
338
|
+
const none = { kind, name, statements: 0, effects: [], ms: 0 };
|
|
339
|
+
if (override !== undefined && 'off' in override) {
|
|
340
|
+
return { ...none, outcome: 'off', reason: override.off.trim() };
|
|
341
|
+
}
|
|
342
|
+
const chosen = chooseActor(subject.policy, subject.permissions, options.roles, tenant, override);
|
|
343
|
+
if (chosen.kind === 'refused') return { ...none, outcome: 'refused', reason: chosen.reason };
|
|
344
|
+
const rows = actorRowsOf(options, graph);
|
|
345
|
+
const actor = seated(chosen.actor, tenant, rows);
|
|
346
|
+
// B is chosen as A was, in B's own tenant: the same role, so the policy is not what stops it.
|
|
347
|
+
const theirs = chooseActor(subject.policy, subject.permissions, options.roles, other, override);
|
|
348
|
+
const intruder = withoutCrossTenant(
|
|
349
|
+
theirs.kind === 'actor' ? seated(theirs.actor, other, rows) : null,
|
|
350
|
+
);
|
|
351
|
+
|
|
352
|
+
const scope = { tenant, actorId: actor?.id, subject: name, actorEntity: rows?.entity.$name };
|
|
353
|
+
const bindInput = overrideInput(name, override);
|
|
354
|
+
const declaredRuns = override !== undefined && 'actor' in override ? override.runs : undefined;
|
|
355
|
+
// A field naming the actor entity is tried as the actor's own row, then as somebody else's:
|
|
356
|
+
// both readings are legitimate and only the handler knows which it meant. Same minimal draw.
|
|
357
|
+
const twoWay =
|
|
358
|
+
declaredRuns === undefined &&
|
|
359
|
+
actor !== null &&
|
|
360
|
+
namesActorEntity(sampler.sampleInput(subject.input), options.entities, scope);
|
|
361
|
+
const lookups = [
|
|
362
|
+
seededIds(graph, options.entities, { ...scope, self: true }),
|
|
363
|
+
seededIds(graph, options.entities, { ...scope, self: false }),
|
|
364
|
+
] as const;
|
|
365
|
+
const run: Prepared = {
|
|
366
|
+
subject,
|
|
367
|
+
options,
|
|
368
|
+
tenant,
|
|
369
|
+
actor,
|
|
370
|
+
runs: declaredRuns ?? (twoWay ? 2 : 1),
|
|
371
|
+
declaredRuns,
|
|
372
|
+
lookups,
|
|
373
|
+
bindInput,
|
|
374
|
+
};
|
|
375
|
+
// A's own first input, A's ids in it: B sends exactly what A legitimately would.
|
|
376
|
+
const drawn = bindIds(drawInput(sampler, subject.input, 0, 0), lookups[0]);
|
|
377
|
+
const input = bindInput === undefined ? drawn : await bindInput(drawn, actor);
|
|
378
|
+
let reachedWith: unknown;
|
|
379
|
+
const record = await proveTenant({
|
|
380
|
+
taint: tenantTaint(graph, tenant),
|
|
381
|
+
input,
|
|
382
|
+
public: publicReason(subject.policy),
|
|
383
|
+
read: kind === 'query',
|
|
384
|
+
sweep: kind === 'job',
|
|
385
|
+
attack: () => attack(subject, input, intruder, other, room),
|
|
386
|
+
control: async () => {
|
|
387
|
+
const done = await control(run, room, sampler);
|
|
388
|
+
reachedWith = done.input;
|
|
389
|
+
return done;
|
|
390
|
+
},
|
|
391
|
+
snapshot: (seeded) => options.writer.snapshot(seeded),
|
|
392
|
+
});
|
|
393
|
+
// After the tenant proof, outside the room's begin/end: the crash runs' effects are their own.
|
|
394
|
+
return subject.handle === undefined || record.outcome !== 'reached'
|
|
395
|
+
? record
|
|
396
|
+
: { ...record, crash: await proveCrash(subject.handle, reachedWith, actor) };
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
export async function runExercisePass(
|
|
400
|
+
options: ExercisePassOptions,
|
|
401
|
+
): Promise<readonly ExerciseRecord[]> {
|
|
402
|
+
const subjects = await subjectsOf();
|
|
403
|
+
const known = new Set(subjects.map((subject) => subject.name));
|
|
404
|
+
assertOverridesKnown(options.overrides, known);
|
|
405
|
+
if (options.only !== undefined && !known.has(options.only)) {
|
|
406
|
+
throw unknownDeclaration([options.only], known, 'x exercise');
|
|
407
|
+
}
|
|
408
|
+
const chosen = subjects.filter((s) => options.only === undefined || s.name === options.only);
|
|
409
|
+
const sampler = await loadSampler();
|
|
410
|
+
// Not `Math.random`: the frozen RNG answers the same sequence in every process.
|
|
411
|
+
const passSeed = options.seed ?? (crypto.getRandomValues(new Uint32Array(1))[0] as number);
|
|
412
|
+
const room = await openExerciseRoom();
|
|
413
|
+
const live = chosen.filter((subject) => !isOff(options.overrides, subject.name));
|
|
414
|
+
const records: ExerciseRecord[] = [];
|
|
415
|
+
let graph: SeedGraph;
|
|
416
|
+
try {
|
|
417
|
+
// Seeded before the first primitive runs: a failure here is the pass's, never one primitive's.
|
|
418
|
+
graph = await seedPass(options.entities, options.writer, {
|
|
419
|
+
primitives: live.length,
|
|
420
|
+
seed: passSeed,
|
|
421
|
+
...(options.overrides?.factories === undefined
|
|
422
|
+
? {}
|
|
423
|
+
: { factories: options.overrides.factories }),
|
|
424
|
+
});
|
|
425
|
+
// In order, one at a time: the seams the room installs are the process's, not the primitive's.
|
|
426
|
+
for (const subject of chosen) {
|
|
427
|
+
const index = live.indexOf(subject);
|
|
428
|
+
const pair = tenantPair(graph, Math.max(0, index));
|
|
429
|
+
records.push(await exerciseOne(subject, options, pair, graph, room, sampler));
|
|
430
|
+
}
|
|
431
|
+
} catch (error) {
|
|
432
|
+
await room[Symbol.asyncDispose]();
|
|
433
|
+
throw error;
|
|
434
|
+
}
|
|
435
|
+
// The sweeps: every reached primitive again at two sizes, then every statement planned.
|
|
436
|
+
const reached = new Set(records.filter((r) => r.outcome === 'reached').map((r) => r.name));
|
|
437
|
+
return attachSweeps(records, {
|
|
438
|
+
subjects: chosen.filter((subject) => reached.has(subject.name)),
|
|
439
|
+
entities: options.entities,
|
|
440
|
+
writer: options.writer,
|
|
441
|
+
main: graph,
|
|
442
|
+
seed: passSeed,
|
|
443
|
+
room,
|
|
444
|
+
setup: async (subject, sized, tenant, self) => {
|
|
445
|
+
const full = chosen.find((one) => one.name === subject.name);
|
|
446
|
+
return full === undefined
|
|
447
|
+
? undefined
|
|
448
|
+
: await sizedCall(full, options, sampler, { graph: sized, tenant, self });
|
|
449
|
+
},
|
|
450
|
+
});
|
|
451
|
+
}
|
package/src/factories.ts
CHANGED
|
@@ -11,10 +11,13 @@ import { seededRandom, seededUuid } from './determinism';
|
|
|
11
11
|
import { FactoryTraitUnknownError } from './errors';
|
|
12
12
|
import { persistRow } from './factory-persist';
|
|
13
13
|
|
|
14
|
+
/**
|
|
15
|
+
* The two members of `entity()`'s value a factory reads, structurally: `@ultimat3/entity` is a
|
|
16
|
+
* dynamic-only dependency of this barrel. `$table` is the PHYSICAL table, what the persister writes.
|
|
17
|
+
*/
|
|
14
18
|
export interface EntityLike {
|
|
15
|
-
readonly
|
|
16
|
-
readonly
|
|
17
|
-
readonly columns: Readonly<Record<string, unknown>>;
|
|
19
|
+
readonly $table: string;
|
|
20
|
+
readonly $columns: Readonly<Record<string, unknown>>;
|
|
18
21
|
}
|
|
19
22
|
|
|
20
23
|
/** The seeded generators a `defaults` or trait body draws from. Never `Math.random` directly. */
|
|
@@ -106,7 +109,7 @@ export function defineFactory<TRow extends object, TTraits extends TraitMap<TRow
|
|
|
106
109
|
options: FactoryOptions<TRow, TTraits>,
|
|
107
110
|
): Factory<TRow, Extract<keyof TTraits, string>> {
|
|
108
111
|
type Name = Extract<keyof TTraits, string>;
|
|
109
|
-
const seed = options.seed ?? seedFor(entity
|
|
112
|
+
const seed = options.seed ?? seedFor(entity.$table);
|
|
110
113
|
const traits: TraitMap<TRow> = options.traits ?? {};
|
|
111
114
|
const declared = Object.keys(traits).sort() as Name[];
|
|
112
115
|
// Erased to one shape for iteration; the public `AssociationMap` already held each entry to its
|
|
@@ -130,7 +133,7 @@ export function defineFactory<TRow extends object, TTraits extends TraitMap<TRow
|
|
|
130
133
|
// was accepted the same way and applied nothing at all.
|
|
131
134
|
const assertTrait = (name: string): string => {
|
|
132
135
|
if (!Object.hasOwn(traits, name)) {
|
|
133
|
-
throw new FactoryTraitUnknownError({ table: entity
|
|
136
|
+
throw new FactoryTraitUnknownError({ table: entity.$table, trait: name, declared });
|
|
134
137
|
}
|
|
135
138
|
return name;
|
|
136
139
|
};
|
|
@@ -172,12 +175,12 @@ export function defineFactory<TRow extends object, TTraits extends TraitMap<TRow
|
|
|
172
175
|
if (!(column in overrides)) linked[column] = await link.create();
|
|
173
176
|
}
|
|
174
177
|
const row = { ...base, ...(linked as Partial<TRow>), ...overrides };
|
|
175
|
-
await persistRow(entity
|
|
178
|
+
await persistRow(entity.$table, row);
|
|
176
179
|
return row;
|
|
177
180
|
};
|
|
178
181
|
|
|
179
182
|
const view = (applied: readonly string[]): Factory<TRow, Name> => ({
|
|
180
|
-
table: entity
|
|
183
|
+
table: entity.$table,
|
|
181
184
|
traits: declared,
|
|
182
185
|
build: (over = {}) => buildRow(applied, over),
|
|
183
186
|
buildMany: (count, over = {}) => Array.from({ length: count }, () => buildRow(applied, over)),
|
package/src/factory-registry.ts
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
|
-
// One factory per registered entity,
|
|
2
|
-
//
|
|
1
|
+
// One factory per registered entity, each value read off its column's declaration — kind,
|
|
2
|
+
// nullability, enumerated values, money parts, default — through the seed graph's one value
|
|
3
|
+
// source. Enough for the rows a test does not care about; `defineFactory` is for the ones it does.
|
|
3
4
|
|
|
4
5
|
import type { EntityLike, Factory, FactoryIds } from './factories';
|
|
5
6
|
import { defineFactory } from './factories';
|
|
7
|
+
import { seedGraphUnsatisfied } from './seed-graph-error';
|
|
8
|
+
import { SEED_SOFT_DELETE_PROPERTY } from './seed-graph-shape';
|
|
9
|
+
import { candidatesFor, isSeedColumn } from './seed-graph-values';
|
|
6
10
|
|
|
7
11
|
export type EntityRegistry = Readonly<Record<string, EntityLike>>;
|
|
8
12
|
|
|
@@ -25,9 +29,13 @@ export function factoriesFor<TRegistry extends EntityRegistry>(
|
|
|
25
29
|
...(seed === undefined ? {} : { seed }),
|
|
26
30
|
defaults: (index, ids): Record<string, unknown> => {
|
|
27
31
|
const row: Record<string, unknown> = {};
|
|
28
|
-
|
|
29
|
-
row[
|
|
30
|
-
|
|
32
|
+
Object.entries(entity.$columns).forEach(([property, column], position) => {
|
|
33
|
+
row[property] = defaultFor(
|
|
34
|
+
{ table: entity.$table, property, column, position },
|
|
35
|
+
index,
|
|
36
|
+
ids,
|
|
37
|
+
);
|
|
38
|
+
});
|
|
31
39
|
return row;
|
|
32
40
|
},
|
|
33
41
|
});
|
|
@@ -35,16 +43,42 @@ export function factoriesFor<TRegistry extends EntityRegistry>(
|
|
|
35
43
|
return out as FactoryRegistry<TRegistry>;
|
|
36
44
|
}
|
|
37
45
|
|
|
46
|
+
/** One column of one entity: what `defaultFor` reads. The name labels a refusal, nothing more. */
|
|
47
|
+
export interface DefaultColumn {
|
|
48
|
+
readonly table: string;
|
|
49
|
+
readonly property: string;
|
|
50
|
+
readonly column: unknown;
|
|
51
|
+
readonly position: number;
|
|
52
|
+
}
|
|
53
|
+
|
|
38
54
|
/**
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
55
|
+
* The first value the column's own declaration admits — its default, else what its kind offers.
|
|
56
|
+
* A value that is not an `entity()` column, or a column that refuses every generated value, is
|
|
57
|
+
* `X_SEED_GRAPH_UNSATISFIED`: the fix is a `defineFactory` naming that column.
|
|
42
58
|
*/
|
|
43
|
-
export function defaultFor(
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
if (column.
|
|
48
|
-
|
|
49
|
-
|
|
59
|
+
export function defaultFor(target: DefaultColumn, index: number, ids: FactoryIds): unknown {
|
|
60
|
+
const { column, property, table } = target;
|
|
61
|
+
// A built row is a LIVE one: a soft-delete instant would hide it from every read, as `seedGraph`
|
|
62
|
+
// already keeps it `null` (`seed-graph.ts`).
|
|
63
|
+
if (property === SEED_SOFT_DELETE_PROPERTY && isSeedColumn(column) && !column.$meta.notNull) {
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
const values = isSeedColumn(column)
|
|
67
|
+
? candidatesFor(property, column, {
|
|
68
|
+
ordinal: index,
|
|
69
|
+
canary: String(index),
|
|
70
|
+
position: target.position,
|
|
71
|
+
uuid: () => ids.uuid(),
|
|
72
|
+
})
|
|
73
|
+
: [];
|
|
74
|
+
if (values.length === 0) {
|
|
75
|
+
throw seedGraphUnsatisfied({
|
|
76
|
+
entity: table,
|
|
77
|
+
what: isSeedColumn(column)
|
|
78
|
+
? `no generated value is storable in ${property} — its column refuses every one`
|
|
79
|
+
: `${property} is not an entity() column, so it declares nothing to generate from`,
|
|
80
|
+
columns: [property],
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
return values[0];
|
|
50
84
|
}
|