@ultimat3/testing 30.2.0 → 30.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +2 -2
- package/README.md +2 -1
- package/package.json +17 -17
- package/src/define-exercise.ts +11 -0
- package/src/errors.ts +2 -0
- package/src/exercise-job-tenant.ts +103 -0
- package/src/exercise-report.ts +12 -2
- package/src/exercise-setup.ts +68 -0
- package/src/exercise-subjects.ts +133 -0
- package/src/exercise-sweeps.ts +6 -10
- package/src/exercise.ts +30 -92
- package/src/factory-registry.ts +14 -2
- package/src/index.ts +1 -0
- package/src/plan-oracle.ts +5 -2
- package/src/plan-sweep-fix.ts +59 -7
- package/src/plan-sweep-report.ts +22 -3
- package/src/registry-leak-guard.ts +2 -2
- package/src/seed-graph-error.ts +5 -1
- package/src/seed-graph-order.ts +9 -5
- package/src/seed-graph-pass.ts +98 -3
- package/src/seed-graph-patterns.ts +51 -0
- package/src/seed-graph-plan.ts +23 -1
- package/src/seed-graph-values.ts +64 -5
- package/src/seed-graph.ts +41 -5
- package/src/seed-pattern-sample.ts +216 -0
- package/src/seed-writer.ts +47 -26
- package/src/subject-proof.ts +3 -1
package/CLAUDE.md
CHANGED
|
@@ -51,13 +51,13 @@ import is `registry-isolation.ts`, own entry point, off the barrel. Full rule: `
|
|
|
51
51
|
| A passkey test signs for real | `softAuthenticator` is `@ultimat3/auth/soft-authenticator`'s, never re-exported here (one value, one path: `flight-copies`) and never a second one (auth's own suites need it, and auth cannot import testing) |
|
|
52
52
|
| A reached job is crash-proven | `exercise-crash.ts`: `crashProof` with the control's input and actor, after the tenant proof; only `diverged` is red (`X_JOB_CRASH_DIVERGED`). Committed: job bodies escape transactions |
|
|
53
53
|
| The tenant proof is taint only | `tenant-proof.ts`: B, A, B; B is A's role minus `tenancy:cross`. Leak = seeded canary or unhanded id (`tenant-canary.ts`); write = moved digest. Only `crossed` is red |
|
|
54
|
-
| Every statement the pass sent is planned | `plan-sweep.ts` (`sweepPlans`): first relation an app table, `PREPARE` in a READ-ONLY transaction with seq/bitmap scans and sorts off, judged by `plan-oracle.ts`; red = `X_QUERY_UNINDEXED` / `X_QUERY_SORT_UNSERVED` with the `indexes:` entry (`plan-sweep-fix.ts`). `scan:` or `expectedSeqScan` is exempt, listed; no generic plan is `X_QUERY_PLAN_UNAVAILABLE`, never a pass |
|
|
54
|
+
| Every statement the pass sent is planned | `plan-sweep.ts` (`sweepPlans`): first relation an app table, `PREPARE` in a READ-ONLY transaction with seq/bitmap scans and sorts off, judged by `plan-oracle.ts`; red = `X_QUERY_UNINDEXED` / `X_QUERY_SORT_UNSERVED` with the `indexes:` entry (`plan-sweep-fix.ts`). `scan:` or `expectedSeqScan` is exempt, listed; credited: a partial unique whose predicate the statement states (one row), a declared equality-then-range index under a limit; no generic plan is `X_QUERY_PLAN_UNAVAILABLE`, never a pass |
|
|
55
55
|
| A shape that grows with the rows is red | `scale-sweep-run.ts`: two more tenants at `SCALE_SIZES` (2, 8 rows a table), every reached primitive once in each; `judgeScale` (pure) reds `X_QUERY_SCALES_WITH_ROWS`. Under `expectedQueryLoop` it is listed with its reason |
|
|
56
56
|
| An erase is PROVED, not asserted | `subject-proof.ts` (`subjectProof`, run by cli's `exercise-subject.ts`): canaries in every table, one person erased, every stored row searched; a survivor outside `retain` is `X_SUBJECT_ERASE_INCOMPLETE`, naming table and column, never the value |
|
|
57
57
|
| A captured failure replays as a test | `repro.ts`: `reproduce(bundle, { driver })` seeds the rows it read, freezes the clock at its instant, invokes as its actor; masked fields are stood in (`repro-stand-in.ts`). An unregistered name ANSWERS as the failure. `reproJob(runJobs, failure)` for a dead job. What `x repro --write` emits |
|
|
58
58
|
| A vendor's answers are a cassette | `cassette.ts`: replay = one `mockFetch` route per recorded URL, seal kept (`X_CASSETTE_MISSING`, `X_CASSETTE_UNUSED`). Record = `ULTIMATE_CASSETTE=record` + `record.hosts`, refused under `CI` (`X_CASSETTE_RECORD_REFUSED`), headers allow-listed, credential keys masked, bytes scanned before disk (`X_CASSETTE_SECRET`). The request side is never written |
|
|
59
59
|
| One seed writer | `entitySeedWriter({ entities, client \| driver })` (`seed-writer.ts`) writes through each entity's own repository; `snapshot` is required; `entity` loads on the first write, so the barrel stays entity-free |
|
|
60
|
-
| Factory values are DECLARED | `factoriesFor` and `seedGraph` share `seed-graph-values.ts`: default, enum, kind, through `$parse`, never the name. `seed-graph*.ts` type-import `entity` only |
|
|
60
|
+
| Factory values are DECLARED | `factoriesFor` and `seedGraph` share `seed-graph-values.ts`: default, enum, kind, a `matches()` pattern (`seed-pattern-sample.ts`, read off the CHECK by `seed-graph-patterns.ts`), `$meta.schema` (sampler `require`d on first use), through `$parse`, never the name. `seed-graph*.ts` type-import `entity` only |
|
|
61
61
|
| e2e without a driver | `e2eTest` becomes `test.skip`; the gate reports the step SKIPPED (or `X_VERIFY_SUITE_VANISHED` when `x.verify.json` requires it), never passed |
|
|
62
62
|
| The seam has an inverse | `resetE2eDriver()`: `useE2eDriver` writes MODULE scope, so a file that installs a browser undoes it |
|
|
63
63
|
| Built vs declared | `clock` `mail` `network` `runJobs` `statements` `subscribe` are built in-process; `page` `budget` `signIn` `deploy` are declared and wait for a driver (`X_TEST_FIXTURE_UNAVAILABLE`). |
|
package/README.md
CHANGED
|
@@ -34,7 +34,8 @@ unitTest('the clock moves only when the test moves it', ({ clock }) => {
|
|
|
34
34
|
| `seed-graph-error.ts` | `X_SEED_GRAPH_UNSATISFIED`, its fix naming the `defineFactory` to write; `X_SEED_GRAPH_ENTITY_UNKNOWN`, a plan and a writer built from two entity sets |
|
|
35
35
|
| `seed-writer.ts` | `entitySeedWriter({ entities, client })` or `({ entities, driver? })` — the `SeedWriter` that writes a graph through each entity's own repository: Postgres or PGlite through a client, or a driver. `@ultimat3/entity` loads on the first write. `snapshot(rows)`, required on every `SeedWriter`, reads rows back as one digest each (`md5(t::text)` over a client), `null` for a row gone |
|
|
36
36
|
| `exercise.ts` | `runExercisePass({ entities, writer, roles, seed?, overrides?, only? })` — every registered action, query and job once, each on its own committed tenant pair, one `ExerciseRecord` per primitive, the tenant proof's verdict on its `tenant`. What `x exercise` runs |
|
|
37
|
-
| `define-exercise.ts` | `defineExercise({ <name>: { actor, input?, runs? } \| { off: '<why>' } }, { actor?: <entity>, factories?: { <entity name>: defineFactory(…) } })` — an app's overrides (the second argument names the entity whose rows are the actors, and the factories that build a table no generated row satisfies — a cross-column `satisfies` — keyed by entity name, unknown ones `X_DECLARATION_UNKNOWN`), `apps/web/exercise.ts`'s one export; a blank `off` is `X_CONTRACT_EXERCISE_UNDECLARED`, an unknown name `X_DECLARATION_UNKNOWN` when the pass runs |
|
|
37
|
+
| `define-exercise.ts` | `defineExercise({ <name>: { actor, input?, runs? } \| { off: '<why>' } }, { actor?: <entity>, factories?: { <entity name>: defineFactory(…) }, setup?: () => mockFetch(…) \| cassette(…) })` — an app's overrides (the second argument names the entity whose rows are the actors, and the factories that build a table no generated row satisfies — a cross-column `satisfies` — keyed by entity name, unknown ones `X_DECLARATION_UNKNOWN`), `apps/web/exercise.ts`'s one export; a blank `off` is `X_CONTRACT_EXERCISE_UNDECLARED`, an unknown name `X_DECLARATION_UNKNOWN` when the pass runs |
|
|
38
|
+
| `exercise-setup.ts` | `withExerciseSetup(room, setup)`: the app's `setup` run inside the sealed room before the first row, its handle disposed before the room restores the network; an `allowHost` in it is `X_EXERCISE_SETUP_UNSEALED`. Types `ExerciseSetup`, `ExerciseSetupHandle` |
|
|
38
39
|
| `exercise-actor.ts` · `exercise-input.ts` · `exercise-report.ts` | the first `x policy explain` actor the policy allows, minted in the primitive's tenant · the input: override, then a seeded id by field stem, then the minimal draw (`@ultimat3/schema/sample`, loaded dynamically) · the room one run executes in (network sealed, mail captured, clock frozen) and its record, `overrideLine` the entry to paste |
|
|
39
40
|
| `tenant-proof.ts` | internal (`CROSS_TENANT_SCOPE` is `@ultimat3/entity`'s): `proveTenant` — B with A's ids, A's own call, B again; each B leg between two snapshots of A's rows. `withoutCrossTenant` strips `tenancy:cross` (`CROSS_TENANT_SCOPE`) from B; `publicReason` decides `public` |
|
|
40
41
|
| `tenant-canary.ts` | internal: `tenantTaint` — one tenant's rows, ids and marks (row canaries, key parts of `MIN_TAINT_ID_LENGTH` 8 or more no other tenant shares); `findTaint` — every path holding one |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/testing",
|
|
3
|
-
"version": "30.
|
|
3
|
+
"version": "30.4.0",
|
|
4
4
|
"description": "Test harness: cloned template DBs per worker, frozen clock, sealed network, 6 test types",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ultimate",
|
|
@@ -46,21 +46,21 @@
|
|
|
46
46
|
"test": "bun test"
|
|
47
47
|
},
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@ultimat3/action": "30.
|
|
50
|
-
"@ultimat3/cache": "30.
|
|
51
|
-
"@ultimat3/core": "30.
|
|
52
|
-
"@ultimat3/db": "30.
|
|
53
|
-
"@ultimat3/entity": "30.
|
|
54
|
-
"@ultimat3/http": "30.
|
|
55
|
-
"@ultimat3/i18n": "30.
|
|
56
|
-
"@ultimat3/jobs": "30.
|
|
57
|
-
"@ultimat3/mail": "30.
|
|
58
|
-
"@ultimat3/policy": "30.
|
|
59
|
-
"@ultimat3/pwa": "30.
|
|
60
|
-
"@ultimat3/query": "30.
|
|
61
|
-
"@ultimat3/realtime": "30.
|
|
62
|
-
"@ultimat3/render": "30.
|
|
63
|
-
"@ultimat3/schema": "30.
|
|
64
|
-
"@ultimat3/time": "30.
|
|
49
|
+
"@ultimat3/action": "30.4.0",
|
|
50
|
+
"@ultimat3/cache": "30.4.0",
|
|
51
|
+
"@ultimat3/core": "30.4.0",
|
|
52
|
+
"@ultimat3/db": "30.4.0",
|
|
53
|
+
"@ultimat3/entity": "30.4.0",
|
|
54
|
+
"@ultimat3/http": "30.4.0",
|
|
55
|
+
"@ultimat3/i18n": "30.4.0",
|
|
56
|
+
"@ultimat3/jobs": "30.4.0",
|
|
57
|
+
"@ultimat3/mail": "30.4.0",
|
|
58
|
+
"@ultimat3/policy": "30.4.0",
|
|
59
|
+
"@ultimat3/pwa": "30.4.0",
|
|
60
|
+
"@ultimat3/query": "30.4.0",
|
|
61
|
+
"@ultimat3/realtime": "30.4.0",
|
|
62
|
+
"@ultimat3/render": "30.4.0",
|
|
63
|
+
"@ultimat3/schema": "30.4.0",
|
|
64
|
+
"@ultimat3/time": "30.4.0"
|
|
65
65
|
}
|
|
66
66
|
}
|
package/src/define-exercise.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import type { ExerciseOptions } from '@ultimat3/action';
|
|
7
7
|
import { renderCauseValue, renderFixLiteral, UltimateError } from '@ultimat3/core';
|
|
8
|
+
import type { ExerciseSetup } from './exercise-setup';
|
|
8
9
|
import type { SeedFactory } from './seed-graph-plan';
|
|
9
10
|
|
|
10
11
|
/** 39's option type, exactly: `{ actor, input?, runs? }` or `{ off: '<reason>' }`. */
|
|
@@ -18,6 +19,8 @@ export interface ExerciseOverrides {
|
|
|
18
19
|
readonly actorEntity?: string;
|
|
19
20
|
/** By entity name: the pass's graph builds those rows with them (`seedGraph`'s `factories`). */
|
|
20
21
|
readonly factories?: Readonly<Record<string, SeedFactory>>;
|
|
22
|
+
/** The app's hook, run once inside the pass's sealed room (`exercise-setup.ts`). */
|
|
23
|
+
readonly setup?: ExerciseSetup;
|
|
21
24
|
}
|
|
22
25
|
|
|
23
26
|
/** What the whole pass needs to know of the app, beside the per-primitive entries. */
|
|
@@ -34,6 +37,13 @@ export interface ExerciseAppOptions {
|
|
|
34
37
|
* refused when the pass runs.
|
|
35
38
|
*/
|
|
36
39
|
readonly factories?: Readonly<Record<string, SeedFactory>>;
|
|
40
|
+
/**
|
|
41
|
+
* Run in the exercise process once the network is sealed, before the first row: install the
|
|
42
|
+
* `mockFetch` routes or `cassette()` your vendor-calling handlers need, so they run for real.
|
|
43
|
+
* What it returns (a cassette, any `Disposable`) is disposed after the pass. `allowHost` here is
|
|
44
|
+
* `X_EXERCISE_SETUP_UNSEALED` — the pass never sends a call out.
|
|
45
|
+
*/
|
|
46
|
+
readonly setup?: ExerciseSetup;
|
|
37
47
|
}
|
|
38
48
|
|
|
39
49
|
/** `X_CONTRACT_EXERCISE_UNDECLARED`, 39's code for the same silence: an `off` with no reason. */
|
|
@@ -59,6 +69,7 @@ export function defineExercise(
|
|
|
59
69
|
entries,
|
|
60
70
|
...(app.actor === undefined ? {} : { actorEntity: app.actor.$name }),
|
|
61
71
|
...(app.factories === undefined ? {} : { factories: app.factories }),
|
|
72
|
+
...(app.setup === undefined ? {} : { setup: app.setup }),
|
|
62
73
|
});
|
|
63
74
|
}
|
|
64
75
|
|
package/src/errors.ts
CHANGED
|
@@ -69,6 +69,7 @@ export const TESTING_ERROR_CODES = [
|
|
|
69
69
|
'X_CASSETTE_SECRET',
|
|
70
70
|
'X_CASSETTE_RECORD_REFUSED',
|
|
71
71
|
'X_TEST_MODULE_MOCK_UNRESTORABLE',
|
|
72
|
+
'X_EXERCISE_SETUP_UNSEALED',
|
|
72
73
|
] as const;
|
|
73
74
|
|
|
74
75
|
export type TestingErrorCode = (typeof TESTING_ERROR_CODES)[number];
|
|
@@ -133,6 +134,7 @@ export const TESTING_ERROR_TITLES: Readonly<Record<TestingErrorCode, string>> =
|
|
|
133
134
|
X_CASSETTE_SECRET: 'a cassette about to be written holds a secret value',
|
|
134
135
|
X_CASSETTE_RECORD_REFUSED: 'a cassette recording was refused',
|
|
135
136
|
X_TEST_MODULE_MOCK_UNRESTORABLE: 'a mocked module could not be put back at the file boundary',
|
|
137
|
+
X_EXERCISE_SETUP_UNSEALED: 'an exercise setup let a host through the seal',
|
|
136
138
|
};
|
|
137
139
|
|
|
138
140
|
// Titles must be registered for `format()` to render the contract's first line. Every code above is
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
// A job's input, seated in the tenant the pass enqueues it under. A job DECLARES its org as a
|
|
2
|
+
// function of its input (`tenant: (input) => input.orgId`), so a drawn input names a random org and
|
|
3
|
+
// the enqueue refuses it (`X_JOB_TENANT_MISMATCH`). The fields that function reads are found by
|
|
4
|
+
// reading them — a recording proxy, no field-name guess — and each is set to the seated tenant.
|
|
5
|
+
|
|
6
|
+
/** The handle's half the binder needs: `AnyJobHandle.tenantFor`, structurally. */
|
|
7
|
+
export interface TenantOf {
|
|
8
|
+
readonly name: string;
|
|
9
|
+
tenantFor(input: unknown): string | undefined;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/** `seated`: the input names the tenant. `fixed`: no field it reads moves the org it answers. */
|
|
13
|
+
export type JobTenantBinding =
|
|
14
|
+
| { readonly kind: 'seated'; readonly input: unknown }
|
|
15
|
+
| { readonly kind: 'fixed'; readonly declared: string | undefined };
|
|
16
|
+
|
|
17
|
+
type Path = readonly string[];
|
|
18
|
+
|
|
19
|
+
const isPlainObject = (value: unknown): value is Record<string, unknown> => {
|
|
20
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) return false;
|
|
21
|
+
const prototype: unknown = Object.getPrototypeOf(value);
|
|
22
|
+
return prototype === null || prototype === Object.prototype;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
/** What `tenantFor` answered, or `undefined` for a throw: an unreadable input names no org. */
|
|
26
|
+
const answer = (handle: TenantOf, input: unknown): string | undefined => {
|
|
27
|
+
try {
|
|
28
|
+
return handle.tenantFor(input);
|
|
29
|
+
} catch {
|
|
30
|
+
return undefined;
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
/** How deep an absent parent is filled in to read past it: `payload.input.orgId` is two. */
|
|
35
|
+
const ABSENT_DEPTH = 4;
|
|
36
|
+
|
|
37
|
+
/** The value at `path`, `undefined` when any step of it is absent. */
|
|
38
|
+
const getAt = (input: unknown, path: Path): unknown =>
|
|
39
|
+
path.reduce<unknown>((value, key) => (isPlainObject(value) ? value[key] : undefined), input);
|
|
40
|
+
|
|
41
|
+
/** Every property path `tenantFor` read, at any depth of plain objects — a missing one included. */
|
|
42
|
+
function readOnce(handle: TenantOf, input: unknown): readonly Path[] {
|
|
43
|
+
const read: Path[] = [];
|
|
44
|
+
const watch = (value: unknown, at: Path): unknown =>
|
|
45
|
+
!isPlainObject(value)
|
|
46
|
+
? value
|
|
47
|
+
: new Proxy(value, {
|
|
48
|
+
get(target, key, receiver) {
|
|
49
|
+
if (typeof key !== 'string') return Reflect.get(target, key, receiver);
|
|
50
|
+
const path = [...at, key];
|
|
51
|
+
read.push(path);
|
|
52
|
+
return watch(Reflect.get(target, key, receiver), path);
|
|
53
|
+
},
|
|
54
|
+
});
|
|
55
|
+
answer(handle, watch(input, []));
|
|
56
|
+
return read;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Every path read, past absent parents too: a read that found nothing is filled with `{}` and read
|
|
61
|
+
* again, so `payload.input.orgId` — a scoped backfill's tenant, its `input` left out of the
|
|
62
|
+
* minimal draw — is found, not lost to the throw reading `orgId` of `undefined`.
|
|
63
|
+
*/
|
|
64
|
+
function pathsRead(handle: TenantOf, input: unknown, depth = 0): readonly Path[] {
|
|
65
|
+
const reads = readOnce(handle, input);
|
|
66
|
+
if (depth >= ABSENT_DEPTH) return reads;
|
|
67
|
+
const deeper = reads
|
|
68
|
+
.filter((path) => getAt(input, path) === undefined)
|
|
69
|
+
.flatMap((path) => pathsRead(handle, setAt(input, path, {}), depth + 1));
|
|
70
|
+
const seen = new Set<string>();
|
|
71
|
+
return [...reads, ...deeper].filter((path) => {
|
|
72
|
+
const key = JSON.stringify(path);
|
|
73
|
+
if (seen.has(key)) return false;
|
|
74
|
+
seen.add(key);
|
|
75
|
+
return true;
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** `input` with `path` set to `value`, copied along the path only. */
|
|
80
|
+
function setAt(input: unknown, path: Path, value: unknown): unknown {
|
|
81
|
+
const [head, ...rest] = path;
|
|
82
|
+
if (head === undefined) return value;
|
|
83
|
+
const base = isPlainObject(input) ? input : {};
|
|
84
|
+
return { ...base, [head]: setAt(base[head], rest, value) };
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* `input` naming `tenant` wherever `handle.tenantFor` reads it. A `'none'` job, or one already
|
|
89
|
+
* naming it, comes back as given. A read path is kept only if setting it moves the answer to
|
|
90
|
+
* `tenant` — a field read and then ignored is left as drawn.
|
|
91
|
+
*/
|
|
92
|
+
export function bindJobTenant(handle: TenantOf, input: unknown, tenant: string): JobTenantBinding {
|
|
93
|
+
const declared = answer(handle, input);
|
|
94
|
+
if (declared === tenant) return { kind: 'seated', input };
|
|
95
|
+
const reads = pathsRead(handle, input);
|
|
96
|
+
// `tenant: 'none'` is never called: no org named, none read, so none to disagree with.
|
|
97
|
+
if (declared === undefined && reads.length === 0) return { kind: 'seated', input };
|
|
98
|
+
const moving = reads.filter((path) => answer(handle, setAt(input, path, tenant)) === tenant);
|
|
99
|
+
const bound = moving.reduce<unknown>((value, path) => setAt(value, path, tenant), input);
|
|
100
|
+
return moving.length > 0 && answer(handle, bound) === tenant
|
|
101
|
+
? { kind: 'seated', input: bound }
|
|
102
|
+
: { kind: 'fixed', declared };
|
|
103
|
+
}
|
package/src/exercise-report.ts
CHANGED
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
import type { RowObserver } from '@ultimat3/entity';
|
|
14
14
|
import { captureDeterminism, installDeterminism, restoreCapturedDeterminism } from './determinism';
|
|
15
15
|
import { testMail } from './fixture-mail';
|
|
16
|
+
import { routeMatches } from './mock-route';
|
|
16
17
|
import type { PlanProof, SentStatement } from './plan-sweep';
|
|
17
18
|
import type { ScaleProof, StatementTally } from './scale-sweep';
|
|
18
19
|
import {
|
|
@@ -99,6 +100,11 @@ export interface ExerciseRecord {
|
|
|
99
100
|
readonly ms: number;
|
|
100
101
|
/** For an `unreached` one: the line to paste into `defineExercise({ … })`. */
|
|
101
102
|
readonly override?: string;
|
|
103
|
+
/**
|
|
104
|
+
* For an `unreached` one no override reaches: the edit that does, in place of the paste — a job
|
|
105
|
+
* whose `tenant(input)` names one fixed org (`exercise-subjects.ts`'s `fixedTenant`).
|
|
106
|
+
*/
|
|
107
|
+
readonly fix?: string;
|
|
102
108
|
/** On every record that ran — absent only on `off` and `refused`. The fields above are A's. */
|
|
103
109
|
readonly tenant?: TenantProof;
|
|
104
110
|
/** On a reached job only: its control's input, killed at every step boundary. */
|
|
@@ -223,6 +229,10 @@ export async function openExerciseRoom(): Promise<ExerciseRoom> {
|
|
|
223
229
|
},
|
|
224
230
|
end: (): Observed => {
|
|
225
231
|
const calls = requestLog().slice(logFrom);
|
|
232
|
+
// A call a route answered never left the process: the app's `setup` installed it.
|
|
233
|
+
const { mocks } = captureNetwork();
|
|
234
|
+
const refused = (call: (typeof calls)[number]): boolean =>
|
|
235
|
+
!local(call.url) && !mocks.some((route) => routeMatches(route, call));
|
|
226
236
|
const effects: ExerciseEffect[] = [
|
|
227
237
|
...[...tally(rows)].map(([key, count]): ExerciseEffect => {
|
|
228
238
|
const [entity = '', op = ''] = key.split('\u0000');
|
|
@@ -236,11 +246,11 @@ export async function openExerciseRoom(): Promise<ExerciseRoom> {
|
|
|
236
246
|
kind: 'network',
|
|
237
247
|
method: call.method,
|
|
238
248
|
host: hostOf(call.url),
|
|
239
|
-
refused:
|
|
249
|
+
refused: refused(call),
|
|
240
250
|
}),
|
|
241
251
|
),
|
|
242
252
|
];
|
|
243
|
-
const refusedHosts = calls.filter(
|
|
253
|
+
const refusedHosts = calls.filter(refused).map((c) => hostOf(c.url));
|
|
244
254
|
const payloads: EffectPayload[] = [
|
|
245
255
|
...mail.outbox().map(({ message }, i) => ({
|
|
246
256
|
path: `effect.mail[${i}]`,
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// `defineExercise(…, { setup })`, run: the app's hook inside the pass's sealed room, before the
|
|
2
|
+
// first row is seeded, disposed before the room restores the network. What it installs — a
|
|
3
|
+
// `mockFetch` route, a `cassette()` — answers a vendor call; `allowHost` would send one for real.
|
|
4
|
+
|
|
5
|
+
import { renderCauseValue, UltimateError } from '@ultimat3/core';
|
|
6
|
+
import type { ExerciseRoom } from './exercise-report';
|
|
7
|
+
import { captureNetwork } from './sealed-network';
|
|
8
|
+
|
|
9
|
+
/** What a setup may hand back to be undone after the pass: a `cassette()`, a `using` resource. */
|
|
10
|
+
export type ExerciseSetupHandle = Disposable | AsyncDisposable;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Runs once per pass, in the exercise process, after the seal and before any primitive. Returns
|
|
14
|
+
* nothing when there is nothing to undo — a `mockFetch` route goes with the room.
|
|
15
|
+
*/
|
|
16
|
+
export type ExerciseSetup = () =>
|
|
17
|
+
| void
|
|
18
|
+
| ExerciseSetupHandle
|
|
19
|
+
| Promise<void>
|
|
20
|
+
| Promise<ExerciseSetupHandle>
|
|
21
|
+
| Promise<ExerciseSetupHandle | undefined>;
|
|
22
|
+
|
|
23
|
+
const setupUnsealed = (hosts: readonly string[]): UltimateError =>
|
|
24
|
+
new UltimateError({
|
|
25
|
+
code: 'X_EXERCISE_SETUP_UNSEALED',
|
|
26
|
+
cause: `defineExercise(…, { setup }) called allowHost(${hosts.map((host) => renderCauseValue(host)).join(', ')}): the pass would send a primitive's call out for real, where every vendor is answered by mockFetch or cassette()`,
|
|
27
|
+
fix: "mockFetch('https://<host>/*', handler) # or cassette(import.meta.dir, '<name>') — in apps/web/exercise.ts's setup, in place of allowHost('<host>')",
|
|
28
|
+
meta: { hosts },
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
async function release(handle: unknown): Promise<void> {
|
|
32
|
+
if (typeof handle !== 'object' || handle === null) return;
|
|
33
|
+
if (Symbol.asyncDispose in handle) await (handle as AsyncDisposable)[Symbol.asyncDispose]();
|
|
34
|
+
else if (Symbol.dispose in handle) (handle as Disposable)[Symbol.dispose]();
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* `room` with `setup` run inside it. Its dispose releases what setup returned FIRST — a cassette's
|
|
39
|
+
* own check reads the routes the pass used — then the room. A setup that throws, or lets a host
|
|
40
|
+
* through, disposes both before the pass refuses.
|
|
41
|
+
*/
|
|
42
|
+
export async function withExerciseSetup(
|
|
43
|
+
room: ExerciseRoom,
|
|
44
|
+
setup: ExerciseSetup | undefined,
|
|
45
|
+
): Promise<ExerciseRoom> {
|
|
46
|
+
if (setup === undefined) return room;
|
|
47
|
+
const before = new Set(captureNetwork().allowed);
|
|
48
|
+
let handle: unknown;
|
|
49
|
+
try {
|
|
50
|
+
handle = await setup();
|
|
51
|
+
} catch (error) {
|
|
52
|
+
await room[Symbol.asyncDispose]();
|
|
53
|
+
throw error;
|
|
54
|
+
}
|
|
55
|
+
const dispose = async (): Promise<void> => {
|
|
56
|
+
try {
|
|
57
|
+
await release(handle);
|
|
58
|
+
} finally {
|
|
59
|
+
await room[Symbol.asyncDispose]();
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
const opened = captureNetwork().allowed.filter((host) => !before.has(host));
|
|
63
|
+
if (opened.length > 0) {
|
|
64
|
+
await dispose();
|
|
65
|
+
throw setupUnsealed(opened);
|
|
66
|
+
}
|
|
67
|
+
return { ...room, [Symbol.asyncDispose]: dispose };
|
|
68
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
// The primitives an exercise pass drives, each as one call: every registered action, query and job,
|
|
2
|
+
// loaded on demand. A job is enqueued in the tenant it is called for, its input SEATED there first
|
|
3
|
+
// (`exercise-job-tenant.ts`): a drawn org is a random one, and the enqueue refuses it.
|
|
4
|
+
|
|
5
|
+
import type { Actor } from '@ultimat3/core';
|
|
6
|
+
import { renderCauseValue, renderFixShellArg, UltimateError } from '@ultimat3/core';
|
|
7
|
+
import type { AnyJobHandle, JobExecution } from '@ultimat3/jobs';
|
|
8
|
+
import { registeredJobs } from '@ultimat3/jobs';
|
|
9
|
+
import type { Policy } from '@ultimat3/policy';
|
|
10
|
+
import type { StandardSchemaV1 } from '@ultimat3/schema';
|
|
11
|
+
import type { JobTenantBinding } from './exercise-job-tenant';
|
|
12
|
+
import { bindJobTenant } from './exercise-job-tenant';
|
|
13
|
+
import { withExerciseRequest } from './exercise-request';
|
|
14
|
+
import type { RunJobs } from './fixture-jobs';
|
|
15
|
+
|
|
16
|
+
/** `failure` set: the worker recorded one (a job's). Absent: answered, with `output`. */
|
|
17
|
+
export interface Answer {
|
|
18
|
+
readonly failure?: unknown;
|
|
19
|
+
readonly output?: unknown;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** One primitive, as the pass drives it: the schema to draw from and the one call to make. */
|
|
23
|
+
export interface Subject {
|
|
24
|
+
readonly kind: 'action' | 'query' | 'job';
|
|
25
|
+
readonly name: string;
|
|
26
|
+
readonly input: StandardSchemaV1;
|
|
27
|
+
readonly policy: Policy | undefined;
|
|
28
|
+
/** Every permission the policy references — what `x policy explain` matches grants against. */
|
|
29
|
+
readonly permissions: readonly string[];
|
|
30
|
+
/** A job's handle: what the crash proof kills at every step boundary once the control reached it. */
|
|
31
|
+
readonly handle?: AnyJobHandle;
|
|
32
|
+
/** A job's: `input` naming `tenant` wherever its declared `tenant(input)` reads the org. */
|
|
33
|
+
seat?(input: unknown, tenant: string): JobTenantBinding;
|
|
34
|
+
/** Resolves to the failure a worker recorded on the row (a job's), or what the call answered. */
|
|
35
|
+
call(input: unknown, actor: Actor | null, tenant: string, jobs: RunJobs): Promise<Answer>;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** `input` seated in `tenant` when the subject declares a seat and one exists; else as given. */
|
|
39
|
+
export const seatIn = (subject: Pick<Subject, 'seat'>, input: unknown, tenant: string): unknown => {
|
|
40
|
+
const seat = subject.seat?.(input, tenant);
|
|
41
|
+
return seat?.kind === 'seated' ? seat.input : input;
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Why a job whose org no input field moves is not run: the pass seats every primitive in a FRESH
|
|
46
|
+
* tenant of its own, and a constant org is none of them. Seeding that one org instead would hand
|
|
47
|
+
* every such job the same tenant — no fresh pair, no tenant proof — so it is reported, never run
|
|
48
|
+
* in a tenant whose rows the pass did not seed. The repair is the declaration's.
|
|
49
|
+
*/
|
|
50
|
+
export function fixedTenant(name: string, declared: string | undefined) {
|
|
51
|
+
const org = declared === undefined ? 'no org' : `org ${renderCauseValue(declared)}`;
|
|
52
|
+
return {
|
|
53
|
+
code: 'X_JOB_TENANT_MISMATCH',
|
|
54
|
+
reason: `its tenant(input) answers ${org} whatever its input carries, so it cannot run in the fresh tenant the pass seeded for it`,
|
|
55
|
+
fix: `tenant: (input) => input.orgId # in job({ name: ${JSON.stringify(name)} })'s declaration: the org as input, so every enqueuer — this pass too — names it; then: x exercise ${renderFixShellArg(name, '<name>')} --json`,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const ANSWERED: ReadonlySet<string> = new Set(['completed', 'suspended']);
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* What a failed run hands the verdict. A run whose error carried a code is that code again, so
|
|
63
|
+
* core's `exerciseDefect` judges it by the one rule an action's refusal meets: a coded refusal
|
|
64
|
+
* under 500 is the job answering "no", a 5xx or an uncoded throw is a defect. The message alone
|
|
65
|
+
* read every coded refusal — an app's 409 included — as an uncoded throw: X_CONTRACT_DEFECT.
|
|
66
|
+
*/
|
|
67
|
+
const jobFailure = (run: JobExecution | undefined): unknown => {
|
|
68
|
+
if (run === undefined) return 'the job never ran';
|
|
69
|
+
const message = run.error ?? `the run ended ${run.outcome}`;
|
|
70
|
+
if (run.code === undefined) return message;
|
|
71
|
+
return new UltimateError({
|
|
72
|
+
code: run.code,
|
|
73
|
+
cause: message,
|
|
74
|
+
fix: `x errors explain ${renderFixShellArg(run.code, '<code>')} --json`,
|
|
75
|
+
});
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
const jobSubject = (handle: AnyJobHandle): Subject => ({
|
|
79
|
+
kind: 'job',
|
|
80
|
+
name: handle.name,
|
|
81
|
+
input: handle.input,
|
|
82
|
+
policy: undefined,
|
|
83
|
+
permissions: [],
|
|
84
|
+
handle,
|
|
85
|
+
seat: (input, tenant) => bindJobTenant(handle, input, tenant),
|
|
86
|
+
call: async (input, actor, tenant, jobs) => {
|
|
87
|
+
const queued = await jobs.enqueue(handle, input, { tenantId: tenant });
|
|
88
|
+
const trace = await jobs.drain(actor === null ? {} : { actor });
|
|
89
|
+
const own = trace.executions.find((run) => run.jobId === queued.id);
|
|
90
|
+
// The worker's failure is on the row, never a throw: the attempt's error and its code.
|
|
91
|
+
// Suspended is a body that ran and parked (`step.sleep`, `waitForEvent`): reached.
|
|
92
|
+
return own !== undefined && ANSWERED.has(own.outcome)
|
|
93
|
+
? { output: own.result }
|
|
94
|
+
: { failure: jobFailure(own) };
|
|
95
|
+
},
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
export async function subjectsOf(): Promise<readonly Subject[]> {
|
|
99
|
+
// On demand, as every fixture here loads them: the barrel stays off `action` and `query`.
|
|
100
|
+
const { invoke, listActions } = await import('@ultimat3/action');
|
|
101
|
+
const { listQueries, runQuery } = await import('@ultimat3/query');
|
|
102
|
+
return [
|
|
103
|
+
...listActions().map(
|
|
104
|
+
(target): Subject => ({
|
|
105
|
+
kind: 'action',
|
|
106
|
+
name: target.name,
|
|
107
|
+
input: target.input,
|
|
108
|
+
policy: target.policy,
|
|
109
|
+
permissions: target.describe().permissions,
|
|
110
|
+
call: async (input, actor) => ({
|
|
111
|
+
output: await withExerciseRequest(target.name, actor, () =>
|
|
112
|
+
invoke(target, input, { surface: 'http' }),
|
|
113
|
+
),
|
|
114
|
+
}),
|
|
115
|
+
}),
|
|
116
|
+
),
|
|
117
|
+
...listQueries().map(
|
|
118
|
+
(target): Subject => ({
|
|
119
|
+
kind: 'query',
|
|
120
|
+
name: target.name,
|
|
121
|
+
input: target.input,
|
|
122
|
+
policy: target.policy,
|
|
123
|
+
permissions: target.describe().permissions,
|
|
124
|
+
call: async (input, actor) => ({
|
|
125
|
+
output: await withExerciseRequest(target.name, actor, () =>
|
|
126
|
+
runQuery(target, input, { surface: 'http' }),
|
|
127
|
+
),
|
|
128
|
+
}),
|
|
129
|
+
}),
|
|
130
|
+
),
|
|
131
|
+
...registeredJobs().map(jobSubject),
|
|
132
|
+
];
|
|
133
|
+
}
|
package/src/exercise-sweeps.ts
CHANGED
|
@@ -3,8 +3,6 @@
|
|
|
3
3
|
// database). Both verdicts land on the record of the primitive that sent the statements.
|
|
4
4
|
|
|
5
5
|
import type { Actor } from '@ultimat3/core';
|
|
6
|
-
import type { Policy } from '@ultimat3/policy';
|
|
7
|
-
import type { StandardSchemaV1 } from '@ultimat3/schema';
|
|
8
6
|
import type { ExerciseOverrides } from './define-exercise';
|
|
9
7
|
import type { ActorRows, RoleGrants } from './exercise-actor';
|
|
10
8
|
import { asSeededRow, chooseActor } from './exercise-actor';
|
|
@@ -12,6 +10,8 @@ import type { Sampler } from './exercise-input';
|
|
|
12
10
|
import { bindIds, drawInput, seededIds } from './exercise-input';
|
|
13
11
|
import { overrideInput } from './exercise-override-input';
|
|
14
12
|
import type { ExerciseRecord } from './exercise-report';
|
|
13
|
+
import type { Subject } from './exercise-subjects';
|
|
14
|
+
import { seatIn } from './exercise-subjects';
|
|
15
15
|
import { sweepPlans } from './plan-sweep';
|
|
16
16
|
import type { ScaleSweepOptions } from './scale-sweep-run';
|
|
17
17
|
import { scaleSweep } from './scale-sweep-run';
|
|
@@ -37,13 +37,8 @@ export const actorRowsOf = (options: CallOptions, graph: SeedGraph): ActorRows |
|
|
|
37
37
|
export const seated = (actor: Actor | null, tenant: string, rows: ActorRows | undefined) =>
|
|
38
38
|
rows === undefined ? actor : asSeededRow(actor, tenant, rows);
|
|
39
39
|
|
|
40
|
-
/** A primitive as a sized call needs it: its schema and
|
|
41
|
-
|
|
42
|
-
readonly name: string;
|
|
43
|
-
readonly input: StandardSchemaV1;
|
|
44
|
-
readonly policy: Policy | undefined;
|
|
45
|
-
readonly permissions: readonly string[];
|
|
46
|
-
}
|
|
40
|
+
/** A primitive as a sized call needs it: its schema, its policy, and a job's seat. */
|
|
41
|
+
type Callable = Pick<Subject, 'name' | 'input' | 'policy' | 'permissions' | 'seat'>;
|
|
47
42
|
|
|
48
43
|
/** One call of `subject` in `tenant` of a sized graph: the control's actor and first input. */
|
|
49
44
|
export async function sizedCall(
|
|
@@ -67,7 +62,8 @@ export async function sizedCall(
|
|
|
67
62
|
const lookup = seededIds(graph, options.entities, { ...scope, self });
|
|
68
63
|
const drawn = bindIds(drawInput(sampler, subject.input, 0, 0), lookup);
|
|
69
64
|
const bindInput = overrideInput(subject.name, override);
|
|
70
|
-
|
|
65
|
+
const given = bindInput === undefined ? drawn : await bindInput(drawn, actor);
|
|
66
|
+
return { actor, input: seatIn(subject, given, tenant) };
|
|
71
67
|
}
|
|
72
68
|
|
|
73
69
|
/**
|