@ultimat3/testing 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 developerz.ai
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,130 @@
1
+ # @ultimat3/testing
2
+
3
+ The harness. Never mock the database — clone it. Never assert on wall-clock time — advance the
4
+ frozen clock. Never let a test reach the network unmocked — it fails by design.
5
+
6
+ ## What it owns
7
+
8
+ | Module | Owns |
9
+ |---|---|
10
+ | `harness.ts` | `describeApp()` / `testApp()` — boot an app in-process with its own database |
11
+ | `template-db.ts` | N workers, N databases, one migrated template, `CREATE DATABASE ... TEMPLATE` |
12
+ | `determinism.ts` | frozen clock, seeded RNG, seeded uuids, `assertDeterministic` |
13
+ | `sealed-network.ts` | any unmocked egress fails, with the URL and the mock line |
14
+ | `factories.ts` | typed factories from the entity registry, seeded |
15
+ | `test-types.ts` | the six test types and their helpers |
16
+ | `matchers.ts` | `toBeUltimateError` `toDenyPolicy` `toEmitSteps` `toMatchOpenApi` `toBeWithinBudget` `toRejectInput` |
17
+ | `fixtures.ts` | the registry + `test('…', ({ clock }) => …)` injection |
18
+ | `fixture-{clock,mail,jobs,network}.ts` | the four fixtures the framework builds in-process |
19
+ | `fixture-drivers.ts` | the five it declares but a driver must build — `page` `budget` `signIn` `deploy` `subscribe` |
20
+ | `framework-fixtures.ts` | registers both sets; the app registers only what it owns |
21
+ | `preload.ts` | the bunfig preload that installs all of the above |
22
+
23
+ ## Install
24
+
25
+ ```toml
26
+ # bunfig.toml
27
+ [test]
28
+ preload = ["@ultimat3/testing/preload"]
29
+ ```
30
+
31
+ ## Fixtures
32
+
33
+ `test` from this package passes a fixture bag as the first argument, and builds only what the
34
+ body destructures — a test that never names `runJobs` never starts a queue.
35
+
36
+ ```ts
37
+ import { expect, test } from '@ultimat3/testing';
38
+
39
+ test('the three-day sleep releases the worker', async ({ clock, runJobs }) => {
40
+ await runJobs(onboardOrg, { orgId });
41
+ expect(await runJobs.inFlight()).toBe(0); // suspended, not waiting
42
+ clock.advance('3d');
43
+ expect(await runJobs.due()).toBe(1);
44
+ });
45
+ ```
46
+
47
+ | Fixture | Is | Built by |
48
+ |---|---|---|
49
+ | `clock` | `now()` · `advance('3d')` · `set(instant)` on the frozen clock | the preload |
50
+ | `mail` | `outbox()` · `lastTo(address)` · `failOnce(mail)` over an in-memory transport | the preload |
51
+ | `network` | `offline()` · `drop()` · `online()` · `state()` over the sealed network | the preload |
52
+ | `runJobs` | a worker: call it to enqueue+drain, then `drain()` `due()` `inFlight()` `depth()` | the preload |
53
+ | `page` | the browser: `goto` `gotoStreamed` `getByRole` `evaluate` `waitForServiceWorker` | a browser driver |
54
+ | `budget` | `jsBytes(route)` measured off the built output | a browser driver |
55
+ | `signIn` | put the browser session in a member's shoes | a browser driver |
56
+ | `deploy` | `newBuild()` — same app, new build id, page still open | a browser driver |
57
+ | `subscribe` | one subscriber's `rows()` `patches()` `settled()` `lsn()` | a replicator |
58
+ | anything else | whatever the app registers | the app's `scripts/test-setup.ts` |
59
+
60
+ The last five are **declared but not built**: the name resolves, and destructuring one in a process
61
+ with no driver fails as `X_TEST_FIXTURE_UNAVAILABLE`, naming the driver rather than telling you to
62
+ register a fixture that is not yours to define. A driver arrives through the same registry —
63
+ `defineFixtures` merges, last registration wins — so there is no second seam to learn.
64
+
65
+ The declaration is also the driver's type: `defineFixtures` holds every name `Fixtures` declares to
66
+ the type it was declared with, so a half-built `page` is a compile error at the registration rather
67
+ than a missing method three awaits into a later test.
68
+
69
+ `mail`, `network` and `runJobs` install a process-global driver for the length of one test and hand the previous one back afterwards — the state they *found*, not a fixed default, so an outer fixture already offline stays offline when an inner one disposes. A fixture that takes over a global does the same: implement `Symbol.dispose` or `Symbol.asyncDispose` on what the factory returns, and `fixtureTest` calls it in reverse build order — including when the test body throws. Going offline is the `network` fixture's job and only its job; the gate's writer is not exported, because a test that set it directly would skip that disposal and take every later file down with it.
70
+
71
+ An app adds its own with `defineFixtures` and widens the type by augmenting `Fixtures`:
72
+
73
+ ```ts
74
+ defineFixtures({ seed: () => loadSeed, actorFor: () => actorFor });
75
+
76
+ declare module '@ultimat3/testing' {
77
+ interface Fixtures {
78
+ readonly seed: (name: string) => SeedHandle;
79
+ }
80
+ }
81
+ ```
82
+
83
+ Destructuring a name nobody registered fails with `X_TEST_FIXTURE_UNKNOWN`, which names the set
84
+ that *is* registered — never `undefined is not an object` from inside the body. A name that is
85
+ registered but has no driver fails with `X_TEST_FIXTURE_UNAVAILABLE` instead; the two are different
86
+ instructions, so they are different codes.
87
+
88
+ ## The six test types
89
+
90
+ | Helper | Asserts | `x verify` step |
91
+ |---|---|---|
92
+ | `unitTest` | pure logic, no I/O | `unit` |
93
+ | `contractTest` | OpenAPI diff vs the committed spec, MCP exposure | `contract` |
94
+ | `liveTest` | exactly what each subscriber receives | `live` |
95
+ | `jobTest` | step sequence, retries, idempotency | `job` |
96
+ | `e2eTest` | Playwright incl. offline mode + SW update | `e2e` |
97
+ | `evalTest` | LLM output scoring against a threshold | `eval` |
98
+
99
+ Each helper prefixes the test name with its type (`job · onboards an org`), which is what
100
+ `bun test --test-name-pattern "job · "` selects — the six lines of `x verify` come from the tests
101
+ themselves, not from a directory convention.
102
+
103
+ ## Parallel databases
104
+
105
+ ```ts
106
+ const db = await acquireWorkerDatabase({ adminUrl, migrate });
107
+ // worker 0 -> ultimate_test_template_w0
108
+ // worker 1 -> ultimate_test_template_w1
109
+ ```
110
+
111
+ The first worker creates the template under a Postgres advisory lock and migrates it once; every
112
+ worker then clones it copy-on-write. With no Postgres configured it falls back to PGlite, so
113
+ `bun test` works on a laptop with nothing installed.
114
+
115
+ ## Sealed network
116
+
117
+ ```
118
+ X_TEST_NETWORK_SEALED
119
+ cause: POST https://api.stripe.com/v1/charges was not mocked (allowed hosts: none)
120
+ fix: mockFetch('https://api.stripe.com/v1/charges', () => new Response('{}')) — or allowHost('api.stripe.com') if it must be real
121
+ ```
122
+
123
+ A server this process booted is exempt: `createServer().start()` announces its socket through
124
+ core's `markListening()`, so a test may call its own `handle.url()` on a kernel-assigned port with
125
+ the seal fully on. Unsealing (`ULTIMATE_TEST_ALLOW_NET=1`) stays reserved for a deliberate live
126
+ integration — never for a socket test.
127
+
128
+ ## Errors
129
+
130
+ `X_TEST_NETWORK_SEALED` `X_TEST_DB_UNAVAILABLE` `X_TEST_NONDETERMINISTIC` `X_TEST_FIXTURE_UNKNOWN`
package/package.json ADDED
@@ -0,0 +1,40 @@
1
+ {
2
+ "name": "@ultimat3/testing",
3
+ "version": "1.0.0",
4
+ "description": "Test harness: cloned template DBs per worker, frozen clock, sealed network, 6 test types",
5
+ "license": "MIT",
6
+ "type": "module",
7
+ "repository": {
8
+ "type": "git",
9
+ "url": "git+https://github.com/developerz-ai/ultimate.git",
10
+ "directory": "packages/testing"
11
+ },
12
+ "publishConfig": {
13
+ "access": "public",
14
+ "provenance": true
15
+ },
16
+ "exports": {
17
+ ".": "./src/index.ts",
18
+ "./preload": "./src/preload.ts"
19
+ },
20
+ "files": [
21
+ "src",
22
+ "!src/**/*.test.ts",
23
+ "README.md",
24
+ "LICENSE"
25
+ ],
26
+ "engines": {
27
+ "bun": ">=1.3.0"
28
+ },
29
+ "scripts": {
30
+ "typecheck": "tsc --noEmit -p tsconfig.json",
31
+ "test": "bun test"
32
+ },
33
+ "dependencies": {
34
+ "@ultimat3/core": "1.0.0",
35
+ "@ultimat3/db": "1.0.0",
36
+ "@ultimat3/jobs": "1.0.0",
37
+ "@ultimat3/mail": "1.0.0",
38
+ "@ultimat3/time": "1.0.0"
39
+ }
40
+ }
@@ -0,0 +1,133 @@
1
+ // Determinism, installed globally by the test preload. Anything nondeterministic in a test is a
2
+ // bug: a frozen clock means "advance it", a seeded RNG means "same values every run", and both are
3
+ // restorable so a test that genuinely needs real time can opt out explicitly.
4
+
5
+ import { NondeterministicError } from './errors';
6
+
7
+ export const DEFAULT_SEED = 20260101;
8
+ export const DEFAULT_NOW = '2026-01-01T00:00:00.000Z';
9
+
10
+ const RealDate = Date;
11
+ const realRandom = Math.random;
12
+ // Captured up front: the brand check below is only unforgeable while this is the engine's own
13
+ // `getTime` and not whatever a later assignment to `Date.prototype.getTime` left there.
14
+ const dateGetTime = RealDate.prototype.getTime;
15
+
16
+ let frozenAt = new RealDate(DEFAULT_NOW).getTime();
17
+ let installed = false;
18
+
19
+ /** mulberry32: 32 bits of state, uniform enough for tests, identical across platforms and runs. */
20
+ export function seededRandom(seed: number): () => number {
21
+ let state = seed >>> 0;
22
+ return () => {
23
+ state = (state + 0x6d2b79f5) >>> 0;
24
+ let t = state;
25
+ t = Math.imul(t ^ (t >>> 15), t | 1);
26
+ t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
27
+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
28
+ };
29
+ }
30
+
31
+ /** A seeded id generator, so factories produce stable uuids without a real RNG. */
32
+ export function seededUuid(seed: number): () => string {
33
+ const next = seededRandom(seed);
34
+ const hex = (count: number): string =>
35
+ Array.from({ length: count }, () => Math.floor(next() * 16).toString(16)).join('');
36
+ return () => `${hex(8)}-${hex(4)}-4${hex(3)}-a${hex(3)}-${hex(12)}`;
37
+ }
38
+
39
+ class FrozenDate extends RealDate {
40
+ constructor(...args: readonly unknown[]) {
41
+ if (args.length === 0) {
42
+ super(frozenAt);
43
+ return;
44
+ }
45
+ // Delegating with a spread keeps every real Date constructor overload working unchanged.
46
+ super(...(args as ConstructorParameters<typeof RealDate>));
47
+ }
48
+
49
+ static override now(): number {
50
+ return frozenAt;
51
+ }
52
+
53
+ /**
54
+ * While the harness is installed `globalThis.Date` is this subclass, so `value instanceof Date`
55
+ * asks "is it a FrozenDate" — and a Date the runtime built for itself is not one: a timestamptz
56
+ * off a Postgres socket, a `structuredClone`, anything from another realm. Every
57
+ * `value instanceof Date` guard in the framework would then reject a real Date under test and
58
+ * nowhere else, which is the worst place a difference can be. Freezing the clock must not change
59
+ * what a Date *is*, so answer for the internal slot rather than for a prototype chain.
60
+ *
61
+ * `value instanceof RealDate` is not that answer: `instanceof` is per-realm, so a Date built in
62
+ * a `node:vm` context or a worker — the "another realm" case above — still comes back false. A
63
+ * `[object Date]` from `Object.prototype.toString` is not it either: any object carrying
64
+ * `Symbol.toStringTag: 'Date'` passes that. `getTime` throws for anything without a
65
+ * `[[DateValue]]` slot, and that slot is the one thing neither a fake nor a realm can hide.
66
+ */
67
+ static override [Symbol.hasInstance](value: unknown): boolean {
68
+ try {
69
+ dateGetTime.call(value);
70
+ return true;
71
+ } catch {
72
+ return false;
73
+ }
74
+ }
75
+ }
76
+
77
+ export interface DeterminismOptions {
78
+ readonly seed?: number;
79
+ readonly now?: string | number;
80
+ }
81
+
82
+ /** Install the frozen clock and the seeded RNG globally. Idempotent. */
83
+ export function installDeterminism(options: DeterminismOptions = {}): void {
84
+ frozenAt = new RealDate(options.now ?? DEFAULT_NOW).getTime();
85
+ const next = seededRandom(options.seed ?? DEFAULT_SEED);
86
+ globalThis.Date = FrozenDate as unknown as DateConstructor;
87
+ Math.random = next;
88
+ installed = true;
89
+ }
90
+
91
+ export function restoreDeterminism(): void {
92
+ globalThis.Date = RealDate;
93
+ Math.random = realRandom;
94
+ installed = false;
95
+ }
96
+
97
+ export const isDeterminismInstalled = (): boolean => installed;
98
+
99
+ /** Move the frozen clock forward. The only legal way for time to pass inside a test. */
100
+ export function advanceClock(ms: number): Date {
101
+ frozenAt += ms;
102
+ return new RealDate(frozenAt);
103
+ }
104
+
105
+ export const frozenNow = (): Date => new RealDate(frozenAt);
106
+
107
+ export function setFrozenClock(now: string | number): void {
108
+ frozenAt = new RealDate(now).getTime();
109
+ }
110
+
111
+ /** Run `body` with the clock frozen at `now`, then restore whatever was there before. */
112
+ export async function frozenClock<T>(now: string, body: () => T | Promise<T>): Promise<T> {
113
+ const previous = frozenAt;
114
+ frozenAt = new RealDate(now).getTime();
115
+ try {
116
+ return await body();
117
+ } finally {
118
+ frozenAt = previous;
119
+ }
120
+ }
121
+
122
+ /**
123
+ * Run a body twice and fail if the results differ. Used by the harness to catch the tests that
124
+ * only pass because they ran first.
125
+ */
126
+ export async function assertDeterministic<T>(what: string, body: () => T | Promise<T>): Promise<T> {
127
+ const first = await body();
128
+ const second = await body();
129
+ const a = JSON.stringify(first);
130
+ const b = JSON.stringify(second);
131
+ if (a !== b) throw new NondeterministicError({ what, first: a, second: b });
132
+ return first;
133
+ }
package/src/errors.ts ADDED
@@ -0,0 +1,202 @@
1
+ // The X_* codes owned by @ultimat3/testing. A test failure has to be as actionable as a runtime
2
+ // failure — the fix line here is the mock to add, the service to start, or the seed to freeze.
3
+ import { registerErrorCodes, UltimateError } from '@ultimat3/core';
4
+
5
+ export const TESTING_ERROR_CODES = [
6
+ 'X_TEST_NETWORK_SEALED',
7
+ 'X_TEST_NETWORK_OFFLINE',
8
+ 'X_TEST_DB_UNAVAILABLE',
9
+ 'X_TEST_NONDETERMINISTIC',
10
+ 'X_TEST_FIXTURE_UNKNOWN',
11
+ 'X_TEST_FIXTURE_UNAVAILABLE',
12
+ 'X_TEST_EVAL_THRESHOLD',
13
+ 'X_TEST_SCHEMA_EXPECTED',
14
+ 'X_TEST_JOB_EXPECTED',
15
+ 'X_TEST_NETWORK_RACE',
16
+ ] as const;
17
+
18
+ export type TestingErrorCode = (typeof TESTING_ERROR_CODES)[number];
19
+
20
+ export const TESTING_ERROR_TITLES: Readonly<Record<TestingErrorCode, string>> = {
21
+ X_TEST_NETWORK_SEALED: 'a test tried to reach the network',
22
+ X_TEST_NETWORK_OFFLINE: 'the test network is offline',
23
+ X_TEST_DB_UNAVAILABLE: 'no Postgres for the test template',
24
+ X_TEST_NONDETERMINISTIC: 'a test read wall-clock time or unseeded randomness',
25
+ X_TEST_FIXTURE_UNKNOWN: 'a test requested a fixture nobody registered',
26
+ X_TEST_FIXTURE_UNAVAILABLE: 'a declared fixture has no driver in this process',
27
+ X_TEST_EVAL_THRESHOLD: 'an evalTest() score fell below its threshold',
28
+ X_TEST_SCHEMA_EXPECTED: 'a matcher expected a Standard Schema and got something else',
29
+ X_TEST_JOB_EXPECTED: 'a matcher expected a job declaration and got something else',
30
+ X_TEST_NETWORK_RACE: 'a request raced unsealNetwork() and lost the patched fetch',
31
+ };
32
+
33
+ // Titles must be registered for `format()` to render the contract's first line. Every code above is
34
+ // owned here and none is borrowed, so the call is unconditional: a second package claiming one has
35
+ // to fail as X_ERROR_CODE_DUPLICATE, not quietly keep whichever title was registered first.
36
+ registerErrorCodes(
37
+ Object.fromEntries(
38
+ Object.entries(TESTING_ERROR_TITLES).map(([code, title]) => [code, { title }]),
39
+ ),
40
+ );
41
+
42
+ const docsFor = (code: TestingErrorCode): string => `https://ultimate.dev/errors/${code}`;
43
+
44
+ /** A test reached the network without a mock or an allowlist entry. Always a bug, never a flake. */
45
+ export class NetworkSealedError extends UltimateError {
46
+ constructor(input: { url: string; method: string; allowed: readonly string[] }) {
47
+ super({
48
+ code: 'X_TEST_NETWORK_SEALED',
49
+ cause: `${input.method} ${input.url} was not mocked (allowed hosts: ${
50
+ input.allowed.length > 0 ? input.allowed.join(', ') : 'none'
51
+ })`,
52
+ fix: `mockFetch('${input.url}', () => new Response('{}')) — or allowHost('${hostOf(input.url)}') if it must be real`,
53
+ docs: docsFor('X_TEST_NETWORK_SEALED'),
54
+ });
55
+ }
56
+ }
57
+
58
+ /** No Postgres and no PGlite: the harness cannot give the test a database to clone. */
59
+ export class TestDatabaseUnavailableError extends UltimateError {
60
+ constructor(input: { cause: string }) {
61
+ super({
62
+ code: 'X_TEST_DB_UNAVAILABLE',
63
+ cause: input.cause,
64
+ fix: 'x dev (embedded Postgres), or set TEST_DATABASE_URL to a running Postgres',
65
+ docs: docsFor('X_TEST_DB_UNAVAILABLE'),
66
+ });
67
+ }
68
+ }
69
+
70
+ /** Two runs of the same test produced different values. The seed or the clock is not frozen. */
71
+ export class NondeterministicError extends UltimateError {
72
+ constructor(input: { what: string; first: string; second: string }) {
73
+ super({
74
+ code: 'X_TEST_NONDETERMINISTIC',
75
+ cause: `${input.what} produced "${input.first}" then "${input.second}"`,
76
+ fix: 'wrap the test in frozenClock() / seededRandom(), or remove the wall-clock read',
77
+ docs: docsFor('X_TEST_NONDETERMINISTIC'),
78
+ });
79
+ }
80
+ }
81
+
82
+ export function hostOf(url: string): string {
83
+ try {
84
+ return new URL(url).host;
85
+ } catch {
86
+ return url;
87
+ }
88
+ }
89
+
90
+ /**
91
+ * A test destructured a fixture nobody registered. Naming the registered set matters: the
92
+ * failure would otherwise surface as `undefined is not an object` deep inside the body,
93
+ * pointing at the use rather than the missing registration.
94
+ */
95
+ export class FixtureUnknownError extends UltimateError {
96
+ constructor(input: { name: string; registered: readonly string[] }) {
97
+ super({
98
+ code: 'X_TEST_FIXTURE_UNKNOWN',
99
+ cause:
100
+ input.registered.length === 0
101
+ ? `test requested fixture "${input.name}" but none are registered`
102
+ : `test requested fixture "${input.name}"; registered: ${input.registered.join(', ')}`,
103
+ fix: `register it at test setup: defineFixtures({ ${input.name}: () => buildIt() })`,
104
+ docs: docsFor('X_TEST_FIXTURE_UNKNOWN'),
105
+ });
106
+ }
107
+ }
108
+
109
+ export const fixtureUnknown = (name: string, registered: readonly string[]): UltimateError =>
110
+ new FixtureUnknownError({ name, registered });
111
+
112
+ /**
113
+ * Different failure from `X_TEST_FIXTURE_UNKNOWN`, and the distinction is the whole point: the
114
+ * name IS registered, so "register it" is the wrong instruction. What is missing is the driver
115
+ * underneath — a browser, a replicator — which the framework declares but deliberately does not
116
+ * bundle. Naming what it needs turns "undefined is not an object" into a decision the reader can
117
+ * make: install the driver, or stop asking for the fixture.
118
+ */
119
+ export class FixtureUnavailableError extends UltimateError {
120
+ constructor(input: { name: string; needs: string }) {
121
+ super({
122
+ code: 'X_TEST_FIXTURE_UNAVAILABLE',
123
+ cause: `fixture "${input.name}" is declared but nothing in this process drives it — it needs ${input.needs}`,
124
+ fix: `install one in the test preload: defineFixtures({ ${input.name}: () => yourDriver() })`,
125
+ docs: docsFor('X_TEST_FIXTURE_UNAVAILABLE'),
126
+ });
127
+ }
128
+ }
129
+
130
+ export const fixtureUnavailable = (name: string, needs: string): UltimateError =>
131
+ new FixtureUnavailableError({ name, needs });
132
+
133
+ /**
134
+ * A request made while `network.offline()` (or `network.drop()`) is in force. Coded rather than a
135
+ * bare `TypeError` because a test that lands here uncaught needs to know which of the two it was:
136
+ * the app's offline path not running, or a fixture left offline by the test before it.
137
+ */
138
+ export class NetworkOfflineError extends UltimateError {
139
+ constructor(input: { url: string; method: string; mode: 'offline' | 'dropped' }) {
140
+ super({
141
+ code: 'X_TEST_NETWORK_OFFLINE',
142
+ cause: `${input.method} ${input.url} while the test network is ${input.mode}`,
143
+ fix: 'network.online() before the call — or assert the offline path instead of the request',
144
+ docs: docsFor('X_TEST_NETWORK_OFFLINE'),
145
+ });
146
+ }
147
+ }
148
+
149
+ /** `evalTest()`'s score fell below its declared threshold. A test failure, not a warning. */
150
+ export class TestEvalThresholdError extends UltimateError {
151
+ constructor(input: { name: string; threshold: number; detail: string }) {
152
+ super({
153
+ code: 'X_TEST_EVAL_THRESHOLD',
154
+ cause: `eval "${input.name}" scored below ${input.threshold}: ${input.detail}`,
155
+ fix: 'improve the prompt under test, or lower the threshold passed to evalTest()',
156
+ docs: docsFor('X_TEST_EVAL_THRESHOLD'),
157
+ });
158
+ }
159
+ }
160
+
161
+ /** `toRejectInput`/`toAcceptInput` were handed something other than a Standard Schema. */
162
+ export class TestSchemaExpectedError extends UltimateError {
163
+ constructor() {
164
+ super({
165
+ code: 'X_TEST_SCHEMA_EXPECTED',
166
+ cause: 'toRejectInput/toAcceptInput expect a Standard Schema (`t`), not the action',
167
+ // Names the call, not the intent: "assert against action.input" left the reader to work out
168
+ // which call to edit, and a fix is only executable if it can be pasted over the failing line.
169
+ fix: 'call toRejectInput(action.input) — the schema, not toRejectInput(action) or the query',
170
+ docs: docsFor('X_TEST_SCHEMA_EXPECTED'),
171
+ });
172
+ }
173
+ }
174
+
175
+ /** `toEmitSteps`/`recordSteps` were handed something other than a job declaration. */
176
+ export class TestJobExpectedError extends UltimateError {
177
+ constructor() {
178
+ super({
179
+ code: 'X_TEST_JOB_EXPECTED',
180
+ cause: 'toEmitSteps expects a job declaration built with job(...)',
181
+ // Same rule as X_TEST_SCHEMA_EXPECTED's: the paste-able call, not a description of it.
182
+ fix: 'call toEmitSteps(myJob) with the job export, not toEmitSteps(myJob.run)',
183
+ docs: docsFor('X_TEST_JOB_EXPECTED'),
184
+ });
185
+ }
186
+ }
187
+
188
+ /**
189
+ * `sealNetwork()` always sets the original `fetch` before installing its patch, so this can only
190
+ * fire if `unsealNetwork()` ran concurrently with a request from the same seal — a race, not a
191
+ * reachable steady state.
192
+ */
193
+ export class NetworkRaceError extends UltimateError {
194
+ constructor() {
195
+ super({
196
+ code: 'X_TEST_NETWORK_RACE',
197
+ cause: 'sealed network lost its original fetch mid-request',
198
+ fix: 'do not call unsealNetwork() while a request from the same test is still in flight',
199
+ docs: docsFor('X_TEST_NETWORK_RACE'),
200
+ });
201
+ }
202
+ }
@@ -0,0 +1,99 @@
1
+ // Typed factories derived from the entity registry. Rows come from the entity's own columns, so a
2
+ // new NOT NULL column breaks the factory at compile time instead of at the first insert — and the
3
+ // values are seeded, so two runs of the same suite produce byte-identical rows.
4
+
5
+ import { seededRandom, seededUuid } from './determinism';
6
+
7
+ export interface EntityLike {
8
+ readonly kind: 'entity';
9
+ readonly table: string;
10
+ readonly columns: Readonly<Record<string, unknown>>;
11
+ }
12
+
13
+ export interface Factory<TRow> {
14
+ readonly table: string;
15
+ build(over?: Partial<TRow>): TRow;
16
+ buildMany(count: number, over?: Partial<TRow>): readonly TRow[];
17
+ /** Restart the sequence so a second describe block sees the same ids as the first. */
18
+ reset(): void;
19
+ }
20
+
21
+ export interface FactoryOptions<TRow> {
22
+ readonly seed?: number;
23
+ /** Values for every column the entity requires; called once per built row. */
24
+ defaults(index: number, ids: { uuid(): string; number(): number }): TRow;
25
+ }
26
+
27
+ export function defineFactory<TRow extends object>(
28
+ entity: EntityLike,
29
+ options: FactoryOptions<TRow>,
30
+ ): Factory<TRow> {
31
+ const seed = options.seed ?? 1;
32
+ let uuid = seededUuid(seed);
33
+ let random = seededRandom(seed);
34
+ let index = 0;
35
+ const ids = {
36
+ uuid: () => uuid(),
37
+ number: () => Math.floor(random() * 1_000_000),
38
+ };
39
+ return {
40
+ table: entity.table,
41
+ build: (over = {}) => {
42
+ index += 1;
43
+ return { ...options.defaults(index, ids), ...over };
44
+ },
45
+ buildMany: (count, over = {}) =>
46
+ Array.from({ length: count }, () => {
47
+ index += 1;
48
+ return { ...options.defaults(index, ids), ...over };
49
+ }),
50
+ reset: () => {
51
+ uuid = seededUuid(seed);
52
+ random = seededRandom(seed);
53
+ index = 0;
54
+ },
55
+ };
56
+ }
57
+
58
+ export type EntityRegistry = Readonly<Record<string, EntityLike>>;
59
+
60
+ export type FactoryRegistry<TRegistry extends EntityRegistry> = {
61
+ readonly [K in keyof TRegistry]: Factory<Record<string, unknown>>;
62
+ };
63
+
64
+ /**
65
+ * Build one factory per registered entity, with column-name-driven defaults. Enough for the rows a
66
+ * test does not care about; pass `defineFactory` explicitly for the rows it does.
67
+ */
68
+ export function factoriesFor<TRegistry extends EntityRegistry>(
69
+ registry: TRegistry,
70
+ seed = 1,
71
+ ): FactoryRegistry<TRegistry> {
72
+ const out: Record<string, Factory<Record<string, unknown>>> = {};
73
+ for (const [name, entity] of Object.entries(registry)) {
74
+ out[name] = defineFactory<Record<string, unknown>>(entity, {
75
+ seed,
76
+ defaults: (index, ids) => {
77
+ const row: Record<string, unknown> = {};
78
+ for (const column of Object.keys(entity.columns)) {
79
+ row[column] = defaultFor(column, index, ids);
80
+ }
81
+ return row;
82
+ },
83
+ });
84
+ }
85
+ return out as FactoryRegistry<TRegistry>;
86
+ }
87
+
88
+ function defaultFor(
89
+ column: string,
90
+ index: number,
91
+ ids: { uuid(): string; number(): number },
92
+ ): unknown {
93
+ if (column === 'id' || column.endsWith('Id')) return ids.uuid();
94
+ if (column.endsWith('At')) return new Date(0);
95
+ if (column.endsWith('Minor')) return ids.number();
96
+ if (column.endsWith('Currency')) return 'USD';
97
+ if (column.startsWith('is') || column.startsWith('has')) return false;
98
+ return `${column}-${index}`;
99
+ }
@@ -0,0 +1,28 @@
1
+ // The `clock` fixture: the only legal way for time to pass inside a test.
2
+ //
3
+ // `clock.advance('3d')` is synchronous — a job test asserts on what is due on the very next
4
+ // line — so the duration parser is resolved while the fixture is built, not when it is used.
5
+
6
+ import { advanceClock, frozenNow, setFrozenClock } from './determinism';
7
+
8
+ /** `'3d'` | `'30s'` | `1500`. Same vocabulary as a job's `timeout` and a step's `sleep`. */
9
+ export type TestDuration = string | number;
10
+
11
+ export interface TestClock {
12
+ /** The frozen instant. Never the wall clock. */
13
+ now(): Date;
14
+ advance(duration: TestDuration): Date;
15
+ set(instant: string | number): Date;
16
+ }
17
+
18
+ export async function createTestClock(): Promise<TestClock> {
19
+ const { toMs } = await import('@ultimat3/time');
20
+ return {
21
+ now: frozenNow,
22
+ advance: (duration) => advanceClock(toMs(duration)),
23
+ set: (instant) => {
24
+ setFrozenClock(instant);
25
+ return frozenNow();
26
+ },
27
+ };
28
+ }