@ultimat3/testing 30.4.0 → 30.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +2 -2
- package/package.json +17 -17
- package/src/errors.ts +4 -2
- package/src/exercise-crash.ts +1 -0
- package/src/exercise-defect.ts +71 -0
- package/src/exercise-report.ts +6 -1
- package/src/exercise.ts +15 -1
- package/src/index.ts +3 -0
- package/src/job-crash-proof.ts +28 -4
- package/src/job-sim-effects.ts +7 -2
- package/src/job-sim-errors.ts +36 -5
- package/src/job-sim-world.ts +4 -1
- package/src/mock-route.ts +21 -0
- package/src/sealed-dns.ts +86 -0
- package/src/sealed-network.ts +48 -7
- package/src/seed-graph-values.ts +8 -3
package/CLAUDE.md
CHANGED
|
@@ -16,7 +16,7 @@ import is `registry-isolation.ts`, own entry point, off the barrel. Full rule: `
|
|
|
16
16
|
| No mocks of the DB | clone a template database; `template-db.ts` is the only DB path |
|
|
17
17
|
| No wall clock | `withFrozenClock` / `advanceClock`; `Date.now()` is frozen by the preload |
|
|
18
18
|
| Frozen ≠ different | `globalThis.Date` becomes a subclass; `FrozenDate[Symbol.hasInstance]` brands on the `[[DateValue]]` slot (`Date.prototype.getTime.call(value)` throws or not) |
|
|
19
|
-
| No unmocked egress | `sealed-network.ts` seals `fetch`; `sealed-sockets.ts` `WebSocket`, `Bun.connect` (not loopback); a miss is `X_TEST_NETWORK_SEALED` |
|
|
19
|
+
| No unmocked egress | `sealed-network.ts` seals `fetch`; `sealed-sockets.ts` `WebSocket`, `Bun.connect` (not loopback); `sealed-dns.ts` the three `lookup`s (a mocked host answers `MOCKED_HOST_ADDRESS`); a miss is `X_TEST_NETWORK_SEALED` |
|
|
20
20
|
| One `mockFetch`, and it grows | a verb in the match, ordered answers, a pinned dial's name (`mock-route.ts`) and `requestLog` (`request-log.ts`). A stream bound for a real server logs `null` (teeing drains it). `original(input, init)` goes as handed: `tls`, `unix`, `proxy` are in no `Request`. `cassette()`: below |
|
|
21
21
|
| Self is not egress | a port core's `markListening()` announced passes through — a socket test never unseals |
|
|
22
22
|
| Offline is a state, not a mock | `network.offline()` / `.drop()` fail every dial, loopback too, as `X_TEST_NETWORK_OFFLINE` before any mock: the app's offline path runs |
|
|
@@ -49,7 +49,7 @@ import is `registry-isolation.ts`, own entry point, off the barrel. Full rule: `
|
|
|
49
49
|
| The exercise pass runs what is registered | `exercise.ts` (`runExercisePass`): every action, query and job once, its own tenant pair off one `seedGraph`, an actor `exercise-actor.ts` picks, 39's `runExercise` as the verdict. Rows COMMITTED. Overrides only via `defineExercise` (blank `off` refused) |
|
|
50
50
|
| An override `input` is awaited, as the call's actor | `exercise-override-input.ts`: run inside the call's own request, so `useContext().actor` is the seated actor; the one way to a signed input. `async` ones are left out of `x drill`'s load |
|
|
51
51
|
| A passkey test signs for real | `softAuthenticator` is `@ultimat3/auth/soft-authenticator`'s, never re-exported here (one value, one path: `flight-copies`) and never a second one (auth's own suites need it, and auth cannot import testing) |
|
|
52
|
-
| A reached job is crash-proven | `exercise-crash.ts`: `crashProof` with the control's input and actor, after the tenant proof; only `diverged` is red (`X_JOB_CRASH_DIVERGED`). Committed: job bodies escape transactions |
|
|
52
|
+
| A reached job is crash-proven | `exercise-crash.ts`: `crashProof` with the control's input and actor, after the tenant proof; only `diverged` is red (`X_JOB_CRASH_DIVERGED`), its place `crashEffectPlace` (`inside` a step via jobs' `currentStep()`, or `after` one). Committed: job bodies escape transactions |
|
|
53
53
|
| The tenant proof is taint only | `tenant-proof.ts`: B, A, B; B is A's role minus `tenancy:cross`. Leak = seeded canary or unhanded id (`tenant-canary.ts`); write = moved digest. Only `crossed` is red |
|
|
54
54
|
| Every statement the pass sent is planned | `plan-sweep.ts` (`sweepPlans`): first relation an app table, `PREPARE` in a READ-ONLY transaction with seq/bitmap scans and sorts off, judged by `plan-oracle.ts`; red = `X_QUERY_UNINDEXED` / `X_QUERY_SORT_UNSERVED` with the `indexes:` entry (`plan-sweep-fix.ts`). `scan:` or `expectedSeqScan` is exempt, listed; credited: a partial unique whose predicate the statement states (one row), a declared equality-then-range index under a limit; no generic plan is `X_QUERY_PLAN_UNAVAILABLE`, never a pass |
|
|
55
55
|
| A shape that grows with the rows is red | `scale-sweep-run.ts`: two more tenants at `SCALE_SIZES` (2, 8 rows a table), every reached primitive once in each; `judgeScale` (pure) reds `X_QUERY_SCALES_WITH_ROWS`. Under `expectedQueryLoop` it is listed with its reason |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/testing",
|
|
3
|
-
"version": "30.
|
|
3
|
+
"version": "30.5.0",
|
|
4
4
|
"description": "Test harness: cloned template DBs per worker, frozen clock, sealed network, 6 test types",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ultimate",
|
|
@@ -46,21 +46,21 @@
|
|
|
46
46
|
"test": "bun test"
|
|
47
47
|
},
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@ultimat3/action": "30.
|
|
50
|
-
"@ultimat3/cache": "30.
|
|
51
|
-
"@ultimat3/core": "30.
|
|
52
|
-
"@ultimat3/db": "30.
|
|
53
|
-
"@ultimat3/entity": "30.
|
|
54
|
-
"@ultimat3/http": "30.
|
|
55
|
-
"@ultimat3/i18n": "30.
|
|
56
|
-
"@ultimat3/jobs": "30.
|
|
57
|
-
"@ultimat3/mail": "30.
|
|
58
|
-
"@ultimat3/policy": "30.
|
|
59
|
-
"@ultimat3/pwa": "30.
|
|
60
|
-
"@ultimat3/query": "30.
|
|
61
|
-
"@ultimat3/realtime": "30.
|
|
62
|
-
"@ultimat3/render": "30.
|
|
63
|
-
"@ultimat3/schema": "30.
|
|
64
|
-
"@ultimat3/time": "30.
|
|
49
|
+
"@ultimat3/action": "30.5.0",
|
|
50
|
+
"@ultimat3/cache": "30.5.0",
|
|
51
|
+
"@ultimat3/core": "30.5.0",
|
|
52
|
+
"@ultimat3/db": "30.5.0",
|
|
53
|
+
"@ultimat3/entity": "30.5.0",
|
|
54
|
+
"@ultimat3/http": "30.5.0",
|
|
55
|
+
"@ultimat3/i18n": "30.5.0",
|
|
56
|
+
"@ultimat3/jobs": "30.5.0",
|
|
57
|
+
"@ultimat3/mail": "30.5.0",
|
|
58
|
+
"@ultimat3/policy": "30.5.0",
|
|
59
|
+
"@ultimat3/pwa": "30.5.0",
|
|
60
|
+
"@ultimat3/query": "30.5.0",
|
|
61
|
+
"@ultimat3/realtime": "30.5.0",
|
|
62
|
+
"@ultimat3/render": "30.5.0",
|
|
63
|
+
"@ultimat3/schema": "30.5.0",
|
|
64
|
+
"@ultimat3/time": "30.5.0"
|
|
65
65
|
}
|
|
66
66
|
}
|
package/src/errors.ts
CHANGED
|
@@ -163,7 +163,7 @@ export class NetworkSealedError extends UltimateError {
|
|
|
163
163
|
url: string;
|
|
164
164
|
method: string;
|
|
165
165
|
allowed: readonly string[];
|
|
166
|
-
transport?: 'fetch' | 'socket' | undefined;
|
|
166
|
+
transport?: 'fetch' | 'socket' | 'dns' | undefined;
|
|
167
167
|
}) {
|
|
168
168
|
const allowCall = `allowHost('${hostOf(input.url)}')`;
|
|
169
169
|
const allow = `${allowCall} if it must be real`;
|
|
@@ -175,7 +175,9 @@ export class NetworkSealedError extends UltimateError {
|
|
|
175
175
|
fix:
|
|
176
176
|
input.transport === 'socket'
|
|
177
177
|
? `${allowCall} # only if it must be real — otherwise inject the transport: pass the code under test its \`connect\`/\`client\` option, or dial a Bun.listen server on 127.0.0.1 (loopback is never sealed)`
|
|
178
|
-
:
|
|
178
|
+
: input.transport === 'dns'
|
|
179
|
+
? `mockFetch('https://${hostOf(input.url)}/*', () => new Response('{}')) # a mocked host resolves — or ${allow}`
|
|
180
|
+
: `mockFetch('${input.url}', () => new Response('{}')) — or ${allow}`,
|
|
179
181
|
});
|
|
180
182
|
}
|
|
181
183
|
}
|
package/src/exercise-crash.ts
CHANGED
|
@@ -18,6 +18,7 @@ const findingOf = (divergence: CrashDivergence): CrashFinding => {
|
|
|
18
18
|
clean: divergence.clean,
|
|
19
19
|
killed: divergence.killed,
|
|
20
20
|
...(typeof after === 'string' ? { after } : {}),
|
|
21
|
+
...(divergence.inside === undefined ? {} : { inside: divergence.inside }),
|
|
21
22
|
};
|
|
22
23
|
};
|
|
23
24
|
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// What a 5xx tells the author beyond its code: the coded error's cause, the statement or the fields
|
|
2
|
+
// it names, every value the call's input held under a credential key or a `.sensitive()` field
|
|
3
|
+
// masked out of the text. Without it, every X_DB_STATEMENT_FAILED in the pass was rerun by hand.
|
|
4
|
+
|
|
5
|
+
import { captureValue, isUltimateError, REDACTED, stringField } from '@ultimat3/core';
|
|
6
|
+
import type { StandardSchemaV1 } from '@ultimat3/schema';
|
|
7
|
+
|
|
8
|
+
/** What `ExerciseRecord.defect` holds. */
|
|
9
|
+
export interface ExerciseDefect {
|
|
10
|
+
/** The error's own cause, credential values masked, cut at `MAX_DEFECT_CAUSE`. */
|
|
11
|
+
readonly cause: string;
|
|
12
|
+
/** The SQL a database refusal names (`… — statement: <sql>`), parameters as `$n`. */
|
|
13
|
+
readonly statement?: string;
|
|
14
|
+
/** The paths a validation failure names (`items.0.sku`). */
|
|
15
|
+
readonly fields?: readonly string[];
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** A cause long enough for a SQLSTATE, its message and a capped statement; past it, cut. */
|
|
19
|
+
export const MAX_DEFECT_CAUSE = 600;
|
|
20
|
+
|
|
21
|
+
const STATEMENT = / — statement: (.+)$/s;
|
|
22
|
+
|
|
23
|
+
/** Every scalar the masked copy replaced: what must not appear in the cause either. */
|
|
24
|
+
function maskedScalars(original: unknown, masked: unknown, into: string[]): void {
|
|
25
|
+
if (masked === REDACTED) {
|
|
26
|
+
if (typeof original === 'string' || typeof original === 'number') into.push(String(original));
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
if (typeof original !== 'object' || original === null) return;
|
|
30
|
+
if (typeof masked !== 'object' || masked === null) return;
|
|
31
|
+
for (const [key, entry] of Object.entries(original)) {
|
|
32
|
+
maskedScalars(entry, Reflect.get(masked, key), into);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const fieldsOf = (meta: Readonly<Record<string, unknown>> | undefined): readonly string[] => {
|
|
37
|
+
const issues = meta?.['issues'];
|
|
38
|
+
if (!Array.isArray(issues)) return [];
|
|
39
|
+
return issues.flatMap((issue: unknown) => {
|
|
40
|
+
const path = stringField(issue, 'path');
|
|
41
|
+
return path === undefined || path === '' ? [] : [path];
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* The detail of a CODED failure, or `undefined` for an uncoded throw — whose reason already is the
|
|
47
|
+
* throw. `schema` is the primitive's input: its `.sensitive()` fields are masked as `redactKeys`
|
|
48
|
+
* ones are (core's `captureValue`), so the record holds no value the audit log would not.
|
|
49
|
+
*/
|
|
50
|
+
export async function defectDetail(
|
|
51
|
+
error: unknown,
|
|
52
|
+
input: unknown,
|
|
53
|
+
schema: StandardSchemaV1,
|
|
54
|
+
): Promise<ExerciseDefect | undefined> {
|
|
55
|
+
if (!isUltimateError(error)) return undefined;
|
|
56
|
+
// On demand: `@ultimat3/schema` is a dynamic-only dependency of this barrel (`CLAUDE.md`).
|
|
57
|
+
const { maskSensitive } = await import('@ultimat3/schema');
|
|
58
|
+
const secrets: string[] = [];
|
|
59
|
+
maskedScalars(input, captureValue(maskSensitive(schema, input, REDACTED)).value, secrets);
|
|
60
|
+
let cause = error.cause;
|
|
61
|
+
for (const value of [...new Set(secrets)].sort((a, b) => b.length - a.length)) {
|
|
62
|
+
if (value !== '') cause = cause.split(value).join(REDACTED);
|
|
63
|
+
}
|
|
64
|
+
const statement = STATEMENT.exec(cause)?.[1]?.trim();
|
|
65
|
+
const fields = fieldsOf(error.meta);
|
|
66
|
+
return {
|
|
67
|
+
cause: cause.length > MAX_DEFECT_CAUSE ? `${cause.slice(0, MAX_DEFECT_CAUSE - 1)}…` : cause,
|
|
68
|
+
...(statement === undefined || statement === '' ? {} : { statement }),
|
|
69
|
+
...(fields.length === 0 ? {} : { fields }),
|
|
70
|
+
};
|
|
71
|
+
}
|
package/src/exercise-report.ts
CHANGED
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
} from '@ultimat3/db';
|
|
13
13
|
import type { RowObserver } from '@ultimat3/entity';
|
|
14
14
|
import { captureDeterminism, installDeterminism, restoreCapturedDeterminism } from './determinism';
|
|
15
|
+
import type { ExerciseDefect } from './exercise-defect';
|
|
15
16
|
import { testMail } from './fixture-mail';
|
|
16
17
|
import { routeMatches } from './mock-route';
|
|
17
18
|
import type { PlanProof, SentStatement } from './plan-sweep';
|
|
@@ -72,8 +73,10 @@ export interface CrashFinding {
|
|
|
72
73
|
readonly effect: string;
|
|
73
74
|
readonly clean: number;
|
|
74
75
|
readonly killed: number;
|
|
75
|
-
/** The last step the clean run had
|
|
76
|
+
/** The last step the clean run had completed before the effect; absent, before the first. */
|
|
76
77
|
readonly after?: string;
|
|
78
|
+
/** The step whose body made the effect, when it was made inside one: it runs again on a retry. */
|
|
79
|
+
readonly inside?: string;
|
|
77
80
|
}
|
|
78
81
|
|
|
79
82
|
/** The crash proof's verdict on a reached job: killed at every step boundary (`crashProof`). */
|
|
@@ -95,6 +98,8 @@ export interface ExerciseRecord {
|
|
|
95
98
|
readonly outcome: ExerciseOutcome;
|
|
96
99
|
readonly code?: string;
|
|
97
100
|
readonly reason?: string;
|
|
101
|
+
/** On an `unreached` one that answered a 5xx: its cause, statement and fields, values masked. */
|
|
102
|
+
readonly defect?: ExerciseDefect;
|
|
98
103
|
readonly statements: number;
|
|
99
104
|
readonly effects: readonly ExerciseEffect[];
|
|
100
105
|
readonly ms: number;
|
package/src/exercise.ts
CHANGED
|
@@ -15,6 +15,7 @@ import { assertOverridesKnown, unknownDeclaration } from './define-exercise';
|
|
|
15
15
|
import type { RoleGrants } from './exercise-actor';
|
|
16
16
|
import { chooseActor } from './exercise-actor';
|
|
17
17
|
import { proveCrash } from './exercise-crash';
|
|
18
|
+
import { defectDetail } from './exercise-defect';
|
|
18
19
|
import type { IdLookup, Sampler } from './exercise-input';
|
|
19
20
|
import {
|
|
20
21
|
bindIds,
|
|
@@ -212,7 +213,20 @@ async function control(
|
|
|
212
213
|
const observed = room.end();
|
|
213
214
|
const effects = [...observed.effects, ...(await enqueued(subject, jobs, executions))];
|
|
214
215
|
const verdictOf = judged(subject, { verdict, answered, lastError, lastInput }, observed);
|
|
215
|
-
|
|
216
|
+
// A 5xx's cause, statement and fields: the code alone sent the author to reproduce it by hand.
|
|
217
|
+
const defect =
|
|
218
|
+
verdict.kind === 'defect' && verdictOf.outcome === 'unreached'
|
|
219
|
+
? await defectDetail(verdict.error, verdict.input, subject.input)
|
|
220
|
+
: undefined;
|
|
221
|
+
const record = {
|
|
222
|
+
kind,
|
|
223
|
+
name,
|
|
224
|
+
statements: observed.statements,
|
|
225
|
+
effects,
|
|
226
|
+
ms,
|
|
227
|
+
...verdictOf,
|
|
228
|
+
...(defect === undefined ? {} : { defect }),
|
|
229
|
+
};
|
|
216
230
|
return { record, output, input: lastInput };
|
|
217
231
|
}
|
|
218
232
|
|
package/src/index.ts
CHANGED
|
@@ -200,6 +200,7 @@ export {
|
|
|
200
200
|
export type { ExercisePassOptions } from './exercise';
|
|
201
201
|
export { runExercisePass } from './exercise';
|
|
202
202
|
export type { ActorChoice, RoleGrants } from './exercise-actor';
|
|
203
|
+
export type { ExerciseDefect } from './exercise-defect';
|
|
203
204
|
export type {
|
|
204
205
|
CrashFinding,
|
|
205
206
|
CrashProof,
|
|
@@ -376,6 +377,7 @@ export { crashProof } from './job-crash-proof';
|
|
|
376
377
|
export { jobDriverConformance } from './job-driver-conformance';
|
|
377
378
|
export type { SimEffect } from './job-sim-effects';
|
|
378
379
|
export {
|
|
380
|
+
crashEffectPlace,
|
|
379
381
|
JobCrashDivergedError,
|
|
380
382
|
JobSimInvariantError,
|
|
381
383
|
JobSimNondeterministicError,
|
|
@@ -444,6 +446,7 @@ export {
|
|
|
444
446
|
allowHost,
|
|
445
447
|
captureNetwork,
|
|
446
448
|
isNetworkSealed,
|
|
449
|
+
MOCKED_HOST_ADDRESS,
|
|
447
450
|
mockFetch,
|
|
448
451
|
mockJson,
|
|
449
452
|
networkState,
|
package/src/job-crash-proof.ts
CHANGED
|
@@ -33,6 +33,8 @@ export interface CrashDivergence {
|
|
|
33
33
|
readonly effect: string;
|
|
34
34
|
readonly clean: number;
|
|
35
35
|
readonly killed: number;
|
|
36
|
+
/** The step whose body made the effect, when every occurrence was made inside that one step. */
|
|
37
|
+
readonly inside?: string;
|
|
36
38
|
readonly error: JobCrashDivergedError;
|
|
37
39
|
}
|
|
38
40
|
|
|
@@ -155,8 +157,22 @@ async function runOnce<I>(
|
|
|
155
157
|
}
|
|
156
158
|
|
|
157
159
|
/**
|
|
158
|
-
*
|
|
159
|
-
*
|
|
160
|
+
* The one step whose body made every occurrence of `key` in both runs, or `undefined` when any
|
|
161
|
+
* occurrence was made between steps (or in two different steps).
|
|
162
|
+
*/
|
|
163
|
+
function insideOf(key: string, runs: readonly Ran[]): string | undefined {
|
|
164
|
+
const steps = new Set(
|
|
165
|
+
runs.flatMap((run) => run.effects.filter((one) => one.key === key).map((one) => one.inside)),
|
|
166
|
+
);
|
|
167
|
+
const [only] = steps;
|
|
168
|
+
return steps.size === 1 ? only : undefined;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Effects the killed run made more often than the reference — or less often, when both ended alike,
|
|
173
|
+
* neither unwound and the effect is made between steps. Inside a step body that did not complete,
|
|
174
|
+
* fewer is a kill that spent one of the step's attempts (the queue's lease accounting), not a call
|
|
175
|
+
* the crash skipped. An unwound run is also held to the reference's compensators: each once, last
|
|
160
176
|
* in first out, for exactly the forward steps that completed in it.
|
|
161
177
|
*/
|
|
162
178
|
function compare(reference: Ran, killed: Ran): readonly [string, number, number][] {
|
|
@@ -169,7 +185,9 @@ function compare(reference: Ran, killed: Ran): readonly [string, number, number]
|
|
|
169
185
|
for (const key of keys) {
|
|
170
186
|
const c = before.get(key) ?? 0;
|
|
171
187
|
const k = after.get(key) ?? 0;
|
|
172
|
-
if (k > c || (k < c && alike
|
|
188
|
+
if (k > c || (k < c && alike && insideOf(key, [reference, killed]) === undefined)) {
|
|
189
|
+
out.push([key, c, k]);
|
|
190
|
+
}
|
|
173
191
|
}
|
|
174
192
|
if (unwound) {
|
|
175
193
|
const expected = reference.compensated.filter((name) =>
|
|
@@ -222,19 +240,25 @@ export async function crashProof<I>(
|
|
|
222
240
|
: clean;
|
|
223
241
|
for (const [effect, c, k] of compare(reference, killed)) {
|
|
224
242
|
const after = afterOf(effect);
|
|
243
|
+
const inside = insideOf(effect, [reference, killed]);
|
|
244
|
+
const where = {
|
|
245
|
+
...(after === undefined ? {} : { after }),
|
|
246
|
+
...(inside === undefined ? {} : { inside }),
|
|
247
|
+
};
|
|
225
248
|
divergences.push({
|
|
226
249
|
boundary: label,
|
|
227
250
|
mode,
|
|
228
251
|
effect,
|
|
229
252
|
clean: c,
|
|
230
253
|
killed: k,
|
|
254
|
+
...(inside === undefined ? {} : { inside }),
|
|
231
255
|
error: new JobCrashDivergedError({
|
|
232
256
|
job: handle.name,
|
|
233
257
|
boundary: label,
|
|
234
258
|
effect,
|
|
235
259
|
clean: c,
|
|
236
260
|
killed: k,
|
|
237
|
-
...
|
|
261
|
+
...where,
|
|
238
262
|
}),
|
|
239
263
|
});
|
|
240
264
|
}
|
package/src/job-sim-effects.ts
CHANGED
|
@@ -18,15 +18,17 @@ export interface SimEffect {
|
|
|
18
18
|
readonly key: string;
|
|
19
19
|
/** The worker whose body made it; absent for the harness. */
|
|
20
20
|
readonly worker?: string;
|
|
21
|
-
/** The last step that worker's run had
|
|
21
|
+
/** The last step that worker's run had COMPLETED when the effect happened. */
|
|
22
22
|
readonly after?: string;
|
|
23
|
+
/** The step whose body made it (`@ultimat3/jobs`' `currentStep()`); absent between steps. */
|
|
24
|
+
readonly inside?: string;
|
|
23
25
|
}
|
|
24
26
|
|
|
25
27
|
export interface EffectLog extends AsyncDisposable {
|
|
26
28
|
/** The worker a call in flight belongs to: what `run` scoped it to. */
|
|
27
29
|
readonly scope: AsyncContext<string>;
|
|
28
30
|
record(key: string): void;
|
|
29
|
-
/** A worker's run
|
|
31
|
+
/** A worker's run completed this step — what the next effect it makes is placed after. */
|
|
30
32
|
stepWritten(worker: string, step: string): void;
|
|
31
33
|
effects(): readonly SimEffect[];
|
|
32
34
|
clear(): void;
|
|
@@ -52,6 +54,7 @@ const methodOf = (input: RequestInfo | URL, init: RequestInit | undefined): stri
|
|
|
52
54
|
|
|
53
55
|
export async function openEffectLog(options: EffectLogOptions): Promise<EffectLog> {
|
|
54
56
|
const { setRowObserver } = await import('@ultimat3/entity');
|
|
57
|
+
const { currentStep } = await import('@ultimat3/jobs');
|
|
55
58
|
const mail = await import('@ultimat3/mail');
|
|
56
59
|
const scope = asyncContext<string>('the job simulation');
|
|
57
60
|
const log: SimEffect[] = [];
|
|
@@ -61,10 +64,12 @@ export async function openEffectLog(options: EffectLogOptions): Promise<EffectLo
|
|
|
61
64
|
const worker = scope.get();
|
|
62
65
|
if (worker !== undefined && options.isDead(worker)) return;
|
|
63
66
|
const after = worker === undefined ? undefined : lastStep.get(worker);
|
|
67
|
+
const inside = currentStep();
|
|
64
68
|
log.push({
|
|
65
69
|
key,
|
|
66
70
|
...(worker === undefined ? {} : { worker }),
|
|
67
71
|
...(after === undefined ? {} : { after }),
|
|
72
|
+
...(inside === undefined ? {} : { inside }),
|
|
68
73
|
});
|
|
69
74
|
};
|
|
70
75
|
|
package/src/job-sim-errors.ts
CHANGED
|
@@ -17,18 +17,49 @@ export interface CrashDivergedInput {
|
|
|
17
17
|
readonly effect: string;
|
|
18
18
|
readonly clean: number;
|
|
19
19
|
readonly killed: number;
|
|
20
|
-
/** The last step
|
|
20
|
+
/** The last step completed before the effect, in the clean run. */
|
|
21
21
|
readonly after?: string;
|
|
22
|
+
/** The step whose body made the effect, when it was made inside one. */
|
|
23
|
+
readonly inside?: string;
|
|
22
24
|
}
|
|
23
25
|
|
|
26
|
+
/**
|
|
27
|
+
* Where an effect was made, as the crash proof's two reporters phrase it — this error and the CLI's
|
|
28
|
+
* finding (`@ultimat3/cli`'s `exercise-findings.ts`) — so the two cannot name different places.
|
|
29
|
+
* `remedy` is prose for a `fix:`'s comment, after the call it pastes.
|
|
30
|
+
*/
|
|
31
|
+
export function crashEffectPlace(where: {
|
|
32
|
+
readonly after?: string | undefined;
|
|
33
|
+
readonly inside?: string | undefined;
|
|
34
|
+
}): { readonly place: string; readonly remedy: string } {
|
|
35
|
+
if (where.inside !== undefined) {
|
|
36
|
+
const step = renderCauseValue(where.inside);
|
|
37
|
+
return {
|
|
38
|
+
place: `it runs inside step ${step}, whose body runs again until the step completes`,
|
|
39
|
+
remedy: `make the body of step ${step} idempotent: key its external call on \`\${jobId}:${where.inside}\`, or move the effect into a step of its own`,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
const at =
|
|
43
|
+
where.after === undefined
|
|
44
|
+
? 'before the first step'
|
|
45
|
+
: `after step ${renderCauseValue(where.after)}`;
|
|
46
|
+
return {
|
|
47
|
+
place: `it runs outside every step, ${at}`,
|
|
48
|
+
remedy: `wrap the effect made ${at} in step.run('<name>', async () => …); key an external call on \`\${jobId}:<step>\``,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const howOften = (killed: number, clean: number): string =>
|
|
53
|
+
killed > clean ? 'so the retry makes it again' : 'so the crash skipped it';
|
|
54
|
+
|
|
24
55
|
export class JobCrashDivergedError extends UltimateError {
|
|
25
56
|
constructor(input: CrashDivergedInput) {
|
|
26
|
-
const
|
|
27
|
-
|
|
57
|
+
const { place, remedy } = crashEffectPlace(input);
|
|
58
|
+
const name = input.inside ?? `${input.boundary}-effect`;
|
|
28
59
|
super({
|
|
29
60
|
code: 'X_JOB_CRASH_DIVERGED',
|
|
30
|
-
cause: `job ${renderCauseValue(input.job)} killed at ${renderCauseValue(input.boundary)} made ${renderCauseValue(input.effect)} ${input.killed}× against ${input.clean}× in a clean run:
|
|
31
|
-
fix: `step.run(${renderFixLiteral(
|
|
61
|
+
cause: `job ${renderCauseValue(input.job)} killed at ${renderCauseValue(input.boundary)} made ${renderCauseValue(input.effect)} ${input.killed}× against ${input.clean}× in a clean run: ${place}, ${howOften(input.killed, input.clean)}`,
|
|
62
|
+
fix: `step.run(${renderFixLiteral(name, '<name>')}, async () => …) # in job(${renderFixLiteral(input.job, '<job>')}), for ${renderFixLiteral(input.effect, '<effect>')}: ${remedy}; then: x test job --filter <the test file>`,
|
|
32
63
|
meta: { ...input },
|
|
33
64
|
});
|
|
34
65
|
}
|
package/src/job-sim-world.ts
CHANGED
|
@@ -86,7 +86,10 @@ export async function openSimWorld(options: SimWorldOptions): Promise<SimWorld>
|
|
|
86
86
|
decide: (op) => options.decide(op),
|
|
87
87
|
landed: (op, answer) => {
|
|
88
88
|
remember(answer);
|
|
89
|
-
|
|
89
|
+
// Completed only: a `failed` or `started` record is not a step the body has moved past.
|
|
90
|
+
if (op.call === 'put' && op.step !== undefined && op.status === 'completed') {
|
|
91
|
+
log?.stepWritten(op.worker, op.step);
|
|
92
|
+
}
|
|
90
93
|
if ((op.call === 'enqueue' || op.call === 'enqueueMany') && log !== undefined) {
|
|
91
94
|
const results = (Array.isArray(answer) ? answer : [answer]) as readonly EnqueueResult[];
|
|
92
95
|
results.forEach((result, i) => {
|
package/src/mock-route.ts
CHANGED
|
@@ -79,3 +79,24 @@ export const routeMatches = (
|
|
|
79
79
|
const named = namedUrl(call.url, call.headers);
|
|
80
80
|
return named !== undefined && urlMatches(route.match, named);
|
|
81
81
|
};
|
|
82
|
+
|
|
83
|
+
/** A RegExp's literal text: a `.` in a host is `\.` in its source. */
|
|
84
|
+
const escaped = (text: string): string => text.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&');
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Whether `route` answers some URL on `hostname` — what makes the sealed resolver answer the name.
|
|
88
|
+
* A string names its host outright; a RegExp when it matches the host's origin or spells the host
|
|
89
|
+
* in its source (`/api\.vendor\.example\/v1/`), since no reader can enumerate what a pattern matches.
|
|
90
|
+
*/
|
|
91
|
+
export const routeNamesHost = (route: MockRoute, hostname: string): boolean => {
|
|
92
|
+
const host = hostname.toLowerCase();
|
|
93
|
+
if (route.match instanceof RegExp) {
|
|
94
|
+
const pattern = route.match;
|
|
95
|
+
const origins = [`https://${host}/`, `http://${host}/`];
|
|
96
|
+
if (origins.some((origin) => urlMatches(pattern, origin))) return true;
|
|
97
|
+
const source = pattern.source.toLowerCase();
|
|
98
|
+
return source.includes(escaped(host)) || source.includes(host);
|
|
99
|
+
}
|
|
100
|
+
const literal = route.match.endsWith('*') ? route.match.slice(0, -1) : route.match;
|
|
101
|
+
return URL.canParse(literal) && new URL(literal).hostname.toLowerCase() === host;
|
|
102
|
+
};
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// The resolver half of the sealed network: `Bun.dns.lookup`, `dns.lookup` and `dns.promises.lookup`.
|
|
2
|
+
// Only the patching lives here; WHAT a name resolves to is `sealed-network.ts`'s one gate, handed in,
|
|
3
|
+
// so a host `fetch` would answer by mock resolves and a host it would refuse does not. A named ESM
|
|
4
|
+
// import (`import { lookup } from 'node:dns'`) binds before the seal and is NOT covered (measured).
|
|
5
|
+
|
|
6
|
+
// why: `dns.lookup` and `dns.promises.lookup` are what an app's own resolver calls; Bun ships both.
|
|
7
|
+
import nodeDns from 'node:dns';
|
|
8
|
+
|
|
9
|
+
/** One lookup's verdict: the addresses to answer, a refusal to raise, or `real` for the resolver. */
|
|
10
|
+
export type DnsVerdict =
|
|
11
|
+
| { readonly answer: readonly string[] }
|
|
12
|
+
| { readonly refused: Error }
|
|
13
|
+
| { readonly real: true };
|
|
14
|
+
|
|
15
|
+
export type DnsGate = (hostname: string) => DnsVerdict;
|
|
16
|
+
|
|
17
|
+
interface LookupEntry {
|
|
18
|
+
readonly address: string;
|
|
19
|
+
readonly family: number;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const familyOf = (address: string): number => (address.includes(':') ? 6 : 4);
|
|
23
|
+
|
|
24
|
+
const entriesOf = (addresses: readonly string[]): LookupEntry[] =>
|
|
25
|
+
addresses.map((address) => ({ address, family: familyOf(address) }));
|
|
26
|
+
|
|
27
|
+
type NodeCallback = (error: unknown, address?: unknown, family?: number) => void;
|
|
28
|
+
|
|
29
|
+
/** Node's `(hostname, options?, callback)`; `options.all` answers the list, a number is a family. */
|
|
30
|
+
const wantsAll = (options: unknown): boolean =>
|
|
31
|
+
typeof options === 'object' && options !== null && 'all' in options && options.all === true;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Patch the three resolvers and return the uninstaller, which puts back exactly what was there.
|
|
35
|
+
* Assigned through `Reflect.set`: each is a writable data property the types declare a function.
|
|
36
|
+
*/
|
|
37
|
+
export function installDnsSeal(gate: DnsGate): () => void {
|
|
38
|
+
const originalBun = Reflect.get(Bun.dns, 'lookup') as typeof Bun.dns.lookup;
|
|
39
|
+
const originalNode = Reflect.get(nodeDns, 'lookup') as (...args: unknown[]) => void;
|
|
40
|
+
const originalPromises = Reflect.get(nodeDns.promises, 'lookup') as (
|
|
41
|
+
...args: unknown[]
|
|
42
|
+
) => Promise<unknown>;
|
|
43
|
+
|
|
44
|
+
const sealedBun = async (hostname: string, options?: unknown): Promise<unknown> => {
|
|
45
|
+
const verdict = gate(hostname);
|
|
46
|
+
if ('refused' in verdict) throw verdict.refused;
|
|
47
|
+
if ('answer' in verdict) return entriesOf(verdict.answer).map((one) => ({ ...one, ttl: 0 }));
|
|
48
|
+
return Reflect.apply(originalBun, Bun.dns, [hostname, options]);
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
const sealedNode = (hostname: string, ...rest: unknown[]): void => {
|
|
52
|
+
const verdict = gate(hostname);
|
|
53
|
+
if ('real' in verdict) {
|
|
54
|
+
Reflect.apply(originalNode, nodeDns, [hostname, ...rest]);
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
const callback = rest.findLast((one) => typeof one === 'function') as NodeCallback | undefined;
|
|
58
|
+
const options = rest.length > 1 ? rest[0] : undefined;
|
|
59
|
+
// A real lookup answers on a later tick, and a caller may rely on it: never synchronously.
|
|
60
|
+
queueMicrotask(() => {
|
|
61
|
+
if ('refused' in verdict) return callback?.(verdict.refused);
|
|
62
|
+
const entries = entriesOf(verdict.answer);
|
|
63
|
+
if (wantsAll(options)) return callback?.(null, entries);
|
|
64
|
+
return callback?.(null, entries[0]?.address, entries[0]?.family);
|
|
65
|
+
});
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
const sealedPromises = async (hostname: string, options?: unknown): Promise<unknown> => {
|
|
69
|
+
const verdict = gate(hostname);
|
|
70
|
+
if ('refused' in verdict) throw verdict.refused;
|
|
71
|
+
if ('real' in verdict) {
|
|
72
|
+
return Reflect.apply(originalPromises, nodeDns.promises, [hostname, options]);
|
|
73
|
+
}
|
|
74
|
+
const entries = entriesOf(verdict.answer);
|
|
75
|
+
return wantsAll(options) ? entries : entries[0];
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
Reflect.set(Bun.dns, 'lookup', sealedBun);
|
|
79
|
+
Reflect.set(nodeDns, 'lookup', sealedNode);
|
|
80
|
+
Reflect.set(nodeDns.promises, 'lookup', sealedPromises);
|
|
81
|
+
return () => {
|
|
82
|
+
Reflect.set(Bun.dns, 'lookup', originalBun);
|
|
83
|
+
Reflect.set(nodeDns, 'lookup', originalNode);
|
|
84
|
+
Reflect.set(nodeDns.promises, 'lookup', originalPromises);
|
|
85
|
+
};
|
|
86
|
+
}
|
package/src/sealed-network.ts
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
|
-
// Sealed network
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
// `fetch` here, `WebSocket` and `Bun.connect` through `sealed-sockets.ts`, all behind one gate.
|
|
1
|
+
// Sealed network: egress a test did not mock or allow fails with the URL and the line that fixes
|
|
2
|
+
// it, so nothing reaches the internet unreproducibly. `fetch` here, `WebSocket`/`Bun.connect` via
|
|
3
|
+
// `sealed-sockets.ts`, name resolution via `sealed-dns.ts` — one gate, one set of mocks and allows.
|
|
5
4
|
|
|
6
5
|
import { isSelfOrigin, peerAddressClass } from '@ultimat3/core';
|
|
7
6
|
import { NetworkOfflineError, NetworkRaceError, NetworkSealedError } from './errors';
|
|
8
7
|
import type { MockHandler, MockRoute } from './mock-route';
|
|
9
|
-
import { mockRoute, routeMatches } from './mock-route';
|
|
8
|
+
import { mockRoute, routeMatches, routeNamesHost } from './mock-route';
|
|
10
9
|
import type { LoggedRequest } from './request-log';
|
|
11
10
|
import { hasStreamBody, headersOf, isOpenEnded, readBody, requestProbe } from './request-log';
|
|
11
|
+
import type { DnsVerdict } from './sealed-dns';
|
|
12
|
+
import { installDnsSeal } from './sealed-dns';
|
|
12
13
|
import { installSocketSeal } from './sealed-sockets';
|
|
13
14
|
|
|
14
15
|
export type FetchLike = typeof globalThis.fetch;
|
|
@@ -26,6 +27,8 @@ interface SealState {
|
|
|
26
27
|
original: FetchLike | undefined;
|
|
27
28
|
/** `installSocketSeal`'s undo, held beside `original` so one unseal restores all three dials. */
|
|
28
29
|
restoreSockets: (() => void) | undefined;
|
|
30
|
+
/** `installDnsSeal`'s undo, likewise. */
|
|
31
|
+
restoreDns: (() => void) | undefined;
|
|
29
32
|
network: NetworkState;
|
|
30
33
|
}
|
|
31
34
|
|
|
@@ -35,9 +38,17 @@ const state: SealState = {
|
|
|
35
38
|
log: [],
|
|
36
39
|
original: undefined,
|
|
37
40
|
restoreSockets: undefined,
|
|
41
|
+
restoreDns: undefined,
|
|
38
42
|
network: 'online',
|
|
39
43
|
};
|
|
40
44
|
|
|
45
|
+
/**
|
|
46
|
+
* What a mocked host resolves to: one `public` address (`classifyAddress`), so an SSRF screen
|
|
47
|
+
* passes it, and never a host network — a screen refuses those. Never dialled: a dial pinned to it
|
|
48
|
+
* carries the name in `Host`, which is how the mock answers it (`mock-route.ts`'s `namedUrl`).
|
|
49
|
+
*/
|
|
50
|
+
export const MOCKED_HOST_ADDRESS = '11.11.11.11';
|
|
51
|
+
|
|
41
52
|
const urlOf = (input: RequestInfo | URL): string => {
|
|
42
53
|
if (typeof input === 'string') return input;
|
|
43
54
|
if (input instanceof URL) return input.toString();
|
|
@@ -82,12 +93,40 @@ const socketRefusal = (url: string, method: string): Error | undefined => {
|
|
|
82
93
|
return new NetworkSealedError({ url, method, allowed: [...state.allowed], transport: 'socket' });
|
|
83
94
|
};
|
|
84
95
|
|
|
85
|
-
/**
|
|
86
|
-
|
|
96
|
+
/** A host the allow-list names, whatever port it was allowed on: a lookup carries no port. */
|
|
97
|
+
const isAllowedName = (hostname: string): boolean =>
|
|
98
|
+
[...state.allowed].some((host) => safeHost(`https://${host}`)?.split(':')[0] === hostname);
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* The resolver's gate, in `fetch`'s order: offline refuses every name, this machine and an allowed
|
|
102
|
+
* host resolve for real, a host a mock answers resolves to `MOCKED_HOST_ADDRESS`, the rest is
|
|
103
|
+
* `X_TEST_NETWORK_SEALED` — the code `fetch` would have raised a moment later.
|
|
104
|
+
*/
|
|
105
|
+
const dnsVerdict = (hostname: string): DnsVerdict => {
|
|
106
|
+
const name = hostname.toLowerCase().replace(/^\[|\]$/g, '');
|
|
107
|
+
const url = `dns://${name}`;
|
|
108
|
+
if (state.network !== 'online') {
|
|
109
|
+
return { refused: new NetworkOfflineError({ url, method: 'LOOKUP', mode: state.network }) };
|
|
110
|
+
}
|
|
111
|
+
if (isThisMachine(name) || peerAddressClass(name) !== undefined || isAllowedName(name)) {
|
|
112
|
+
return { real: true };
|
|
113
|
+
}
|
|
114
|
+
if (state.mocks.some((route) => routeNamesHost(route, name))) {
|
|
115
|
+
return { answer: [MOCKED_HOST_ADDRESS] };
|
|
116
|
+
}
|
|
117
|
+
const allowed = [...state.allowed];
|
|
118
|
+
return {
|
|
119
|
+
refused: new NetworkSealedError({ url, method: 'LOOKUP', allowed, transport: 'dns' }),
|
|
120
|
+
};
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
/** Replace global fetch, `WebSocket`, `Bun.connect` and the resolvers. Idempotent: sealing twice
|
|
124
|
+
* keeps the one set of originals around. */
|
|
87
125
|
export function sealNetwork(): void {
|
|
88
126
|
if (state.original !== undefined) return;
|
|
89
127
|
state.original = globalThis.fetch;
|
|
90
128
|
state.restoreSockets = installSocketSeal(socketRefusal);
|
|
129
|
+
state.restoreDns = installDnsSeal(dnsVerdict);
|
|
91
130
|
globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
|
|
92
131
|
const url = urlOf(input);
|
|
93
132
|
const method = methodOf(input, init);
|
|
@@ -151,6 +190,8 @@ export function unsealNetwork(): void {
|
|
|
151
190
|
state.original = undefined;
|
|
152
191
|
state.restoreSockets?.();
|
|
153
192
|
state.restoreSockets = undefined;
|
|
193
|
+
state.restoreDns?.();
|
|
194
|
+
state.restoreDns = undefined;
|
|
154
195
|
}
|
|
155
196
|
|
|
156
197
|
/** Whether the patch is installed. The `network` fixture seals before going offline, so that
|
package/src/seed-graph-values.ts
CHANGED
|
@@ -152,6 +152,12 @@ const parsed = (column: SeedColumn, value: unknown): { readonly ok: unknown } |
|
|
|
152
152
|
}
|
|
153
153
|
};
|
|
154
154
|
|
|
155
|
+
/**
|
|
156
|
+
* What every driver binds as SQL NULL — a JSON `null` included (`pg-row.ts` `bindable`): `t.json()`
|
|
157
|
+
* accepts it and the sampler draws it, so a NOT NULL `jsonb` column must never be offered one.
|
|
158
|
+
*/
|
|
159
|
+
const storesAsSqlNull = (value: unknown): boolean => value === null || value === undefined;
|
|
160
|
+
|
|
155
161
|
/**
|
|
156
162
|
* Every storable value for this column, most plausible first: the declared default, then what the
|
|
157
163
|
* kind offers, then `null` when the column allows it. Empty means nothing generated is storable.
|
|
@@ -167,9 +173,8 @@ export function candidatesFor(
|
|
|
167
173
|
const out: unknown[] = [];
|
|
168
174
|
for (const value of raw) {
|
|
169
175
|
const result = parsed(column, value);
|
|
170
|
-
if (result
|
|
171
|
-
|
|
172
|
-
}
|
|
176
|
+
if (result === undefined || (meta.notNull && storesAsSqlNull(result.ok))) continue;
|
|
177
|
+
if (!out.some((seen) => Bun.deepEquals(seen, result.ok))) out.push(result.ok);
|
|
173
178
|
}
|
|
174
179
|
return keepsPatternsFirst(out, ctx.patterns ?? []);
|
|
175
180
|
}
|