@ultimat3/testing 30.4.0 → 30.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +3 -3
- package/package.json +17 -17
- package/src/errors.ts +4 -2
- package/src/exercise-crash.ts +1 -0
- package/src/exercise-defect.ts +71 -0
- package/src/exercise-job-run.ts +68 -0
- package/src/exercise-report.ts +6 -1
- package/src/exercise-subjects.ts +6 -4
- package/src/exercise.ts +24 -24
- package/src/fixture-jobs.ts +17 -5
- package/src/index.ts +3 -0
- package/src/job-crash-proof.ts +31 -4
- package/src/job-sim-effects.ts +7 -2
- package/src/job-sim-errors.ts +36 -5
- package/src/job-sim-world.ts +20 -6
- package/src/mock-route.ts +21 -0
- package/src/sealed-dns.ts +86 -0
- package/src/sealed-network.ts +48 -7
- package/src/seed-graph-values.ts +8 -3
package/CLAUDE.md
CHANGED
|
@@ -16,7 +16,7 @@ import is `registry-isolation.ts`, own entry point, off the barrel. Full rule: `
|
|
|
16
16
|
| No mocks of the DB | clone a template database; `template-db.ts` is the only DB path |
|
|
17
17
|
| No wall clock | `withFrozenClock` / `advanceClock`; `Date.now()` is frozen by the preload |
|
|
18
18
|
| Frozen ≠ different | `globalThis.Date` becomes a subclass; `FrozenDate[Symbol.hasInstance]` brands on the `[[DateValue]]` slot (`Date.prototype.getTime.call(value)` throws or not) |
|
|
19
|
-
| No unmocked egress | `sealed-network.ts` seals `fetch`; `sealed-sockets.ts` `WebSocket`, `Bun.connect` (not loopback); a miss is `X_TEST_NETWORK_SEALED` |
|
|
19
|
+
| No unmocked egress | `sealed-network.ts` seals `fetch`; `sealed-sockets.ts` `WebSocket`, `Bun.connect` (not loopback); `sealed-dns.ts` the three `lookup`s (a mocked host answers `MOCKED_HOST_ADDRESS`); a miss is `X_TEST_NETWORK_SEALED` |
|
|
20
20
|
| One `mockFetch`, and it grows | a verb in the match, ordered answers, a pinned dial's name (`mock-route.ts`) and `requestLog` (`request-log.ts`). A stream bound for a real server logs `null` (teeing drains it). `original(input, init)` goes as handed: `tls`, `unix`, `proxy` are in no `Request`. `cassette()`: below |
|
|
21
21
|
| Self is not egress | a port core's `markListening()` announced passes through — a socket test never unseals |
|
|
22
22
|
| Offline is a state, not a mock | `network.offline()` / `.drop()` fail every dial, loopback too, as `X_TEST_NETWORK_OFFLINE` before any mock: the app's offline path runs |
|
|
@@ -46,10 +46,10 @@ import is `registry-isolation.ts`, own entry point, off the barrel. Full rule: `
|
|
|
46
46
|
| Injection | `SqlRunner` and `connect` are parameters, so unit tests need no server |
|
|
47
47
|
| Fixtures | the preload registers the framework bag — an app registers only what it alone knows (`seed`, `actorFor`) |
|
|
48
48
|
| Every `e2eTest` is audited | `e2e-a11y.ts` (rules), `e2e-a11y-run.ts` (per-test run, `allow`, dedupe), `ax-node.ts` (the one AX reader); the full tree is read only in `cdp-e2e-page.ts` (`FULL_AX_TREE`, a convention). No switch; `allow` needs a non-blank `because`. And every raw tab: `e2e-a11y-tabs.ts`, checked after each `goto`/`reload` and at the test's end from the preload's `afterEach`; exceptions are `allowA11y([{ role, selector, because }])` |
|
|
49
|
-
| The exercise pass runs what is registered | `exercise.ts` (`runExercisePass`): every action, query and job once, its own tenant pair off one `seedGraph`, an actor `exercise-actor.ts` picks, 39's `runExercise` as the verdict. Rows COMMITTED. Overrides only via `defineExercise` (blank `off` refused) |
|
|
49
|
+
| The exercise pass runs what is registered | `exercise.ts` (`runExercisePass`): every action, query and job once, its own tenant pair off one `seedGraph`, an actor `exercise-actor.ts` picks, 39's `runExercise` as the verdict. Rows COMMITTED. Overrides only via `defineExercise` (blank `off` refused). A job runs ALONE (`exercise-job-run.ts`): a fresh queue, one `tick()`, what it queued left on the caller's queue unrun — never a `drain()` |
|
|
50
50
|
| An override `input` is awaited, as the call's actor | `exercise-override-input.ts`: run inside the call's own request, so `useContext().actor` is the seated actor; the one way to a signed input. `async` ones are left out of `x drill`'s load |
|
|
51
51
|
| A passkey test signs for real | `softAuthenticator` is `@ultimat3/auth/soft-authenticator`'s, never re-exported here (one value, one path: `flight-copies`) and never a second one (auth's own suites need it, and auth cannot import testing) |
|
|
52
|
-
| A reached job is crash-proven | `exercise-crash.ts`: `crashProof` with the control's input and actor, after the tenant proof; only `diverged` is red (`X_JOB_CRASH_DIVERGED`). Committed: job bodies escape transactions |
|
|
52
|
+
| A reached job is crash-proven | `exercise-crash.ts`: `crashProof` with the control's input and actor, after the tenant proof; only `diverged` is red (`X_JOB_CRASH_DIVERGED`), its place `crashEffectPlace` (`inside` a step via jobs' `currentStep()`, or `after` one). Its world is `alone`: only the job's own row is claimed, a child is an `enqueue:` effect, never run. Committed: job bodies escape transactions |
|
|
53
53
|
| The tenant proof is taint only | `tenant-proof.ts`: B, A, B; B is A's role minus `tenancy:cross`. Leak = seeded canary or unhanded id (`tenant-canary.ts`); write = moved digest. Only `crossed` is red |
|
|
54
54
|
| Every statement the pass sent is planned | `plan-sweep.ts` (`sweepPlans`): first relation an app table, `PREPARE` in a READ-ONLY transaction with seq/bitmap scans and sorts off, judged by `plan-oracle.ts`; red = `X_QUERY_UNINDEXED` / `X_QUERY_SORT_UNSERVED` with the `indexes:` entry (`plan-sweep-fix.ts`). `scan:` or `expectedSeqScan` is exempt, listed; credited: a partial unique whose predicate the statement states (one row), a declared equality-then-range index under a limit; no generic plan is `X_QUERY_PLAN_UNAVAILABLE`, never a pass |
|
|
55
55
|
| A shape that grows with the rows is red | `scale-sweep-run.ts`: two more tenants at `SCALE_SIZES` (2, 8 rows a table), every reached primitive once in each; `judgeScale` (pure) reds `X_QUERY_SCALES_WITH_ROWS`. Under `expectedQueryLoop` it is listed with its reason |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/testing",
|
|
3
|
-
"version": "30.
|
|
3
|
+
"version": "30.6.0",
|
|
4
4
|
"description": "Test harness: cloned template DBs per worker, frozen clock, sealed network, 6 test types",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ultimate",
|
|
@@ -46,21 +46,21 @@
|
|
|
46
46
|
"test": "bun test"
|
|
47
47
|
},
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@ultimat3/action": "30.
|
|
50
|
-
"@ultimat3/cache": "30.
|
|
51
|
-
"@ultimat3/core": "30.
|
|
52
|
-
"@ultimat3/db": "30.
|
|
53
|
-
"@ultimat3/entity": "30.
|
|
54
|
-
"@ultimat3/http": "30.
|
|
55
|
-
"@ultimat3/i18n": "30.
|
|
56
|
-
"@ultimat3/jobs": "30.
|
|
57
|
-
"@ultimat3/mail": "30.
|
|
58
|
-
"@ultimat3/policy": "30.
|
|
59
|
-
"@ultimat3/pwa": "30.
|
|
60
|
-
"@ultimat3/query": "30.
|
|
61
|
-
"@ultimat3/realtime": "30.
|
|
62
|
-
"@ultimat3/render": "30.
|
|
63
|
-
"@ultimat3/schema": "30.
|
|
64
|
-
"@ultimat3/time": "30.
|
|
49
|
+
"@ultimat3/action": "30.6.0",
|
|
50
|
+
"@ultimat3/cache": "30.6.0",
|
|
51
|
+
"@ultimat3/core": "30.6.0",
|
|
52
|
+
"@ultimat3/db": "30.6.0",
|
|
53
|
+
"@ultimat3/entity": "30.6.0",
|
|
54
|
+
"@ultimat3/http": "30.6.0",
|
|
55
|
+
"@ultimat3/i18n": "30.6.0",
|
|
56
|
+
"@ultimat3/jobs": "30.6.0",
|
|
57
|
+
"@ultimat3/mail": "30.6.0",
|
|
58
|
+
"@ultimat3/policy": "30.6.0",
|
|
59
|
+
"@ultimat3/pwa": "30.6.0",
|
|
60
|
+
"@ultimat3/query": "30.6.0",
|
|
61
|
+
"@ultimat3/realtime": "30.6.0",
|
|
62
|
+
"@ultimat3/render": "30.6.0",
|
|
63
|
+
"@ultimat3/schema": "30.6.0",
|
|
64
|
+
"@ultimat3/time": "30.6.0"
|
|
65
65
|
}
|
|
66
66
|
}
|
package/src/errors.ts
CHANGED
|
@@ -163,7 +163,7 @@ export class NetworkSealedError extends UltimateError {
|
|
|
163
163
|
url: string;
|
|
164
164
|
method: string;
|
|
165
165
|
allowed: readonly string[];
|
|
166
|
-
transport?: 'fetch' | 'socket' | undefined;
|
|
166
|
+
transport?: 'fetch' | 'socket' | 'dns' | undefined;
|
|
167
167
|
}) {
|
|
168
168
|
const allowCall = `allowHost('${hostOf(input.url)}')`;
|
|
169
169
|
const allow = `${allowCall} if it must be real`;
|
|
@@ -175,7 +175,9 @@ export class NetworkSealedError extends UltimateError {
|
|
|
175
175
|
fix:
|
|
176
176
|
input.transport === 'socket'
|
|
177
177
|
? `${allowCall} # only if it must be real — otherwise inject the transport: pass the code under test its \`connect\`/\`client\` option, or dial a Bun.listen server on 127.0.0.1 (loopback is never sealed)`
|
|
178
|
-
:
|
|
178
|
+
: input.transport === 'dns'
|
|
179
|
+
? `mockFetch('https://${hostOf(input.url)}/*', () => new Response('{}')) # a mocked host resolves — or ${allow}`
|
|
180
|
+
: `mockFetch('${input.url}', () => new Response('{}')) — or ${allow}`,
|
|
179
181
|
});
|
|
180
182
|
}
|
|
181
183
|
}
|
package/src/exercise-crash.ts
CHANGED
|
@@ -18,6 +18,7 @@ const findingOf = (divergence: CrashDivergence): CrashFinding => {
|
|
|
18
18
|
clean: divergence.clean,
|
|
19
19
|
killed: divergence.killed,
|
|
20
20
|
...(typeof after === 'string' ? { after } : {}),
|
|
21
|
+
...(divergence.inside === undefined ? {} : { inside: divergence.inside }),
|
|
21
22
|
};
|
|
22
23
|
};
|
|
23
24
|
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// What a 5xx tells the author beyond its code: the coded error's cause, the statement or the fields
|
|
2
|
+
// it names, every value the call's input held under a credential key or a `.sensitive()` field
|
|
3
|
+
// masked out of the text. Without it, every X_DB_STATEMENT_FAILED in the pass was rerun by hand.
|
|
4
|
+
|
|
5
|
+
import { captureValue, isUltimateError, REDACTED, stringField } from '@ultimat3/core';
|
|
6
|
+
import type { StandardSchemaV1 } from '@ultimat3/schema';
|
|
7
|
+
|
|
8
|
+
/** What `ExerciseRecord.defect` holds. */
|
|
9
|
+
export interface ExerciseDefect {
|
|
10
|
+
/** The error's own cause, credential values masked, cut at `MAX_DEFECT_CAUSE`. */
|
|
11
|
+
readonly cause: string;
|
|
12
|
+
/** The SQL a database refusal names (`… — statement: <sql>`), parameters as `$n`. */
|
|
13
|
+
readonly statement?: string;
|
|
14
|
+
/** The paths a validation failure names (`items.0.sku`). */
|
|
15
|
+
readonly fields?: readonly string[];
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** A cause long enough for a SQLSTATE, its message and a capped statement; past it, cut. */
|
|
19
|
+
export const MAX_DEFECT_CAUSE = 600;
|
|
20
|
+
|
|
21
|
+
const STATEMENT = / — statement: (.+)$/s;
|
|
22
|
+
|
|
23
|
+
/** Every scalar the masked copy replaced: what must not appear in the cause either. */
|
|
24
|
+
function maskedScalars(original: unknown, masked: unknown, into: string[]): void {
|
|
25
|
+
if (masked === REDACTED) {
|
|
26
|
+
if (typeof original === 'string' || typeof original === 'number') into.push(String(original));
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
if (typeof original !== 'object' || original === null) return;
|
|
30
|
+
if (typeof masked !== 'object' || masked === null) return;
|
|
31
|
+
for (const [key, entry] of Object.entries(original)) {
|
|
32
|
+
maskedScalars(entry, Reflect.get(masked, key), into);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const fieldsOf = (meta: Readonly<Record<string, unknown>> | undefined): readonly string[] => {
|
|
37
|
+
const issues = meta?.['issues'];
|
|
38
|
+
if (!Array.isArray(issues)) return [];
|
|
39
|
+
return issues.flatMap((issue: unknown) => {
|
|
40
|
+
const path = stringField(issue, 'path');
|
|
41
|
+
return path === undefined || path === '' ? [] : [path];
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* The detail of a CODED failure, or `undefined` for an uncoded throw — whose reason already is the
|
|
47
|
+
* throw. `schema` is the primitive's input: its `.sensitive()` fields are masked as `redactKeys`
|
|
48
|
+
* ones are (core's `captureValue`), so the record holds no value the audit log would not.
|
|
49
|
+
*/
|
|
50
|
+
export async function defectDetail(
|
|
51
|
+
error: unknown,
|
|
52
|
+
input: unknown,
|
|
53
|
+
schema: StandardSchemaV1,
|
|
54
|
+
): Promise<ExerciseDefect | undefined> {
|
|
55
|
+
if (!isUltimateError(error)) return undefined;
|
|
56
|
+
// On demand: `@ultimat3/schema` is a dynamic-only dependency of this barrel (`CLAUDE.md`).
|
|
57
|
+
const { maskSensitive } = await import('@ultimat3/schema');
|
|
58
|
+
const secrets: string[] = [];
|
|
59
|
+
maskedScalars(input, captureValue(maskSensitive(schema, input, REDACTED)).value, secrets);
|
|
60
|
+
let cause = error.cause;
|
|
61
|
+
for (const value of [...new Set(secrets)].sort((a, b) => b.length - a.length)) {
|
|
62
|
+
if (value !== '') cause = cause.split(value).join(REDACTED);
|
|
63
|
+
}
|
|
64
|
+
const statement = STATEMENT.exec(cause)?.[1]?.trim();
|
|
65
|
+
const fields = fieldsOf(error.meta);
|
|
66
|
+
return {
|
|
67
|
+
cause: cause.length > MAX_DEFECT_CAUSE ? `${cause.slice(0, MAX_DEFECT_CAUSE - 1)}…` : cause,
|
|
68
|
+
...(statement === undefined || statement === '' ? {} : { statement }),
|
|
69
|
+
...(fields.length === 0 ? {} : { fields }),
|
|
70
|
+
};
|
|
71
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// A job primitive's one execution, as the exercise pass runs it: ALONE, on a queue that holds nothing
|
|
2
|
+
// else, for one worker pass. What it enqueues is handed back to the caller's queue unrun, exactly as
|
|
3
|
+
// an action's enqueues are — counted there as `enqueue` effects, searched by the tenant proof, and
|
|
4
|
+
// never executed inside the subject's room, so a child's statements are never the subject's.
|
|
5
|
+
|
|
6
|
+
import type { Actor } from '@ultimat3/core';
|
|
7
|
+
import { assert } from '@ultimat3/core';
|
|
8
|
+
import type { AnyJobHandle, JobExecution, JobRecord } from '@ultimat3/jobs';
|
|
9
|
+
import { jobCursor, jobDriver, MAX_JOB_PAGE, registeredJobs } from '@ultimat3/jobs';
|
|
10
|
+
import type { RunJobs } from './fixture-jobs';
|
|
11
|
+
import { testJobs } from './fixture-jobs';
|
|
12
|
+
|
|
13
|
+
const QUEUED: ReadonlySet<string> = new Set(['ready', 'delayed']);
|
|
14
|
+
|
|
15
|
+
/** Every row the AMBIENT driver holds, page by page: what a fixture's queue has, finished or not. */
|
|
16
|
+
export async function ambientJobRows(): Promise<readonly JobRecord[]> {
|
|
17
|
+
const introspect = jobDriver()?.introspect;
|
|
18
|
+
if (introspect === undefined) return [];
|
|
19
|
+
const records: JobRecord[] = [];
|
|
20
|
+
for (let after: string | undefined; ; ) {
|
|
21
|
+
const page = await introspect.list({
|
|
22
|
+
limit: MAX_JOB_PAGE,
|
|
23
|
+
...(after === undefined ? {} : { after }),
|
|
24
|
+
});
|
|
25
|
+
records.push(...page);
|
|
26
|
+
const last = page[page.length - 1];
|
|
27
|
+
if (last === undefined || page.length < MAX_JOB_PAGE) break;
|
|
28
|
+
after = jobCursor(last);
|
|
29
|
+
}
|
|
30
|
+
return records;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* `handle` run once with `input` in `tenant`, as `actor`'s worker: enqueued on a fresh queue and
|
|
35
|
+
* claimed by ONE pass, so the only run is its own — never a job `into` already held (an earlier
|
|
36
|
+
* call's, an earlier primitive's) and never one it queued. Those it queued land on `into`, unrun.
|
|
37
|
+
*/
|
|
38
|
+
export async function runJobAlone(
|
|
39
|
+
handle: AnyJobHandle,
|
|
40
|
+
input: unknown,
|
|
41
|
+
at: { readonly actor: Actor | null; readonly tenant: string },
|
|
42
|
+
into: RunJobs,
|
|
43
|
+
): Promise<JobExecution | undefined> {
|
|
44
|
+
let own: JobExecution | undefined;
|
|
45
|
+
let queued: readonly JobRecord[];
|
|
46
|
+
{
|
|
47
|
+
await using alone = await testJobs();
|
|
48
|
+
const row = await alone.enqueue(handle, input, { tenantId: at.tenant });
|
|
49
|
+
const trace = await alone.tick(at.actor === null ? {} : { actor: at.actor });
|
|
50
|
+
own = trace.executions.find((run) => run.jobId === row.id);
|
|
51
|
+
queued = (await ambientJobRows()).filter(
|
|
52
|
+
(record) => record.id !== row.id && QUEUED.has(record.state),
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
const handles = new Map(registeredJobs().map((one) => [one.name, one]));
|
|
56
|
+
for (const record of queued) {
|
|
57
|
+
const child = handles.get(record.name);
|
|
58
|
+
assert(
|
|
59
|
+
child !== undefined,
|
|
60
|
+
`job "${handle.name}" queued job "${record.name}" but nothing registered it`,
|
|
61
|
+
`import the module that declares job("${record.name}") from the app's api module — the registry is populated by the import, not by the queue`,
|
|
62
|
+
);
|
|
63
|
+
await into.enqueue(child, record.input, {
|
|
64
|
+
...(record.tenantId === undefined ? {} : { tenantId: record.tenantId }),
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
return own;
|
|
68
|
+
}
|
package/src/exercise-report.ts
CHANGED
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
} from '@ultimat3/db';
|
|
13
13
|
import type { RowObserver } from '@ultimat3/entity';
|
|
14
14
|
import { captureDeterminism, installDeterminism, restoreCapturedDeterminism } from './determinism';
|
|
15
|
+
import type { ExerciseDefect } from './exercise-defect';
|
|
15
16
|
import { testMail } from './fixture-mail';
|
|
16
17
|
import { routeMatches } from './mock-route';
|
|
17
18
|
import type { PlanProof, SentStatement } from './plan-sweep';
|
|
@@ -72,8 +73,10 @@ export interface CrashFinding {
|
|
|
72
73
|
readonly effect: string;
|
|
73
74
|
readonly clean: number;
|
|
74
75
|
readonly killed: number;
|
|
75
|
-
/** The last step the clean run had
|
|
76
|
+
/** The last step the clean run had completed before the effect; absent, before the first. */
|
|
76
77
|
readonly after?: string;
|
|
78
|
+
/** The step whose body made the effect, when it was made inside one: it runs again on a retry. */
|
|
79
|
+
readonly inside?: string;
|
|
77
80
|
}
|
|
78
81
|
|
|
79
82
|
/** The crash proof's verdict on a reached job: killed at every step boundary (`crashProof`). */
|
|
@@ -95,6 +98,8 @@ export interface ExerciseRecord {
|
|
|
95
98
|
readonly outcome: ExerciseOutcome;
|
|
96
99
|
readonly code?: string;
|
|
97
100
|
readonly reason?: string;
|
|
101
|
+
/** On an `unreached` one that answered a 5xx: its cause, statement and fields, values masked. */
|
|
102
|
+
readonly defect?: ExerciseDefect;
|
|
98
103
|
readonly statements: number;
|
|
99
104
|
readonly effects: readonly ExerciseEffect[];
|
|
100
105
|
readonly ms: number;
|
package/src/exercise-subjects.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// The primitives an exercise pass drives, each as one call: every registered action, query and job,
|
|
2
2
|
// loaded on demand. A job is enqueued in the tenant it is called for, its input SEATED there first
|
|
3
|
-
// (`exercise-job-tenant.ts`): a drawn org is a random one, and the enqueue refuses it.
|
|
3
|
+
// (`exercise-job-tenant.ts`): a drawn org is a random one, and the enqueue refuses it. It runs
|
|
4
|
+
// alone (`exercise-job-run.ts`): what it queues is left queued, as an action's is.
|
|
4
5
|
|
|
5
6
|
import type { Actor } from '@ultimat3/core';
|
|
6
7
|
import { renderCauseValue, renderFixShellArg, UltimateError } from '@ultimat3/core';
|
|
@@ -8,6 +9,7 @@ import type { AnyJobHandle, JobExecution } from '@ultimat3/jobs';
|
|
|
8
9
|
import { registeredJobs } from '@ultimat3/jobs';
|
|
9
10
|
import type { Policy } from '@ultimat3/policy';
|
|
10
11
|
import type { StandardSchemaV1 } from '@ultimat3/schema';
|
|
12
|
+
import { runJobAlone } from './exercise-job-run';
|
|
11
13
|
import type { JobTenantBinding } from './exercise-job-tenant';
|
|
12
14
|
import { bindJobTenant } from './exercise-job-tenant';
|
|
13
15
|
import { withExerciseRequest } from './exercise-request';
|
|
@@ -84,9 +86,9 @@ const jobSubject = (handle: AnyJobHandle): Subject => ({
|
|
|
84
86
|
handle,
|
|
85
87
|
seat: (input, tenant) => bindJobTenant(handle, input, tenant),
|
|
86
88
|
call: async (input, actor, tenant, jobs) => {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
const own =
|
|
89
|
+
// Its own execution and nothing else: a drain of `jobs` ran what it queued (one child per row,
|
|
90
|
+
// which the scale sweep read as the job's loop) and whatever an earlier call had left there.
|
|
91
|
+
const own = await runJobAlone(handle, input, { actor, tenant }, jobs);
|
|
90
92
|
// The worker's failure is on the row, never a throw: the attempt's error and its code.
|
|
91
93
|
// Suspended is a body that ran and parked (`step.sleep`, `waitForEvent`): reached.
|
|
92
94
|
return own !== undefined && ANSWERED.has(own.outcome)
|
package/src/exercise.ts
CHANGED
|
@@ -8,13 +8,13 @@
|
|
|
8
8
|
|
|
9
9
|
import type { Actor, ExerciseAttempt, ExerciseVerdict } from '@ultimat3/core';
|
|
10
10
|
import { isUltimateError, renderThrowable, runExercise } from '@ultimat3/core';
|
|
11
|
-
import type { JobExecution, JobRecord } from '@ultimat3/jobs';
|
|
12
11
|
import { registeredJobs } from '@ultimat3/jobs';
|
|
13
12
|
import type { ExerciseOverride, ExerciseOverrides } from './define-exercise';
|
|
14
13
|
import { assertOverridesKnown, unknownDeclaration } from './define-exercise';
|
|
15
14
|
import type { RoleGrants } from './exercise-actor';
|
|
16
15
|
import { chooseActor } from './exercise-actor';
|
|
17
16
|
import { proveCrash } from './exercise-crash';
|
|
17
|
+
import { defectDetail } from './exercise-defect';
|
|
18
18
|
import type { IdLookup, Sampler } from './exercise-input';
|
|
19
19
|
import {
|
|
20
20
|
bindIds,
|
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
namesActorEntity,
|
|
25
25
|
seededIds,
|
|
26
26
|
} from './exercise-input';
|
|
27
|
+
import { ambientJobRows } from './exercise-job-run';
|
|
27
28
|
import type { OverrideInput } from './exercise-override-input';
|
|
28
29
|
import { overrideInput } from './exercise-override-input';
|
|
29
30
|
import type {
|
|
@@ -75,13 +76,15 @@ const codeOf = (error: unknown): string | undefined => {
|
|
|
75
76
|
const reasonOf = (error: unknown): string =>
|
|
76
77
|
isUltimateError(error) ? error.cause : typeof error === 'string' ? error : renderThrowable(error);
|
|
77
78
|
|
|
78
|
-
/**
|
|
79
|
-
|
|
79
|
+
/**
|
|
80
|
+
* Jobs the primitive queued, its own job excepted. Never run: a job's children are left queued as
|
|
81
|
+
* an action's are (`exercise-job-run.ts`), so their statements are theirs, never this record's.
|
|
82
|
+
*/
|
|
83
|
+
async function enqueued(subject: Subject, jobs: RunJobs) {
|
|
80
84
|
const effects: ExerciseEffect[] = [];
|
|
81
85
|
for (const handle of registeredJobs()) {
|
|
82
86
|
if (subject.kind === 'job' && handle.name === subject.name) continue;
|
|
83
|
-
const
|
|
84
|
-
const count = ran + (await jobs.depth(handle));
|
|
87
|
+
const count = await jobs.depth(handle);
|
|
85
88
|
if (count > 0) effects.push({ kind: 'enqueue', job: handle.name, count });
|
|
86
89
|
}
|
|
87
90
|
return effects;
|
|
@@ -92,21 +95,7 @@ async function enqueued(subject: Subject, jobs: RunJobs, runs: readonly JobExecu
|
|
|
92
95
|
* input is what B was handed. Read off the leg's own queue, page by page.
|
|
93
96
|
*/
|
|
94
97
|
async function queuedPayloads(subject: Subject): Promise<readonly EffectPayload[]> {
|
|
95
|
-
|
|
96
|
-
const introspect = jobs.jobDriver()?.introspect;
|
|
97
|
-
if (introspect === undefined) return [];
|
|
98
|
-
const records: JobRecord[] = [];
|
|
99
|
-
for (let after: string | undefined; ; ) {
|
|
100
|
-
const page = await introspect.list({
|
|
101
|
-
limit: jobs.MAX_JOB_PAGE,
|
|
102
|
-
...(after === undefined ? {} : { after }),
|
|
103
|
-
});
|
|
104
|
-
records.push(...page);
|
|
105
|
-
const last = page[page.length - 1];
|
|
106
|
-
if (last === undefined || page.length < jobs.MAX_JOB_PAGE) break;
|
|
107
|
-
after = jobs.jobCursor(last);
|
|
108
|
-
}
|
|
109
|
-
return records
|
|
98
|
+
return (await ambientJobRows())
|
|
110
99
|
.filter((record) => !(subject.kind === 'job' && record.name === subject.name))
|
|
111
100
|
.map((record, i) => ({
|
|
112
101
|
path: `effect.enqueue[${i}]`,
|
|
@@ -173,7 +162,6 @@ async function control(
|
|
|
173
162
|
let lastError: unknown;
|
|
174
163
|
let lastInput: unknown;
|
|
175
164
|
let output: unknown;
|
|
176
|
-
let executions: readonly JobExecution[] = [];
|
|
177
165
|
room.begin(name);
|
|
178
166
|
const started = performance.now();
|
|
179
167
|
const verdict = await runExercise({
|
|
@@ -208,11 +196,23 @@ async function control(
|
|
|
208
196
|
statusOf: (await import('@ultimat3/http')).statusFor,
|
|
209
197
|
});
|
|
210
198
|
const ms = Math.round(performance.now() - started);
|
|
211
|
-
if (kind === 'job') executions = (await jobs.drain()).executions;
|
|
212
199
|
const observed = room.end();
|
|
213
|
-
const effects = [...observed.effects, ...(await enqueued(subject, jobs
|
|
200
|
+
const effects = [...observed.effects, ...(await enqueued(subject, jobs))];
|
|
214
201
|
const verdictOf = judged(subject, { verdict, answered, lastError, lastInput }, observed);
|
|
215
|
-
|
|
202
|
+
// A 5xx's cause, statement and fields: the code alone sent the author to reproduce it by hand.
|
|
203
|
+
const defect =
|
|
204
|
+
verdict.kind === 'defect' && verdictOf.outcome === 'unreached'
|
|
205
|
+
? await defectDetail(verdict.error, verdict.input, subject.input)
|
|
206
|
+
: undefined;
|
|
207
|
+
const record = {
|
|
208
|
+
kind,
|
|
209
|
+
name,
|
|
210
|
+
statements: observed.statements,
|
|
211
|
+
effects,
|
|
212
|
+
ms,
|
|
213
|
+
...verdictOf,
|
|
214
|
+
...(defect === undefined ? {} : { defect }),
|
|
215
|
+
};
|
|
216
216
|
return { record, output, input: lastInput };
|
|
217
217
|
}
|
|
218
218
|
|
package/src/fixture-jobs.ts
CHANGED
|
@@ -69,6 +69,11 @@ export interface RunJobs extends AsyncDisposable {
|
|
|
69
69
|
enqueue<I>(handle: JobHandle<I>, input: I, options?: JobEnqueueAs): Promise<EnqueueResult>;
|
|
70
70
|
/** Claim and execute everything due at the current instant, until nothing is. */
|
|
71
71
|
drain(options?: JobDrainAs): Promise<JobRunTrace>;
|
|
72
|
+
/**
|
|
73
|
+
* ONE worker pass: what is due now runs, and what those runs enqueue waits for the next pass —
|
|
74
|
+
* "it queued one child per row" asserted without running a child.
|
|
75
|
+
*/
|
|
76
|
+
tick(options?: JobDrainAs): Promise<JobRunTrace>;
|
|
72
77
|
/** Live jobs — ready, delayed, running or suspended — optionally for one job only. */
|
|
73
78
|
depth(handle?: AnyJobHandle): Promise<number>;
|
|
74
79
|
/** Live jobs claimable right now. `clock.advance()` is what turns delayed into due. */
|
|
@@ -194,14 +199,20 @@ export async function testJobs(): Promise<RunJobs> {
|
|
|
194
199
|
/** Every execution this fixture has driven, because the trace is cumulative. */
|
|
195
200
|
const history: JobExecution[] = [];
|
|
196
201
|
|
|
202
|
+
const ctxFor = (options: JobDrainAs): Ctx =>
|
|
203
|
+
options.actor === undefined ? anonymousWorker : ctxOf({ role: 'worker', actor: options.actor });
|
|
204
|
+
const traced = (): JobRunTrace => ({ executions: [...history], steps: tallyOf(history) });
|
|
205
|
+
|
|
206
|
+
const tick = async (options: JobDrainAs = {}): Promise<JobRunTrace> => {
|
|
207
|
+
history.push(...(await round(ctxFor(options))));
|
|
208
|
+
return traced();
|
|
209
|
+
};
|
|
210
|
+
|
|
197
211
|
const drain = async (options: JobDrainAs = {}): Promise<JobRunTrace> => {
|
|
198
|
-
const ctx =
|
|
199
|
-
options.actor === undefined
|
|
200
|
-
? anonymousWorker
|
|
201
|
-
: ctxOf({ role: 'worker', actor: options.actor });
|
|
212
|
+
const ctx = ctxFor(options);
|
|
202
213
|
for (let rounds = 0; rounds < MAX_ROUNDS; rounds += 1) {
|
|
203
214
|
const batch = await round(ctx);
|
|
204
|
-
if (batch.length === 0) return
|
|
215
|
+
if (batch.length === 0) return traced();
|
|
205
216
|
history.push(...batch);
|
|
206
217
|
}
|
|
207
218
|
assert(
|
|
@@ -239,6 +250,7 @@ export async function testJobs(): Promise<RunJobs> {
|
|
|
239
250
|
return Object.assign(enqueueThenDrain, {
|
|
240
251
|
enqueue,
|
|
241
252
|
drain,
|
|
253
|
+
tick,
|
|
242
254
|
depth: async (handle?: AnyJobHandle) => (await live(handle?.name)).length,
|
|
243
255
|
due: async () =>
|
|
244
256
|
(await live()).filter(
|
package/src/index.ts
CHANGED
|
@@ -200,6 +200,7 @@ export {
|
|
|
200
200
|
export type { ExercisePassOptions } from './exercise';
|
|
201
201
|
export { runExercisePass } from './exercise';
|
|
202
202
|
export type { ActorChoice, RoleGrants } from './exercise-actor';
|
|
203
|
+
export type { ExerciseDefect } from './exercise-defect';
|
|
203
204
|
export type {
|
|
204
205
|
CrashFinding,
|
|
205
206
|
CrashProof,
|
|
@@ -376,6 +377,7 @@ export { crashProof } from './job-crash-proof';
|
|
|
376
377
|
export { jobDriverConformance } from './job-driver-conformance';
|
|
377
378
|
export type { SimEffect } from './job-sim-effects';
|
|
378
379
|
export {
|
|
380
|
+
crashEffectPlace,
|
|
379
381
|
JobCrashDivergedError,
|
|
380
382
|
JobSimInvariantError,
|
|
381
383
|
JobSimNondeterministicError,
|
|
@@ -444,6 +446,7 @@ export {
|
|
|
444
446
|
allowHost,
|
|
445
447
|
captureNetwork,
|
|
446
448
|
isNetworkSealed,
|
|
449
|
+
MOCKED_HOST_ADDRESS,
|
|
447
450
|
mockFetch,
|
|
448
451
|
mockJson,
|
|
449
452
|
networkState,
|
package/src/job-crash-proof.ts
CHANGED
|
@@ -33,6 +33,8 @@ export interface CrashDivergence {
|
|
|
33
33
|
readonly effect: string;
|
|
34
34
|
readonly clean: number;
|
|
35
35
|
readonly killed: number;
|
|
36
|
+
/** The step whose body made the effect, when every occurrence was made inside that one step. */
|
|
37
|
+
readonly inside?: string;
|
|
36
38
|
readonly error: JobCrashDivergedError;
|
|
37
39
|
}
|
|
38
40
|
|
|
@@ -132,7 +134,10 @@ async function runOnce<I>(
|
|
|
132
134
|
if (op.step.startsWith(COMPENSATE_PREFIX)) compensated.push(op.step);
|
|
133
135
|
else if (op.step !== UNWINDING_STEP) forward.add(op.step);
|
|
134
136
|
};
|
|
137
|
+
// Alone: a child the job queues is its enqueue effect, never run — the clean run that ran it
|
|
138
|
+
// first wrote its rows, and every killed run after found them there (0× against 6×).
|
|
135
139
|
await using world = await openSimWorld({
|
|
140
|
+
alone: true,
|
|
136
141
|
seed: finiteCount('crashProof', 'seed', options.seed ?? 1),
|
|
137
142
|
decide,
|
|
138
143
|
landed,
|
|
@@ -155,8 +160,22 @@ async function runOnce<I>(
|
|
|
155
160
|
}
|
|
156
161
|
|
|
157
162
|
/**
|
|
158
|
-
*
|
|
159
|
-
*
|
|
163
|
+
* The one step whose body made every occurrence of `key` in both runs, or `undefined` when any
|
|
164
|
+
* occurrence was made between steps (or in two different steps).
|
|
165
|
+
*/
|
|
166
|
+
function insideOf(key: string, runs: readonly Ran[]): string | undefined {
|
|
167
|
+
const steps = new Set(
|
|
168
|
+
runs.flatMap((run) => run.effects.filter((one) => one.key === key).map((one) => one.inside)),
|
|
169
|
+
);
|
|
170
|
+
const [only] = steps;
|
|
171
|
+
return steps.size === 1 ? only : undefined;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Effects the killed run made more often than the reference — or less often, when both ended alike,
|
|
176
|
+
* neither unwound and the effect is made between steps. Inside a step body that did not complete,
|
|
177
|
+
* fewer is a kill that spent one of the step's attempts (the queue's lease accounting), not a call
|
|
178
|
+
* the crash skipped. An unwound run is also held to the reference's compensators: each once, last
|
|
160
179
|
* in first out, for exactly the forward steps that completed in it.
|
|
161
180
|
*/
|
|
162
181
|
function compare(reference: Ran, killed: Ran): readonly [string, number, number][] {
|
|
@@ -169,7 +188,9 @@ function compare(reference: Ran, killed: Ran): readonly [string, number, number]
|
|
|
169
188
|
for (const key of keys) {
|
|
170
189
|
const c = before.get(key) ?? 0;
|
|
171
190
|
const k = after.get(key) ?? 0;
|
|
172
|
-
if (k > c || (k < c && alike
|
|
191
|
+
if (k > c || (k < c && alike && insideOf(key, [reference, killed]) === undefined)) {
|
|
192
|
+
out.push([key, c, k]);
|
|
193
|
+
}
|
|
173
194
|
}
|
|
174
195
|
if (unwound) {
|
|
175
196
|
const expected = reference.compensated.filter((name) =>
|
|
@@ -222,19 +243,25 @@ export async function crashProof<I>(
|
|
|
222
243
|
: clean;
|
|
223
244
|
for (const [effect, c, k] of compare(reference, killed)) {
|
|
224
245
|
const after = afterOf(effect);
|
|
246
|
+
const inside = insideOf(effect, [reference, killed]);
|
|
247
|
+
const where = {
|
|
248
|
+
...(after === undefined ? {} : { after }),
|
|
249
|
+
...(inside === undefined ? {} : { inside }),
|
|
250
|
+
};
|
|
225
251
|
divergences.push({
|
|
226
252
|
boundary: label,
|
|
227
253
|
mode,
|
|
228
254
|
effect,
|
|
229
255
|
clean: c,
|
|
230
256
|
killed: k,
|
|
257
|
+
...(inside === undefined ? {} : { inside }),
|
|
231
258
|
error: new JobCrashDivergedError({
|
|
232
259
|
job: handle.name,
|
|
233
260
|
boundary: label,
|
|
234
261
|
effect,
|
|
235
262
|
clean: c,
|
|
236
263
|
killed: k,
|
|
237
|
-
...
|
|
264
|
+
...where,
|
|
238
265
|
}),
|
|
239
266
|
});
|
|
240
267
|
}
|
package/src/job-sim-effects.ts
CHANGED
|
@@ -18,15 +18,17 @@ export interface SimEffect {
|
|
|
18
18
|
readonly key: string;
|
|
19
19
|
/** The worker whose body made it; absent for the harness. */
|
|
20
20
|
readonly worker?: string;
|
|
21
|
-
/** The last step that worker's run had
|
|
21
|
+
/** The last step that worker's run had COMPLETED when the effect happened. */
|
|
22
22
|
readonly after?: string;
|
|
23
|
+
/** The step whose body made it (`@ultimat3/jobs`' `currentStep()`); absent between steps. */
|
|
24
|
+
readonly inside?: string;
|
|
23
25
|
}
|
|
24
26
|
|
|
25
27
|
export interface EffectLog extends AsyncDisposable {
|
|
26
28
|
/** The worker a call in flight belongs to: what `run` scoped it to. */
|
|
27
29
|
readonly scope: AsyncContext<string>;
|
|
28
30
|
record(key: string): void;
|
|
29
|
-
/** A worker's run
|
|
31
|
+
/** A worker's run completed this step — what the next effect it makes is placed after. */
|
|
30
32
|
stepWritten(worker: string, step: string): void;
|
|
31
33
|
effects(): readonly SimEffect[];
|
|
32
34
|
clear(): void;
|
|
@@ -52,6 +54,7 @@ const methodOf = (input: RequestInfo | URL, init: RequestInit | undefined): stri
|
|
|
52
54
|
|
|
53
55
|
export async function openEffectLog(options: EffectLogOptions): Promise<EffectLog> {
|
|
54
56
|
const { setRowObserver } = await import('@ultimat3/entity');
|
|
57
|
+
const { currentStep } = await import('@ultimat3/jobs');
|
|
55
58
|
const mail = await import('@ultimat3/mail');
|
|
56
59
|
const scope = asyncContext<string>('the job simulation');
|
|
57
60
|
const log: SimEffect[] = [];
|
|
@@ -61,10 +64,12 @@ export async function openEffectLog(options: EffectLogOptions): Promise<EffectLo
|
|
|
61
64
|
const worker = scope.get();
|
|
62
65
|
if (worker !== undefined && options.isDead(worker)) return;
|
|
63
66
|
const after = worker === undefined ? undefined : lastStep.get(worker);
|
|
67
|
+
const inside = currentStep();
|
|
64
68
|
log.push({
|
|
65
69
|
key,
|
|
66
70
|
...(worker === undefined ? {} : { worker }),
|
|
67
71
|
...(after === undefined ? {} : { after }),
|
|
72
|
+
...(inside === undefined ? {} : { inside }),
|
|
68
73
|
});
|
|
69
74
|
};
|
|
70
75
|
|
package/src/job-sim-errors.ts
CHANGED
|
@@ -17,18 +17,49 @@ export interface CrashDivergedInput {
|
|
|
17
17
|
readonly effect: string;
|
|
18
18
|
readonly clean: number;
|
|
19
19
|
readonly killed: number;
|
|
20
|
-
/** The last step
|
|
20
|
+
/** The last step completed before the effect, in the clean run. */
|
|
21
21
|
readonly after?: string;
|
|
22
|
+
/** The step whose body made the effect, when it was made inside one. */
|
|
23
|
+
readonly inside?: string;
|
|
22
24
|
}
|
|
23
25
|
|
|
26
|
+
/**
|
|
27
|
+
* Where an effect was made, as the crash proof's two reporters phrase it — this error and the CLI's
|
|
28
|
+
* finding (`@ultimat3/cli`'s `exercise-findings.ts`) — so the two cannot name different places.
|
|
29
|
+
* `remedy` is prose for a `fix:`'s comment, after the call it pastes.
|
|
30
|
+
*/
|
|
31
|
+
export function crashEffectPlace(where: {
|
|
32
|
+
readonly after?: string | undefined;
|
|
33
|
+
readonly inside?: string | undefined;
|
|
34
|
+
}): { readonly place: string; readonly remedy: string } {
|
|
35
|
+
if (where.inside !== undefined) {
|
|
36
|
+
const step = renderCauseValue(where.inside);
|
|
37
|
+
return {
|
|
38
|
+
place: `it runs inside step ${step}, whose body runs again until the step completes`,
|
|
39
|
+
remedy: `make the body of step ${step} idempotent: key its external call on \`\${jobId}:${where.inside}\`, or move the effect into a step of its own`,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
const at =
|
|
43
|
+
where.after === undefined
|
|
44
|
+
? 'before the first step'
|
|
45
|
+
: `after step ${renderCauseValue(where.after)}`;
|
|
46
|
+
return {
|
|
47
|
+
place: `it runs outside every step, ${at}`,
|
|
48
|
+
remedy: `wrap the effect made ${at} in step.run('<name>', async () => …); key an external call on \`\${jobId}:<step>\``,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const howOften = (killed: number, clean: number): string =>
|
|
53
|
+
killed > clean ? 'so the retry makes it again' : 'so the crash skipped it';
|
|
54
|
+
|
|
24
55
|
export class JobCrashDivergedError extends UltimateError {
|
|
25
56
|
constructor(input: CrashDivergedInput) {
|
|
26
|
-
const
|
|
27
|
-
|
|
57
|
+
const { place, remedy } = crashEffectPlace(input);
|
|
58
|
+
const name = input.inside ?? `${input.boundary}-effect`;
|
|
28
59
|
super({
|
|
29
60
|
code: 'X_JOB_CRASH_DIVERGED',
|
|
30
|
-
cause: `job ${renderCauseValue(input.job)} killed at ${renderCauseValue(input.boundary)} made ${renderCauseValue(input.effect)} ${input.killed}× against ${input.clean}× in a clean run:
|
|
31
|
-
fix: `step.run(${renderFixLiteral(
|
|
61
|
+
cause: `job ${renderCauseValue(input.job)} killed at ${renderCauseValue(input.boundary)} made ${renderCauseValue(input.effect)} ${input.killed}× against ${input.clean}× in a clean run: ${place}, ${howOften(input.killed, input.clean)}`,
|
|
62
|
+
fix: `step.run(${renderFixLiteral(name, '<name>')}, async () => …) # in job(${renderFixLiteral(input.job, '<job>')}), for ${renderFixLiteral(input.effect, '<effect>')}: ${remedy}; then: x test job --filter <the test file>`,
|
|
32
63
|
meta: { ...input },
|
|
33
64
|
});
|
|
34
65
|
}
|
package/src/job-sim-world.ts
CHANGED
|
@@ -31,6 +31,8 @@ export const SIM_VISIBILITY_MS = 30_000;
|
|
|
31
31
|
const RENEW_MS = 1;
|
|
32
32
|
const CLAIM_LIMIT = 64;
|
|
33
33
|
const LIVE: ReadonlySet<string> = new Set(['ready', 'delayed', 'running', 'suspended']);
|
|
34
|
+
/** Where an `alone` world files the harness's row: no body enqueues here, so no worker claims theirs. */
|
|
35
|
+
const HARNESS_QUEUE = 'x-sim-harness';
|
|
34
36
|
|
|
35
37
|
export interface SimWorldOptions {
|
|
36
38
|
/** Seeds `Math.random` for the run: a body's jitter and draws are the seed's. */
|
|
@@ -40,6 +42,11 @@ export interface SimWorldOptions {
|
|
|
40
42
|
decide(op: SimOp): SimDecision | Promise<SimDecision>;
|
|
41
43
|
/** Told every landed call, after the world has read it. */
|
|
42
44
|
landed?(op: SimOp, answer: unknown): void;
|
|
45
|
+
/**
|
|
46
|
+
* Only the row the harness filed is ever claimed: what a body enqueues lands, dedupes and is an
|
|
47
|
+
* `enqueue:` effect, and never runs — its effects are its own job's, not this run's.
|
|
48
|
+
*/
|
|
49
|
+
readonly alone?: boolean;
|
|
43
50
|
}
|
|
44
51
|
|
|
45
52
|
export interface SimWorld extends AsyncDisposable {
|
|
@@ -86,7 +93,10 @@ export async function openSimWorld(options: SimWorldOptions): Promise<SimWorld>
|
|
|
86
93
|
decide: (op) => options.decide(op),
|
|
87
94
|
landed: (op, answer) => {
|
|
88
95
|
remember(answer);
|
|
89
|
-
|
|
96
|
+
// Completed only: a `failed` or `started` record is not a step the body has moved past.
|
|
97
|
+
if (op.call === 'put' && op.step !== undefined && op.status === 'completed') {
|
|
98
|
+
log?.stepWritten(op.worker, op.step);
|
|
99
|
+
}
|
|
90
100
|
if ((op.call === 'enqueue' || op.call === 'enqueueMany') && log !== undefined) {
|
|
91
101
|
const results = (Array.isArray(answer) ? answer : [answer]) as readonly EnqueueResult[];
|
|
92
102
|
results.forEach((result, i) => {
|
|
@@ -112,9 +122,11 @@ export async function openSimWorld(options: SimWorldOptions): Promise<SimWorld>
|
|
|
112
122
|
const anonymous = ctxOf({ role: 'worker' });
|
|
113
123
|
const context = options.context ?? (() => anonymous);
|
|
114
124
|
|
|
115
|
-
const queues = (): readonly string[] =>
|
|
116
|
-
|
|
117
|
-
|
|
125
|
+
const queues = (): readonly string[] =>
|
|
126
|
+
options.alone === true
|
|
127
|
+
? [HARNESS_QUEUE]
|
|
128
|
+
: [...new Set([jobs.DEFAULT_QUEUE, ...jobs.registeredJobs().map((handle) => handle.queue)])];
|
|
129
|
+
const claimable = new Set(queues());
|
|
118
130
|
|
|
119
131
|
const rows = async (): Promise<readonly JobRecord[]> => {
|
|
120
132
|
const all: JobRecord[] = [];
|
|
@@ -143,7 +155,7 @@ export async function openSimWorld(options: SimWorldOptions): Promise<SimWorld>
|
|
|
143
155
|
enqueue: async (handle, input) => {
|
|
144
156
|
const result = await base.enqueue({
|
|
145
157
|
name: handle.name,
|
|
146
|
-
queue: handle.queue,
|
|
158
|
+
queue: options.alone === true ? HARNESS_QUEUE : handle.queue,
|
|
147
159
|
input,
|
|
148
160
|
idempotencyKey: handle.idempotencyKeyFor(input),
|
|
149
161
|
maxAttempts: jobs.enqueuedAttempts(handle.retry, handle.compensates),
|
|
@@ -176,7 +188,9 @@ export async function openSimWorld(options: SimWorldOptions): Promise<SimWorld>
|
|
|
176
188
|
rows,
|
|
177
189
|
nextDue: async () => {
|
|
178
190
|
const due = (await rows())
|
|
179
|
-
.filter(
|
|
191
|
+
.filter(
|
|
192
|
+
(row) => LIVE.has(row.state) && (options.alone !== true || claimable.has(row.queue)),
|
|
193
|
+
)
|
|
180
194
|
.map((row) => (row.state === 'running' ? (row.visibleAt ?? row.runAt) : row.runAt));
|
|
181
195
|
return due.length === 0 ? undefined : Math.min(...due);
|
|
182
196
|
},
|
package/src/mock-route.ts
CHANGED
|
@@ -79,3 +79,24 @@ export const routeMatches = (
|
|
|
79
79
|
const named = namedUrl(call.url, call.headers);
|
|
80
80
|
return named !== undefined && urlMatches(route.match, named);
|
|
81
81
|
};
|
|
82
|
+
|
|
83
|
+
/** A RegExp's literal text: a `.` in a host is `\.` in its source. */
|
|
84
|
+
const escaped = (text: string): string => text.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&');
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Whether `route` answers some URL on `hostname` — what makes the sealed resolver answer the name.
|
|
88
|
+
* A string names its host outright; a RegExp when it matches the host's origin or spells the host
|
|
89
|
+
* in its source (`/api\.vendor\.example\/v1/`), since no reader can enumerate what a pattern matches.
|
|
90
|
+
*/
|
|
91
|
+
export const routeNamesHost = (route: MockRoute, hostname: string): boolean => {
|
|
92
|
+
const host = hostname.toLowerCase();
|
|
93
|
+
if (route.match instanceof RegExp) {
|
|
94
|
+
const pattern = route.match;
|
|
95
|
+
const origins = [`https://${host}/`, `http://${host}/`];
|
|
96
|
+
if (origins.some((origin) => urlMatches(pattern, origin))) return true;
|
|
97
|
+
const source = pattern.source.toLowerCase();
|
|
98
|
+
return source.includes(escaped(host)) || source.includes(host);
|
|
99
|
+
}
|
|
100
|
+
const literal = route.match.endsWith('*') ? route.match.slice(0, -1) : route.match;
|
|
101
|
+
return URL.canParse(literal) && new URL(literal).hostname.toLowerCase() === host;
|
|
102
|
+
};
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// The resolver half of the sealed network: `Bun.dns.lookup`, `dns.lookup` and `dns.promises.lookup`.
|
|
2
|
+
// Only the patching lives here; WHAT a name resolves to is `sealed-network.ts`'s one gate, handed in,
|
|
3
|
+
// so a host `fetch` would answer by mock resolves and a host it would refuse does not. A named ESM
|
|
4
|
+
// import (`import { lookup } from 'node:dns'`) binds before the seal and is NOT covered (measured).
|
|
5
|
+
|
|
6
|
+
// why: `dns.lookup` and `dns.promises.lookup` are what an app's own resolver calls; Bun ships both.
|
|
7
|
+
import nodeDns from 'node:dns';
|
|
8
|
+
|
|
9
|
+
/** One lookup's verdict: the addresses to answer, a refusal to raise, or `real` for the resolver. */
|
|
10
|
+
export type DnsVerdict =
|
|
11
|
+
| { readonly answer: readonly string[] }
|
|
12
|
+
| { readonly refused: Error }
|
|
13
|
+
| { readonly real: true };
|
|
14
|
+
|
|
15
|
+
export type DnsGate = (hostname: string) => DnsVerdict;
|
|
16
|
+
|
|
17
|
+
interface LookupEntry {
|
|
18
|
+
readonly address: string;
|
|
19
|
+
readonly family: number;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const familyOf = (address: string): number => (address.includes(':') ? 6 : 4);
|
|
23
|
+
|
|
24
|
+
const entriesOf = (addresses: readonly string[]): LookupEntry[] =>
|
|
25
|
+
addresses.map((address) => ({ address, family: familyOf(address) }));
|
|
26
|
+
|
|
27
|
+
type NodeCallback = (error: unknown, address?: unknown, family?: number) => void;
|
|
28
|
+
|
|
29
|
+
/** Node's `(hostname, options?, callback)`; `options.all` answers the list, a number is a family. */
|
|
30
|
+
const wantsAll = (options: unknown): boolean =>
|
|
31
|
+
typeof options === 'object' && options !== null && 'all' in options && options.all === true;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Patch the three resolvers and return the uninstaller, which puts back exactly what was there.
|
|
35
|
+
* Assigned through `Reflect.set`: each is a writable data property the types declare a function.
|
|
36
|
+
*/
|
|
37
|
+
export function installDnsSeal(gate: DnsGate): () => void {
|
|
38
|
+
const originalBun = Reflect.get(Bun.dns, 'lookup') as typeof Bun.dns.lookup;
|
|
39
|
+
const originalNode = Reflect.get(nodeDns, 'lookup') as (...args: unknown[]) => void;
|
|
40
|
+
const originalPromises = Reflect.get(nodeDns.promises, 'lookup') as (
|
|
41
|
+
...args: unknown[]
|
|
42
|
+
) => Promise<unknown>;
|
|
43
|
+
|
|
44
|
+
const sealedBun = async (hostname: string, options?: unknown): Promise<unknown> => {
|
|
45
|
+
const verdict = gate(hostname);
|
|
46
|
+
if ('refused' in verdict) throw verdict.refused;
|
|
47
|
+
if ('answer' in verdict) return entriesOf(verdict.answer).map((one) => ({ ...one, ttl: 0 }));
|
|
48
|
+
return Reflect.apply(originalBun, Bun.dns, [hostname, options]);
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
const sealedNode = (hostname: string, ...rest: unknown[]): void => {
|
|
52
|
+
const verdict = gate(hostname);
|
|
53
|
+
if ('real' in verdict) {
|
|
54
|
+
Reflect.apply(originalNode, nodeDns, [hostname, ...rest]);
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
const callback = rest.findLast((one) => typeof one === 'function') as NodeCallback | undefined;
|
|
58
|
+
const options = rest.length > 1 ? rest[0] : undefined;
|
|
59
|
+
// A real lookup answers on a later tick, and a caller may rely on it: never synchronously.
|
|
60
|
+
queueMicrotask(() => {
|
|
61
|
+
if ('refused' in verdict) return callback?.(verdict.refused);
|
|
62
|
+
const entries = entriesOf(verdict.answer);
|
|
63
|
+
if (wantsAll(options)) return callback?.(null, entries);
|
|
64
|
+
return callback?.(null, entries[0]?.address, entries[0]?.family);
|
|
65
|
+
});
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
const sealedPromises = async (hostname: string, options?: unknown): Promise<unknown> => {
|
|
69
|
+
const verdict = gate(hostname);
|
|
70
|
+
if ('refused' in verdict) throw verdict.refused;
|
|
71
|
+
if ('real' in verdict) {
|
|
72
|
+
return Reflect.apply(originalPromises, nodeDns.promises, [hostname, options]);
|
|
73
|
+
}
|
|
74
|
+
const entries = entriesOf(verdict.answer);
|
|
75
|
+
return wantsAll(options) ? entries : entries[0];
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
Reflect.set(Bun.dns, 'lookup', sealedBun);
|
|
79
|
+
Reflect.set(nodeDns, 'lookup', sealedNode);
|
|
80
|
+
Reflect.set(nodeDns.promises, 'lookup', sealedPromises);
|
|
81
|
+
return () => {
|
|
82
|
+
Reflect.set(Bun.dns, 'lookup', originalBun);
|
|
83
|
+
Reflect.set(nodeDns, 'lookup', originalNode);
|
|
84
|
+
Reflect.set(nodeDns.promises, 'lookup', originalPromises);
|
|
85
|
+
};
|
|
86
|
+
}
|
package/src/sealed-network.ts
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
|
-
// Sealed network
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
// `fetch` here, `WebSocket` and `Bun.connect` through `sealed-sockets.ts`, all behind one gate.
|
|
1
|
+
// Sealed network: egress a test did not mock or allow fails with the URL and the line that fixes
|
|
2
|
+
// it, so nothing reaches the internet unreproducibly. `fetch` here, `WebSocket`/`Bun.connect` via
|
|
3
|
+
// `sealed-sockets.ts`, name resolution via `sealed-dns.ts` — one gate, one set of mocks and allows.
|
|
5
4
|
|
|
6
5
|
import { isSelfOrigin, peerAddressClass } from '@ultimat3/core';
|
|
7
6
|
import { NetworkOfflineError, NetworkRaceError, NetworkSealedError } from './errors';
|
|
8
7
|
import type { MockHandler, MockRoute } from './mock-route';
|
|
9
|
-
import { mockRoute, routeMatches } from './mock-route';
|
|
8
|
+
import { mockRoute, routeMatches, routeNamesHost } from './mock-route';
|
|
10
9
|
import type { LoggedRequest } from './request-log';
|
|
11
10
|
import { hasStreamBody, headersOf, isOpenEnded, readBody, requestProbe } from './request-log';
|
|
11
|
+
import type { DnsVerdict } from './sealed-dns';
|
|
12
|
+
import { installDnsSeal } from './sealed-dns';
|
|
12
13
|
import { installSocketSeal } from './sealed-sockets';
|
|
13
14
|
|
|
14
15
|
export type FetchLike = typeof globalThis.fetch;
|
|
@@ -26,6 +27,8 @@ interface SealState {
|
|
|
26
27
|
original: FetchLike | undefined;
|
|
27
28
|
/** `installSocketSeal`'s undo, held beside `original` so one unseal restores all three dials. */
|
|
28
29
|
restoreSockets: (() => void) | undefined;
|
|
30
|
+
/** `installDnsSeal`'s undo, likewise. */
|
|
31
|
+
restoreDns: (() => void) | undefined;
|
|
29
32
|
network: NetworkState;
|
|
30
33
|
}
|
|
31
34
|
|
|
@@ -35,9 +38,17 @@ const state: SealState = {
|
|
|
35
38
|
log: [],
|
|
36
39
|
original: undefined,
|
|
37
40
|
restoreSockets: undefined,
|
|
41
|
+
restoreDns: undefined,
|
|
38
42
|
network: 'online',
|
|
39
43
|
};
|
|
40
44
|
|
|
45
|
+
/**
|
|
46
|
+
* What a mocked host resolves to: one `public` address (`classifyAddress`), so an SSRF screen
|
|
47
|
+
* passes it, and never a host network — a screen refuses those. Never dialled: a dial pinned to it
|
|
48
|
+
* carries the name in `Host`, which is how the mock answers it (`mock-route.ts`'s `namedUrl`).
|
|
49
|
+
*/
|
|
50
|
+
export const MOCKED_HOST_ADDRESS = '11.11.11.11';
|
|
51
|
+
|
|
41
52
|
const urlOf = (input: RequestInfo | URL): string => {
|
|
42
53
|
if (typeof input === 'string') return input;
|
|
43
54
|
if (input instanceof URL) return input.toString();
|
|
@@ -82,12 +93,40 @@ const socketRefusal = (url: string, method: string): Error | undefined => {
|
|
|
82
93
|
return new NetworkSealedError({ url, method, allowed: [...state.allowed], transport: 'socket' });
|
|
83
94
|
};
|
|
84
95
|
|
|
85
|
-
/**
|
|
86
|
-
|
|
96
|
+
/** A host the allow-list names, whatever port it was allowed on: a lookup carries no port. */
|
|
97
|
+
const isAllowedName = (hostname: string): boolean =>
|
|
98
|
+
[...state.allowed].some((host) => safeHost(`https://${host}`)?.split(':')[0] === hostname);
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* The resolver's gate, in `fetch`'s order: offline refuses every name, this machine and an allowed
|
|
102
|
+
* host resolve for real, a host a mock answers resolves to `MOCKED_HOST_ADDRESS`, the rest is
|
|
103
|
+
* `X_TEST_NETWORK_SEALED` — the code `fetch` would have raised a moment later.
|
|
104
|
+
*/
|
|
105
|
+
const dnsVerdict = (hostname: string): DnsVerdict => {
|
|
106
|
+
const name = hostname.toLowerCase().replace(/^\[|\]$/g, '');
|
|
107
|
+
const url = `dns://${name}`;
|
|
108
|
+
if (state.network !== 'online') {
|
|
109
|
+
return { refused: new NetworkOfflineError({ url, method: 'LOOKUP', mode: state.network }) };
|
|
110
|
+
}
|
|
111
|
+
if (isThisMachine(name) || peerAddressClass(name) !== undefined || isAllowedName(name)) {
|
|
112
|
+
return { real: true };
|
|
113
|
+
}
|
|
114
|
+
if (state.mocks.some((route) => routeNamesHost(route, name))) {
|
|
115
|
+
return { answer: [MOCKED_HOST_ADDRESS] };
|
|
116
|
+
}
|
|
117
|
+
const allowed = [...state.allowed];
|
|
118
|
+
return {
|
|
119
|
+
refused: new NetworkSealedError({ url, method: 'LOOKUP', allowed, transport: 'dns' }),
|
|
120
|
+
};
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
/** Replace global fetch, `WebSocket`, `Bun.connect` and the resolvers. Idempotent: sealing twice
|
|
124
|
+
* keeps the one set of originals around. */
|
|
87
125
|
export function sealNetwork(): void {
|
|
88
126
|
if (state.original !== undefined) return;
|
|
89
127
|
state.original = globalThis.fetch;
|
|
90
128
|
state.restoreSockets = installSocketSeal(socketRefusal);
|
|
129
|
+
state.restoreDns = installDnsSeal(dnsVerdict);
|
|
91
130
|
globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
|
|
92
131
|
const url = urlOf(input);
|
|
93
132
|
const method = methodOf(input, init);
|
|
@@ -151,6 +190,8 @@ export function unsealNetwork(): void {
|
|
|
151
190
|
state.original = undefined;
|
|
152
191
|
state.restoreSockets?.();
|
|
153
192
|
state.restoreSockets = undefined;
|
|
193
|
+
state.restoreDns?.();
|
|
194
|
+
state.restoreDns = undefined;
|
|
154
195
|
}
|
|
155
196
|
|
|
156
197
|
/** Whether the patch is installed. The `network` fixture seals before going offline, so that
|
package/src/seed-graph-values.ts
CHANGED
|
@@ -152,6 +152,12 @@ const parsed = (column: SeedColumn, value: unknown): { readonly ok: unknown } |
|
|
|
152
152
|
}
|
|
153
153
|
};
|
|
154
154
|
|
|
155
|
+
/**
|
|
156
|
+
* What every driver binds as SQL NULL — a JSON `null` included (`pg-row.ts` `bindable`): `t.json()`
|
|
157
|
+
* accepts it and the sampler draws it, so a NOT NULL `jsonb` column must never be offered one.
|
|
158
|
+
*/
|
|
159
|
+
const storesAsSqlNull = (value: unknown): boolean => value === null || value === undefined;
|
|
160
|
+
|
|
155
161
|
/**
|
|
156
162
|
* Every storable value for this column, most plausible first: the declared default, then what the
|
|
157
163
|
* kind offers, then `null` when the column allows it. Empty means nothing generated is storable.
|
|
@@ -167,9 +173,8 @@ export function candidatesFor(
|
|
|
167
173
|
const out: unknown[] = [];
|
|
168
174
|
for (const value of raw) {
|
|
169
175
|
const result = parsed(column, value);
|
|
170
|
-
if (result
|
|
171
|
-
|
|
172
|
-
}
|
|
176
|
+
if (result === undefined || (meta.notNull && storesAsSqlNull(result.ok))) continue;
|
|
177
|
+
if (!out.some((seen) => Bun.deepEquals(seen, result.ok))) out.push(result.ok);
|
|
173
178
|
}
|
|
174
179
|
return keepsPatternsFirst(out, ctx.patterns ?? []);
|
|
175
180
|
}
|