@ultimat3/testing 30.6.0 → 30.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +3 -3
- package/package.json +17 -17
- package/src/define-exercise.ts +19 -7
- package/src/errors.ts +6 -6
- package/src/exercise-actor-user.ts +63 -0
- package/src/exercise-actor.ts +36 -7
- package/src/exercise-crash.ts +7 -3
- package/src/exercise-drift-hint.ts +28 -0
- package/src/exercise-input.ts +3 -0
- package/src/exercise-job-tenant.ts +10 -5
- package/src/exercise-override-input.ts +26 -5
- package/src/exercise-report.ts +38 -2
- package/src/exercise-sweeps.ts +49 -7
- package/src/exercise.ts +41 -11
- package/src/index.ts +4 -0
- package/src/job-crash-points.ts +36 -0
- package/src/job-crash-proof.ts +23 -6
- package/src/job-sim-errors.ts +24 -0
- package/src/plan-conditions.ts +106 -0
- package/src/plan-oracle.ts +65 -49
- package/src/plan-sort-key.ts +65 -0
- package/src/plan-sweep-fix.ts +87 -21
- package/src/plan-sweep-report.ts +48 -4
- package/src/seed-graph-values.ts +20 -11
- package/src/seed-graph.ts +30 -1
- package/src/tenant-proof-report.ts +12 -3
- package/src/tenant-proof.ts +32 -4
- package/src/seed-pattern-sample.ts +0 -216
package/src/exercise.ts
CHANGED
|
@@ -13,8 +13,10 @@ import type { ExerciseOverride, ExerciseOverrides } from './define-exercise';
|
|
|
13
13
|
import { assertOverridesKnown, unknownDeclaration } from './define-exercise';
|
|
14
14
|
import type { RoleGrants } from './exercise-actor';
|
|
15
15
|
import { chooseActor } from './exercise-actor';
|
|
16
|
+
import { seatActorUsers } from './exercise-actor-user';
|
|
16
17
|
import { proveCrash } from './exercise-crash';
|
|
17
18
|
import { defectDetail } from './exercise-defect';
|
|
19
|
+
import { driftOverrideLine } from './exercise-drift-hint';
|
|
18
20
|
import type { IdLookup, Sampler } from './exercise-input';
|
|
19
21
|
import {
|
|
20
22
|
bindIds,
|
|
@@ -30,6 +32,7 @@ import { overrideInput } from './exercise-override-input';
|
|
|
30
32
|
import type {
|
|
31
33
|
EffectPayload,
|
|
32
34
|
ExerciseEffect,
|
|
35
|
+
ExercisePhaseMs,
|
|
33
36
|
ExerciseRecord,
|
|
34
37
|
ExerciseRoom,
|
|
35
38
|
Observed,
|
|
@@ -61,6 +64,8 @@ export interface ExercisePassOptions {
|
|
|
61
64
|
/** One primitive by name; unknown is `X_DECLARATION_UNKNOWN`. */
|
|
62
65
|
readonly only?: string;
|
|
63
66
|
readonly roles: RoleGrants;
|
|
67
|
+
/** Told once, as the pass returns: how long each phase took, so a slow pass names its slow part. */
|
|
68
|
+
readonly onPhases?: (ms: ExercisePhaseMs) => void;
|
|
64
69
|
}
|
|
65
70
|
|
|
66
71
|
const isOff = (overrides: ExerciseOverrides | undefined, name: string): boolean => {
|
|
@@ -110,7 +115,7 @@ interface Run {
|
|
|
110
115
|
readonly lastInput: unknown;
|
|
111
116
|
}
|
|
112
117
|
|
|
113
|
-
function judged(subject: Subject, run: Run, observed: Observed) {
|
|
118
|
+
function judged(subject: Subject, run: Run, observed: Observed, sampler: Sampler) {
|
|
114
119
|
const idFields = (input: unknown): readonly string[] =>
|
|
115
120
|
typeof input === 'object' && input !== null
|
|
116
121
|
? Object.keys(input).filter((key) => idStem(key) !== undefined)
|
|
@@ -128,7 +133,12 @@ function judged(subject: Subject, run: Run, observed: Observed) {
|
|
|
128
133
|
}
|
|
129
134
|
if (run.verdict.kind === 'pass' && run.answered) return { outcome: 'reached' as const };
|
|
130
135
|
if (run.verdict.kind === 'drift') {
|
|
131
|
-
|
|
136
|
+
// The field no draw can fill, named: `...drawn` alone told the author nothing to write.
|
|
137
|
+
const fields = idFields(run.lastInput);
|
|
138
|
+
return {
|
|
139
|
+
...unreached('X_CONTRACT_DRIFT', 'its input schema refused every generated value'),
|
|
140
|
+
override: driftOverrideLine(subject.name, subject.input, sampler, fields),
|
|
141
|
+
};
|
|
132
142
|
}
|
|
133
143
|
if (run.verdict.kind === 'defect') {
|
|
134
144
|
return unreached(codeOf(run.verdict.error) ?? 'X_CONTRACT_DEFECT', run.verdict.defect);
|
|
@@ -198,7 +208,7 @@ async function control(
|
|
|
198
208
|
const ms = Math.round(performance.now() - started);
|
|
199
209
|
const observed = room.end();
|
|
200
210
|
const effects = [...observed.effects, ...(await enqueued(subject, jobs))];
|
|
201
|
-
const verdictOf = judged(subject, { verdict, answered, lastError, lastInput }, observed);
|
|
211
|
+
const verdictOf = judged(subject, { verdict, answered, lastError, lastInput }, observed, sampler);
|
|
202
212
|
// A 5xx's cause, statement and fields: the code alone sent the author to reproduce it by hand.
|
|
203
213
|
const defect =
|
|
204
214
|
verdict.kind === 'defect' && verdictOf.outcome === 'unreached'
|
|
@@ -272,10 +282,13 @@ async function exerciseOne(
|
|
|
272
282
|
const intruder = withoutCrossTenant(
|
|
273
283
|
theirs.kind === 'actor' ? seated(theirs.actor, other, rows) : null,
|
|
274
284
|
);
|
|
285
|
+
// Both are users, as a request's actor always is: their `x_users` rows, before any call.
|
|
286
|
+
await seatActorUsers([actor, intruder]);
|
|
275
287
|
|
|
276
288
|
const scope = { tenant, actorId: actor?.id, subject: name, actorEntity: rows?.entity.$name };
|
|
277
289
|
const bindInput = overrideInput(name, override);
|
|
278
|
-
|
|
290
|
+
// `off` returned above: any other entry's `runs` is declared, with or without an `actor`.
|
|
291
|
+
const declaredRuns = override === undefined || 'off' in override ? undefined : override.runs;
|
|
279
292
|
// A field naming the actor entity is tried as the actor's own row, then as somebody else's:
|
|
280
293
|
// both readings are legitimate and only the handler knows which it meant. Same minimal draw.
|
|
281
294
|
const twoWay =
|
|
@@ -311,12 +324,14 @@ async function exerciseOne(
|
|
|
311
324
|
}
|
|
312
325
|
const input = seat === undefined ? given : seat.input;
|
|
313
326
|
let reachedWith: unknown;
|
|
327
|
+
const tenantStarted = performance.now();
|
|
314
328
|
const record = await proveTenant({
|
|
315
329
|
taint: tenantTaint(graph, tenant),
|
|
316
330
|
input,
|
|
317
331
|
public: publicReason(subject.policy),
|
|
318
332
|
read: kind === 'query',
|
|
319
333
|
sweep: kind === 'job',
|
|
334
|
+
tenantless: seat?.kind === 'none',
|
|
320
335
|
attack: () => attack(subject, input, intruder, other, room),
|
|
321
336
|
control: async () => {
|
|
322
337
|
const done = await control(run, room, sampler);
|
|
@@ -325,10 +340,16 @@ async function exerciseOne(
|
|
|
325
340
|
},
|
|
326
341
|
snapshot: (seeded) => options.writer.snapshot(seeded),
|
|
327
342
|
});
|
|
343
|
+
// The control's own call is `ms`: the tenant proof's time is everything around it.
|
|
344
|
+
const tenantMs = Math.max(0, Math.round(performance.now() - tenantStarted) - record.ms);
|
|
328
345
|
// After the tenant proof, outside the room's begin/end: the crash runs' effects are their own.
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
346
|
+
if (subject.handle === undefined || record.outcome !== 'reached') {
|
|
347
|
+
return { ...record, proofMs: { tenant: tenantMs } };
|
|
348
|
+
}
|
|
349
|
+
const crashStarted = performance.now();
|
|
350
|
+
const crash = await proveCrash(subject.handle, reachedWith, actor);
|
|
351
|
+
const proofMs = { tenant: tenantMs, crash: Math.round(performance.now() - crashStarted) };
|
|
352
|
+
return { ...record, crash, proofMs };
|
|
332
353
|
}
|
|
333
354
|
|
|
334
355
|
export async function runExercisePass(
|
|
@@ -349,6 +370,8 @@ export async function runExercisePass(
|
|
|
349
370
|
const live = chosen.filter((subject) => !isOff(options.overrides, subject.name));
|
|
350
371
|
const records: ExerciseRecord[] = [];
|
|
351
372
|
let graph: SeedGraph;
|
|
373
|
+
const seedStarted = performance.now();
|
|
374
|
+
let primitivesStarted = seedStarted;
|
|
352
375
|
try {
|
|
353
376
|
// Seeded before the first primitive runs: a failure here is the pass's, never one primitive's.
|
|
354
377
|
graph = await seedPass(options.entities, options.writer, {
|
|
@@ -358,6 +381,7 @@ export async function runExercisePass(
|
|
|
358
381
|
? {}
|
|
359
382
|
: { factories: options.overrides.factories }),
|
|
360
383
|
});
|
|
384
|
+
primitivesStarted = performance.now();
|
|
361
385
|
// In order, one at a time: the seams the room installs are the process's, not the primitive's.
|
|
362
386
|
for (const subject of chosen) {
|
|
363
387
|
const index = live.indexOf(subject);
|
|
@@ -369,9 +393,11 @@ export async function runExercisePass(
|
|
|
369
393
|
await room[Symbol.asyncDispose]();
|
|
370
394
|
throw error;
|
|
371
395
|
}
|
|
396
|
+
const seed = Math.round(primitivesStarted - seedStarted);
|
|
397
|
+
const primitives = Math.round(performance.now() - primitivesStarted);
|
|
372
398
|
// The sweeps: every reached primitive again at two sizes, then every statement planned.
|
|
373
399
|
const reached = new Set(records.filter((r) => r.outcome === 'reached').map((r) => r.name));
|
|
374
|
-
|
|
400
|
+
const swept = await attachSweeps(records, {
|
|
375
401
|
subjects: chosen.filter((subject) => reached.has(subject.name)),
|
|
376
402
|
entities: options.entities,
|
|
377
403
|
// The sized graphs close their cycles as the pass's did: a refused deferred key stays null.
|
|
@@ -382,9 +408,13 @@ export async function runExercisePass(
|
|
|
382
408
|
factories: options.overrides?.factories,
|
|
383
409
|
setup: async (subject, sized, tenant, self) => {
|
|
384
410
|
const full = chosen.find((one) => one.name === subject.name);
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
411
|
+
if (full === undefined) return undefined;
|
|
412
|
+
const call = await sizedCall(full, options, sampler, { graph: sized, tenant, self });
|
|
413
|
+
// Before the sweep's room opens: the sized tenant's actor is a user too.
|
|
414
|
+
if (call !== undefined) await seatActorUsers([call.actor]);
|
|
415
|
+
return call;
|
|
388
416
|
},
|
|
389
417
|
});
|
|
418
|
+
options.onPhases?.({ seed, primitives, ...swept.ms });
|
|
419
|
+
return swept.records;
|
|
390
420
|
}
|
package/src/index.ts
CHANGED
|
@@ -207,7 +207,9 @@ export type {
|
|
|
207
207
|
EffectPayload,
|
|
208
208
|
ExerciseEffect,
|
|
209
209
|
ExerciseOutcome,
|
|
210
|
+
ExercisePhaseMs,
|
|
210
211
|
ExerciseRecord,
|
|
212
|
+
ProofMs,
|
|
211
213
|
TenantFinding,
|
|
212
214
|
TenantLeg,
|
|
213
215
|
TenantProof,
|
|
@@ -366,6 +368,7 @@ export {
|
|
|
366
368
|
normalizeIslandName,
|
|
367
369
|
} from './island-states-resolve';
|
|
368
370
|
export { ISOLATED_ENV, releasePluginsAfterIsolatedFile } from './isolated-plugins';
|
|
371
|
+
export { CRASH_POINT_BOUND } from './job-crash-points';
|
|
369
372
|
export type {
|
|
370
373
|
CrashBoundary,
|
|
371
374
|
CrashDivergence,
|
|
@@ -379,6 +382,7 @@ export type { SimEffect } from './job-sim-effects';
|
|
|
379
382
|
export {
|
|
380
383
|
crashEffectPlace,
|
|
381
384
|
JobCrashDivergedError,
|
|
385
|
+
JobCrashUnboundedError,
|
|
382
386
|
JobSimInvariantError,
|
|
383
387
|
JobSimNondeterministicError,
|
|
384
388
|
} from './job-sim-errors';
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// Which boundaries a crash proof kills at. A loop's steps named by row id are one point of the
|
|
2
|
+
// code, not one per row: each boundary shape is killed at its first and its last occurrence, so the
|
|
3
|
+
// proof costs what the job's code has, never what the database holds. Pure: labels in, indexes out.
|
|
4
|
+
|
|
5
|
+
/** Kill points one mode of a proof may cost; past it the proof refuses, `X_JOB_CRASH_UNBOUNDED`. */
|
|
6
|
+
export const CRASH_POINT_BOUND = 32;
|
|
7
|
+
|
|
8
|
+
const UUID = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi;
|
|
9
|
+
/** A ULID, a hex digest, a cuid: eight or more letters and digits with a digit among them. */
|
|
10
|
+
const TOKEN = /\b(?=[A-Za-z]*\d)[A-Za-z0-9]{8,}\b/g;
|
|
11
|
+
const DIGITS = /\d+/g;
|
|
12
|
+
|
|
13
|
+
/** `fanout:<id>`, `page-<n>`, `sleep:<n>h (sleeping)`: a boundary label with its ids taken out. */
|
|
14
|
+
export const boundaryShape = (label: string): string =>
|
|
15
|
+
label.replace(UUID, '<id>').replace(TOKEN, '<id>').replace(DIGITS, '<n>');
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The 1-based positions to kill at, in order: the first and the last boundary of every shape.
|
|
19
|
+
* The first proves the code before the loop and the loop's own body; the last, every row before it
|
|
20
|
+
* replayed and the code after it — an effect between any two rows is made again at the last.
|
|
21
|
+
*/
|
|
22
|
+
export function crashPoints(labels: readonly string[]): readonly number[] {
|
|
23
|
+
const first = new Map<string, number>();
|
|
24
|
+
const last = new Map<string, number>();
|
|
25
|
+
labels.forEach((label, i) => {
|
|
26
|
+
const shape = boundaryShape(label);
|
|
27
|
+
if (!first.has(shape)) first.set(shape, i + 1);
|
|
28
|
+
last.set(shape, i + 1);
|
|
29
|
+
});
|
|
30
|
+
return [...new Set([...first.values(), ...last.values()])].sort((a, b) => a - b);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** The distinct shapes of `labels`, first seen first — what an unbounded refusal names. */
|
|
34
|
+
export const boundaryShapes = (labels: readonly string[]): readonly string[] => [
|
|
35
|
+
...new Set(labels.map(boundaryShape)),
|
|
36
|
+
];
|
package/src/job-crash-proof.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
// `crashProof(handle, input)`: one clean run, then one run killed at each boundary the
|
|
2
|
-
// crossed —
|
|
3
|
-
// lost. Judged by the effect log: a killed run that makes an effect more often than the clean one
|
|
1
|
+
// `crashProof(handle, input)`: one clean run, then one run killed at each distinct boundary the
|
|
2
|
+
// clean run crossed — a step write landing, a settle; a loop's rows one point (`job-crash-points.ts`)
|
|
3
|
+
// — and one more where that write's answer is lost. Judged by the effect log: a killed run that makes an effect more often than the clean one
|
|
4
4
|
// is `X_JOB_CRASH_DIVERGED`; a lost answer rerunning a step is at-least-once, reported, never red.
|
|
5
5
|
|
|
6
6
|
import type { Ctx } from '@ultimat3/core';
|
|
@@ -8,11 +8,12 @@ import { finiteCount } from '@ultimat3/core';
|
|
|
8
8
|
import type { JobHandle } from '@ultimat3/jobs';
|
|
9
9
|
import { COMPENSATE_PREFIX, UNWINDING_STEP } from '@ultimat3/jobs';
|
|
10
10
|
import { advanceClock, frozenNow } from './determinism';
|
|
11
|
+
import { boundaryShapes, CRASH_POINT_BOUND, crashPoints } from './job-crash-points';
|
|
11
12
|
import type { SimDecision, SimOp } from './job-sim-driver';
|
|
12
13
|
import { SEND_AND_ANSWER } from './job-sim-driver';
|
|
13
14
|
import type { SimEffect } from './job-sim-effects';
|
|
14
15
|
import { effectCounts } from './job-sim-effects';
|
|
15
|
-
import { JobCrashDivergedError } from './job-sim-errors';
|
|
16
|
+
import { JobCrashDivergedError, JobCrashUnboundedError } from './job-sim-errors';
|
|
16
17
|
import type { SimWorld } from './job-sim-world';
|
|
17
18
|
import { openSimWorld } from './job-sim-world';
|
|
18
19
|
|
|
@@ -47,7 +48,10 @@ export interface CrashRerun {
|
|
|
47
48
|
}
|
|
48
49
|
|
|
49
50
|
export interface CrashReport {
|
|
51
|
+
/** The boundaries killed at: each distinct one the clean run crossed, a loop's rows as one. */
|
|
50
52
|
readonly boundaries: readonly CrashBoundary[];
|
|
53
|
+
/** Every boundary the clean run crossed, the forced-fail run's included: what the kills stand for. */
|
|
54
|
+
readonly crossed: number;
|
|
51
55
|
readonly divergences: readonly CrashDivergence[];
|
|
52
56
|
readonly reruns: readonly CrashRerun[];
|
|
53
57
|
}
|
|
@@ -212,6 +216,7 @@ export async function crashProof<I>(
|
|
|
212
216
|
const boundaries: CrashBoundary[] = [];
|
|
213
217
|
const divergences: CrashDivergence[] = [];
|
|
214
218
|
const reruns: CrashRerun[] = [];
|
|
219
|
+
let crossed = 0;
|
|
215
220
|
const modes: readonly CrashBoundary['mode'][] = handle.compensates
|
|
216
221
|
? ['clean', 'forced-fail']
|
|
217
222
|
: ['clean'];
|
|
@@ -228,7 +233,19 @@ export async function crashProof<I>(
|
|
|
228
233
|
: await runOnce(handle, input, { forceFail }, options);
|
|
229
234
|
const afterOf = (key: string): string | undefined =>
|
|
230
235
|
clean.effects.find((effect) => effect.key === key)?.after;
|
|
231
|
-
|
|
236
|
+
// Judged before the first kill: a proof past its bound costs one clean run, then refuses.
|
|
237
|
+
const points = crashPoints(clean.boundaries);
|
|
238
|
+
crossed += clean.boundaries.length;
|
|
239
|
+
if (points.length > CRASH_POINT_BOUND) {
|
|
240
|
+
throw new JobCrashUnboundedError({
|
|
241
|
+
job: handle.name,
|
|
242
|
+
points: points.length,
|
|
243
|
+
bound: CRASH_POINT_BOUND,
|
|
244
|
+
crossed: clean.boundaries.length,
|
|
245
|
+
shapes: boundaryShapes(clean.boundaries),
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
for (const at of points) {
|
|
232
249
|
const label = clean.boundaries[at - 1] ?? '?';
|
|
233
250
|
const killed = await runOnce(handle, input, { at, fault: 'kill', forceFail }, options);
|
|
234
251
|
boundaries.push({
|
|
@@ -272,5 +289,5 @@ export async function crashProof<I>(
|
|
|
272
289
|
}
|
|
273
290
|
}
|
|
274
291
|
}
|
|
275
|
-
return { boundaries, divergences, reruns };
|
|
292
|
+
return { boundaries, crossed, divergences, reruns };
|
|
276
293
|
}
|
package/src/job-sim-errors.ts
CHANGED
|
@@ -101,3 +101,27 @@ export class JobSimNondeterministicError extends UltimateError {
|
|
|
101
101
|
});
|
|
102
102
|
}
|
|
103
103
|
}
|
|
104
|
+
|
|
105
|
+
export interface CrashUnboundedInput {
|
|
106
|
+
readonly job: string;
|
|
107
|
+
/** Kill points the clean run's boundaries asked for, after a loop's rows were taken as one. */
|
|
108
|
+
readonly points: number;
|
|
109
|
+
readonly bound: number;
|
|
110
|
+
/** Boundaries the clean run crossed. */
|
|
111
|
+
readonly crossed: number;
|
|
112
|
+
/** The first distinct boundary shapes, in order: where the names that did not collapse are. */
|
|
113
|
+
readonly shapes: readonly string[];
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Refused before the first kill: a proof that grows with the rows is one no gate can wait for. */
|
|
117
|
+
export class JobCrashUnboundedError extends UltimateError {
|
|
118
|
+
constructor(input: CrashUnboundedInput) {
|
|
119
|
+
const shown = input.shapes.slice(0, 5).join(', ');
|
|
120
|
+
super({
|
|
121
|
+
code: 'X_JOB_CRASH_UNBOUNDED',
|
|
122
|
+
cause: `job ${renderCauseValue(input.job)} crossed ${input.crossed} step boundaries, ${input.points} distinct kill points against a bound of ${input.bound} — its steps are named so a loop's rows do not read as one point (first: ${renderCauseValue(shown)})`,
|
|
123
|
+
fix: `step.run(\`<name>:\${row.id}\`, async () => …) # in job(${renderFixLiteral(input.job, '<job>')}): name a per-row step by the row's id, so its loop is one kill point; then: x test job --filter <the test file>`,
|
|
124
|
+
meta: { ...input, shapes: [...input.shapes] },
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// A plan node's conditions, read as the columns one scanned table compares: by equality, by any
|
|
2
|
+
// other way, and per arm of an OR. An OR is served by one index per arm (a BitmapOr), never by one
|
|
3
|
+
// index over every column it names, so its arms are kept apart from the conjuncts around it.
|
|
4
|
+
|
|
5
|
+
/** The scan whose columns are read: its table and the alias the plan qualifies it by. */
|
|
6
|
+
export interface ConditionScan {
|
|
7
|
+
readonly table: string | undefined;
|
|
8
|
+
readonly alias: string | undefined;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface ConditionColumns {
|
|
12
|
+
readonly equality: string[];
|
|
13
|
+
readonly range: string[];
|
|
14
|
+
/** The first OR whose every arm compares a column of the table: each arm's columns. */
|
|
15
|
+
readonly arms?: readonly (readonly string[])[];
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const IDENT = /(?:"?([A-Za-z_][\w$]*)"?\.)?"?([A-Za-z_][\w$]*)"?/g;
|
|
19
|
+
const EQUALITY = /(?<![<>!])=(?!>)/;
|
|
20
|
+
/** `IS NULL` pins one value as `=` does, and a btree serves it as an index condition. */
|
|
21
|
+
const IS_NULL = /\bIS\s+NULL\b/i;
|
|
22
|
+
|
|
23
|
+
/** The parts of `text` between `word`s that stand outside every parenthesis and literal. */
|
|
24
|
+
function splitTop(text: string, word: 'AND' | 'OR'): string[] {
|
|
25
|
+
const parts: string[] = [];
|
|
26
|
+
let depth = 0;
|
|
27
|
+
let quoted = false;
|
|
28
|
+
let start = 0;
|
|
29
|
+
for (let at = 0; at < text.length; at += 1) {
|
|
30
|
+
const char = text[at];
|
|
31
|
+
if (char === "'") quoted = !quoted;
|
|
32
|
+
if (quoted) continue;
|
|
33
|
+
if (char === '(') depth += 1;
|
|
34
|
+
else if (char === ')') depth -= 1;
|
|
35
|
+
else if (depth === 0 && text.startsWith(` ${word} `, at)) {
|
|
36
|
+
parts.push(text.slice(start, at));
|
|
37
|
+
start = at + word.length + 2;
|
|
38
|
+
at = start - 1;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
parts.push(text.slice(start));
|
|
42
|
+
return parts.map(unwrap);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** `((a = $1))` → `a = $1`: parentheses that wrap the whole text, and only those. */
|
|
46
|
+
function unwrap(text: string): string {
|
|
47
|
+
let rest = text.trim();
|
|
48
|
+
for (;;) {
|
|
49
|
+
if (!rest.startsWith('(') || !rest.endsWith(')')) return rest;
|
|
50
|
+
let depth = 0;
|
|
51
|
+
for (let at = 0; at < rest.length; at += 1) {
|
|
52
|
+
if (rest[at] === '(') depth += 1;
|
|
53
|
+
else if (rest[at] === ')') depth -= 1;
|
|
54
|
+
if (depth === 0 && at < rest.length - 1) return rest;
|
|
55
|
+
}
|
|
56
|
+
rest = rest.slice(1, -1).trim();
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** The columns of the table a conjunct names; a name qualified by another alias is not its. */
|
|
61
|
+
function namesIn(conjunct: string, scan: ConditionScan, columns: ReadonlySet<string>): string[] {
|
|
62
|
+
const names: string[] = [];
|
|
63
|
+
for (const [, qualifier, name] of conjunct.replace(/'[^']*'/g, '').matchAll(IDENT)) {
|
|
64
|
+
if (name === undefined || !columns.has(name)) continue;
|
|
65
|
+
if (qualifier !== undefined && qualifier !== scan.alias && qualifier !== scan.table) continue;
|
|
66
|
+
if (!names.includes(name)) names.push(name);
|
|
67
|
+
}
|
|
68
|
+
return names;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* The columns of `scan` the conditions name, split by how they are compared: `=` and `IS NULL`
|
|
73
|
+
* are equality, anything else a range. A conjunct qualified by another alias is a join's other
|
|
74
|
+
* side; an unqualified name counts when the table has the column. An OR is no conjunct of either.
|
|
75
|
+
*/
|
|
76
|
+
export function columnsIn(
|
|
77
|
+
conditions: readonly string[],
|
|
78
|
+
scan: ConditionScan,
|
|
79
|
+
columns: ReadonlySet<string>,
|
|
80
|
+
): ConditionColumns {
|
|
81
|
+
const equality: string[] = [];
|
|
82
|
+
const range: string[] = [];
|
|
83
|
+
let arms: string[][] | undefined;
|
|
84
|
+
for (const condition of conditions) {
|
|
85
|
+
for (const conjunct of splitTop(unwrap(condition), 'AND')) {
|
|
86
|
+
const alternatives = splitTop(conjunct, 'OR');
|
|
87
|
+
if (alternatives.length > 1) {
|
|
88
|
+
const each = alternatives.map((arm) => {
|
|
89
|
+
const found = columnsIn([arm], scan, columns);
|
|
90
|
+
return [...found.equality, ...found.range];
|
|
91
|
+
});
|
|
92
|
+
if (arms === undefined && each.every((arm) => arm.length > 0)) arms = each;
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
const into = EQUALITY.test(conjunct) || IS_NULL.test(conjunct) ? equality : range;
|
|
96
|
+
for (const name of namesIn(conjunct, scan, columns)) {
|
|
97
|
+
if (!equality.includes(name) && !range.includes(name)) into.push(name);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return {
|
|
102
|
+
equality,
|
|
103
|
+
range: range.filter((name) => !equality.includes(name)),
|
|
104
|
+
...(arms === undefined ? {} : { arms }),
|
|
105
|
+
};
|
|
106
|
+
}
|
package/src/plan-oracle.ts
CHANGED
|
@@ -3,6 +3,10 @@
|
|
|
3
3
|
// a scan or a sort only when nothing else answers, so the verdict reads the schema, not the rows.
|
|
4
4
|
// Pure: JSON in, verdicts out (plan 101, 40-next-query-plans).
|
|
5
5
|
|
|
6
|
+
import { columnsIn } from './plan-conditions';
|
|
7
|
+
import type { PlanUnservable, SortKeyRef } from './plan-sort-key';
|
|
8
|
+
import { sortKeyPart, sortUnservable } from './plan-sort-key';
|
|
9
|
+
|
|
6
10
|
/** What a red node is: a whole-table read, or rows sorted for a limited read. */
|
|
7
11
|
export type PlanVerdictCode = 'X_QUERY_UNINDEXED' | 'X_QUERY_SORT_UNSERVED';
|
|
8
12
|
|
|
@@ -16,8 +20,14 @@ export interface PlanVerdict {
|
|
|
16
20
|
readonly equality: readonly string[];
|
|
17
21
|
/** Physical columns it compares any other way (a range, a pattern). */
|
|
18
22
|
readonly range: readonly string[];
|
|
19
|
-
/** The sort's columns on that table, in order. Empty for a scan. */
|
|
23
|
+
/** The sort's columns on that table, in order. Empty for a scan, or a sort no index serves. */
|
|
20
24
|
readonly order: readonly string[];
|
|
25
|
+
/** A sort no declarable index serves: why (`plan-sort-key.ts`). */
|
|
26
|
+
readonly unservable?: PlanUnservable;
|
|
27
|
+
/** The arms of an OR the read is bounded by, each its columns: one index answers each arm. */
|
|
28
|
+
readonly arms?: readonly (readonly string[])[];
|
|
29
|
+
/** The alias the plan reads the table by, when it is not the table's own name. */
|
|
30
|
+
readonly alias?: string;
|
|
21
31
|
}
|
|
22
32
|
|
|
23
33
|
interface PlanNode {
|
|
@@ -70,46 +80,63 @@ export function planRoot(answer: unknown): unknown {
|
|
|
70
80
|
return record(first)?.['Plan'] ?? first;
|
|
71
81
|
}
|
|
72
82
|
|
|
73
|
-
|
|
74
|
-
const
|
|
75
|
-
|
|
76
|
-
|
|
83
|
+
/** Every scan beneath `node` (the node itself included), of any table. */
|
|
84
|
+
const scansBeneath = (node: PlanNode): readonly PlanNode[] => [
|
|
85
|
+
...(node.table !== undefined && SCANS.has(node.type) ? [node] : []),
|
|
86
|
+
...node.children.flatMap(scansBeneath),
|
|
87
|
+
];
|
|
88
|
+
|
|
89
|
+
/** Nodes that compute what is sorted: past one, the sort key is no table's stored order. */
|
|
90
|
+
const COMPUTED: ReadonlySet<string> = new Set(['Aggregate', 'WindowAgg', 'Group']);
|
|
91
|
+
const computedBeneath = (node: PlanNode): boolean =>
|
|
92
|
+
node.children.some((child) => COMPUTED.has(child.type) || computedBeneath(child));
|
|
77
93
|
|
|
78
94
|
/**
|
|
79
|
-
* The
|
|
80
|
-
*
|
|
81
|
-
*
|
|
95
|
+
* The verdict of a `Sort` under a `Limit`. It belongs to the table its key names — a join lists
|
|
96
|
+
* its other side first, and that side's columns are not what is sorted. A key no index can hold
|
|
97
|
+
* still names a table, with no order to index; a key on a table the sweep does not judge (a
|
|
98
|
+
* declared `scan:`, a framework table) is that table's, and no verdict.
|
|
82
99
|
*/
|
|
83
|
-
function
|
|
100
|
+
function sortVerdict(
|
|
101
|
+
node: PlanNode,
|
|
84
102
|
conditions: readonly string[],
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
const
|
|
89
|
-
const
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
103
|
+
tables: PlanTableColumns,
|
|
104
|
+
): PlanVerdict | undefined {
|
|
105
|
+
const scans = scansBeneath(node);
|
|
106
|
+
const judged = scans.filter((scan) => scan.table !== undefined && tables.has(scan.table));
|
|
107
|
+
const scanOf = (ref: SortKeyRef): PlanNode | undefined =>
|
|
108
|
+
scans.find((scan) =>
|
|
109
|
+
ref.qualifier === undefined
|
|
110
|
+
? scan.table !== undefined && tables.get(scan.table)?.has(ref.name) === true
|
|
111
|
+
: ref.qualifier === scan.alias || ref.qualifier === scan.table,
|
|
112
|
+
);
|
|
113
|
+
const parts = node.sortKey.map(sortKeyPart);
|
|
114
|
+
const owners = parts.flatMap((part) => part.refs).flatMap((ref) => scanOf(ref) ?? []);
|
|
115
|
+
const owner = owners[0];
|
|
116
|
+
if (owner?.table !== undefined && !tables.has(owner.table)) return undefined;
|
|
117
|
+
const scan = owner ?? judged[0];
|
|
118
|
+
const columns = scan?.table === undefined ? undefined : tables.get(scan.table);
|
|
119
|
+
if (scan?.table === undefined || columns === undefined) return undefined;
|
|
120
|
+
const plainUnowned = parts.some((part) => part.column && scanOf(part.column) === undefined);
|
|
121
|
+
const unservable: PlanUnservable | undefined =
|
|
122
|
+
parts.some((part) => part.column === undefined) && computedBeneath(node)
|
|
123
|
+
? 'aggregate'
|
|
124
|
+
: plainUnowned
|
|
125
|
+
? 'derived'
|
|
126
|
+
: sortUnservable(parts, (ref) => scanOf(ref)?.table);
|
|
127
|
+
const found = columnsIn([...conditions, ...scan.conditions], scan, columns);
|
|
128
|
+
const order = columnsIn(node.sortKey, scan, columns);
|
|
129
|
+
return {
|
|
130
|
+
code: 'X_QUERY_SORT_UNSERVED',
|
|
131
|
+
node: node.type,
|
|
132
|
+
table: scan.table,
|
|
133
|
+
equality: found.equality,
|
|
134
|
+
range: [],
|
|
135
|
+
order: unservable === undefined ? [...order.equality, ...order.range] : [],
|
|
136
|
+
...(unservable === undefined ? {} : { unservable }),
|
|
137
|
+
};
|
|
102
138
|
}
|
|
103
139
|
|
|
104
|
-
/** The first scan beneath `node` (the node itself included) that reads one of `tables`. */
|
|
105
|
-
const scanBeneath = (node: PlanNode, tables: PlanTableColumns): PlanNode | undefined =>
|
|
106
|
-
node.table !== undefined && SCANS.has(node.type) && tables.has(node.table)
|
|
107
|
-
? node
|
|
108
|
-
: node.children.reduce<PlanNode | undefined>(
|
|
109
|
-
(found, child) => found ?? scanBeneath(child, tables),
|
|
110
|
-
undefined,
|
|
111
|
-
);
|
|
112
|
-
|
|
113
140
|
/** Physical table → its physical columns: only these tables are judged. */
|
|
114
141
|
export type PlanTableColumns = ReadonlyMap<string, ReadonlySet<string>>;
|
|
115
142
|
|
|
@@ -135,24 +162,13 @@ export function judgePlan(answer: unknown, tables: PlanTableColumns): readonly P
|
|
|
135
162
|
table: node.table,
|
|
136
163
|
...found,
|
|
137
164
|
order: [],
|
|
165
|
+
...(node.alias === undefined || node.alias === node.table ? {} : { alias: node.alias }),
|
|
138
166
|
});
|
|
139
167
|
}
|
|
140
168
|
}
|
|
141
169
|
if (node.type === 'Sort' && limited) {
|
|
142
|
-
const
|
|
143
|
-
|
|
144
|
-
if (scan?.table !== undefined && scanColumns !== undefined) {
|
|
145
|
-
const found = columnsIn([...conditions, ...scan.conditions], scan, scanColumns);
|
|
146
|
-
const order = columnsIn(node.sortKey, scan, scanColumns);
|
|
147
|
-
verdicts.push({
|
|
148
|
-
code: 'X_QUERY_SORT_UNSERVED',
|
|
149
|
-
node: node.type,
|
|
150
|
-
table: scan.table,
|
|
151
|
-
equality: found.equality,
|
|
152
|
-
range: [],
|
|
153
|
-
order: [...order.equality, ...order.range],
|
|
154
|
-
});
|
|
155
|
-
}
|
|
170
|
+
const sorted = sortVerdict(node, conditions, tables);
|
|
171
|
+
if (sorted !== undefined) verdicts.push(sorted);
|
|
156
172
|
}
|
|
157
173
|
const below = limited || node.type === 'Limit';
|
|
158
174
|
// A join's conditions name both sides: each child keeps only the conjuncts about itself.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// A Sort node's key, read as an index would have to hold it: which table each part belongs to,
|
|
2
|
+
// whether it is a plain column, and which way it runs. An index serves a sort only when every part
|
|
3
|
+
// is a plain column of ONE table, all running one way — anything else no `indexes:` entry ends.
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Why no index an entity declares can serve a sort: its key is an expression (`entity()` declares
|
|
7
|
+
* no expression index), a value an aggregate or window computed, a value a subquery derived, the
|
|
8
|
+
* columns of more than one table, or columns running in different directions.
|
|
9
|
+
*/
|
|
10
|
+
export type PlanUnservable = 'expression' | 'aggregate' | 'derived' | 'tables' | 'directions';
|
|
11
|
+
|
|
12
|
+
/** A column a sort key names, as the plan prints it: `p.created_at`, or `title` alone. */
|
|
13
|
+
export interface SortKeyRef {
|
|
14
|
+
readonly qualifier: string | undefined;
|
|
15
|
+
readonly name: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface SortKeyPart {
|
|
19
|
+
/** The column, when the part is one column and nothing else. */
|
|
20
|
+
readonly column: SortKeyRef | undefined;
|
|
21
|
+
/** Every column the part names: the expression's too. */
|
|
22
|
+
readonly refs: readonly SortKeyRef[];
|
|
23
|
+
/** `up` is an ascending walk (nulls last), `down` the same walk backwards; `mixed` is neither. */
|
|
24
|
+
readonly way: 'up' | 'down' | 'mixed';
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const NAME = '"?([A-Za-z_][\\w$]*)"?';
|
|
28
|
+
const PLAIN = new RegExp(`^(?:${NAME}\\.)?${NAME}$`);
|
|
29
|
+
const REF = new RegExp(`(?:${NAME}\\.)?${NAME}`, 'g');
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* One part of a `Sort Key`. Postgres prints `DESC` and a `NULLS` that differs from the direction's
|
|
33
|
+
* default; ascending nulls last and descending nulls first are the two walks of one btree.
|
|
34
|
+
*/
|
|
35
|
+
export function sortKeyPart(key: string): SortKeyPart {
|
|
36
|
+
let rest = key.trim();
|
|
37
|
+
const nulls = /\s+NULLS\s+(FIRST|LAST)$/i.exec(rest);
|
|
38
|
+
if (nulls !== null) rest = rest.slice(0, nulls.index);
|
|
39
|
+
const desc = /\s+DESC$/i.test(rest);
|
|
40
|
+
rest = rest.replace(/\s+(?:DESC|ASC)$/i, '');
|
|
41
|
+
const nullsFirst = nulls === null ? desc : nulls[1]?.toUpperCase() === 'FIRST';
|
|
42
|
+
const way = desc === nullsFirst ? (desc ? 'down' : 'up') : 'mixed';
|
|
43
|
+
const plain = PLAIN.exec(rest);
|
|
44
|
+
const refs = [...rest.replace(/'[^']*'/g, '').matchAll(REF)].flatMap(([, qualifier, name]) =>
|
|
45
|
+
name === undefined ? [] : [{ qualifier, name }],
|
|
46
|
+
);
|
|
47
|
+
const column = plain?.[2] === undefined ? undefined : { qualifier: plain[1], name: plain[2] };
|
|
48
|
+
return { column, refs, way };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Why an index cannot serve these parts, given the table each column belongs to (`undefined` for
|
|
53
|
+
* one the judged tables do not hold), or `undefined` when one index on one table could.
|
|
54
|
+
*/
|
|
55
|
+
export function sortUnservable(
|
|
56
|
+
parts: readonly SortKeyPart[],
|
|
57
|
+
tableOf: (ref: SortKeyRef) => string | undefined,
|
|
58
|
+
): PlanUnservable | undefined {
|
|
59
|
+
if (parts.some((part) => part.column === undefined)) return 'expression';
|
|
60
|
+
const tables = new Set(parts.flatMap((part) => (part.column ? [tableOf(part.column)] : [])));
|
|
61
|
+
if (tables.size > 1) return 'tables';
|
|
62
|
+
const ways = new Set(parts.map((part) => part.way));
|
|
63
|
+
if (ways.has('mixed') || ways.size > 1) return 'directions';
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|