@ultimat3/testing 30.5.0 → 30.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +5 -5
- package/package.json +17 -17
- package/src/define-exercise.ts +19 -7
- package/src/errors.ts +6 -6
- package/src/exercise-actor-user.ts +63 -0
- package/src/exercise-actor.ts +36 -7
- package/src/exercise-crash.ts +7 -3
- package/src/exercise-drift-hint.ts +28 -0
- package/src/exercise-input.ts +3 -0
- package/src/exercise-job-run.ts +68 -0
- package/src/exercise-job-tenant.ts +10 -5
- package/src/exercise-override-input.ts +26 -5
- package/src/exercise-report.ts +38 -2
- package/src/exercise-subjects.ts +6 -4
- package/src/exercise-sweeps.ts +49 -7
- package/src/exercise.ts +50 -34
- package/src/fixture-jobs.ts +17 -5
- package/src/index.ts +4 -0
- package/src/job-crash-points.ts +36 -0
- package/src/job-crash-proof.ts +26 -6
- package/src/job-sim-errors.ts +24 -0
- package/src/job-sim-world.ts +16 -5
- package/src/plan-conditions.ts +106 -0
- package/src/plan-oracle.ts +65 -49
- package/src/plan-sort-key.ts +65 -0
- package/src/plan-sweep-fix.ts +87 -21
- package/src/plan-sweep-report.ts +48 -4
- package/src/seed-graph-values.ts +20 -11
- package/src/seed-graph.ts +30 -1
- package/src/tenant-proof-report.ts +12 -3
- package/src/tenant-proof.ts +32 -4
- package/src/seed-pattern-sample.ts +0 -216
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// A plan node's conditions, read as the columns one scanned table compares: by equality, by any
|
|
2
|
+
// other way, and per arm of an OR. An OR is served by one index per arm (a BitmapOr), never by one
|
|
3
|
+
// index over every column it names, so its arms are kept apart from the conjuncts around it.
|
|
4
|
+
|
|
5
|
+
/** The scan whose columns are read: its table and the alias the plan qualifies it by. */
|
|
6
|
+
export interface ConditionScan {
|
|
7
|
+
readonly table: string | undefined;
|
|
8
|
+
readonly alias: string | undefined;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface ConditionColumns {
|
|
12
|
+
readonly equality: string[];
|
|
13
|
+
readonly range: string[];
|
|
14
|
+
/** The first OR whose every arm compares a column of the table: each arm's columns. */
|
|
15
|
+
readonly arms?: readonly (readonly string[])[];
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const IDENT = /(?:"?([A-Za-z_][\w$]*)"?\.)?"?([A-Za-z_][\w$]*)"?/g;
|
|
19
|
+
const EQUALITY = /(?<![<>!])=(?!>)/;
|
|
20
|
+
/** `IS NULL` pins one value as `=` does, and a btree serves it as an index condition. */
|
|
21
|
+
const IS_NULL = /\bIS\s+NULL\b/i;
|
|
22
|
+
|
|
23
|
+
/** The parts of `text` between `word`s that stand outside every parenthesis and literal. */
|
|
24
|
+
function splitTop(text: string, word: 'AND' | 'OR'): string[] {
|
|
25
|
+
const parts: string[] = [];
|
|
26
|
+
let depth = 0;
|
|
27
|
+
let quoted = false;
|
|
28
|
+
let start = 0;
|
|
29
|
+
for (let at = 0; at < text.length; at += 1) {
|
|
30
|
+
const char = text[at];
|
|
31
|
+
if (char === "'") quoted = !quoted;
|
|
32
|
+
if (quoted) continue;
|
|
33
|
+
if (char === '(') depth += 1;
|
|
34
|
+
else if (char === ')') depth -= 1;
|
|
35
|
+
else if (depth === 0 && text.startsWith(` ${word} `, at)) {
|
|
36
|
+
parts.push(text.slice(start, at));
|
|
37
|
+
start = at + word.length + 2;
|
|
38
|
+
at = start - 1;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
parts.push(text.slice(start));
|
|
42
|
+
return parts.map(unwrap);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** `((a = $1))` → `a = $1`: parentheses that wrap the whole text, and only those. */
|
|
46
|
+
function unwrap(text: string): string {
|
|
47
|
+
let rest = text.trim();
|
|
48
|
+
for (;;) {
|
|
49
|
+
if (!rest.startsWith('(') || !rest.endsWith(')')) return rest;
|
|
50
|
+
let depth = 0;
|
|
51
|
+
for (let at = 0; at < rest.length; at += 1) {
|
|
52
|
+
if (rest[at] === '(') depth += 1;
|
|
53
|
+
else if (rest[at] === ')') depth -= 1;
|
|
54
|
+
if (depth === 0 && at < rest.length - 1) return rest;
|
|
55
|
+
}
|
|
56
|
+
rest = rest.slice(1, -1).trim();
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** The columns of the table a conjunct names; a name qualified by another alias is not its. */
|
|
61
|
+
function namesIn(conjunct: string, scan: ConditionScan, columns: ReadonlySet<string>): string[] {
|
|
62
|
+
const names: string[] = [];
|
|
63
|
+
for (const [, qualifier, name] of conjunct.replace(/'[^']*'/g, '').matchAll(IDENT)) {
|
|
64
|
+
if (name === undefined || !columns.has(name)) continue;
|
|
65
|
+
if (qualifier !== undefined && qualifier !== scan.alias && qualifier !== scan.table) continue;
|
|
66
|
+
if (!names.includes(name)) names.push(name);
|
|
67
|
+
}
|
|
68
|
+
return names;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* The columns of `scan` the conditions name, split by how they are compared: `=` and `IS NULL`
|
|
73
|
+
* are equality, anything else a range. A conjunct qualified by another alias is a join's other
|
|
74
|
+
* side; an unqualified name counts when the table has the column. An OR is no conjunct of either.
|
|
75
|
+
*/
|
|
76
|
+
export function columnsIn(
|
|
77
|
+
conditions: readonly string[],
|
|
78
|
+
scan: ConditionScan,
|
|
79
|
+
columns: ReadonlySet<string>,
|
|
80
|
+
): ConditionColumns {
|
|
81
|
+
const equality: string[] = [];
|
|
82
|
+
const range: string[] = [];
|
|
83
|
+
let arms: string[][] | undefined;
|
|
84
|
+
for (const condition of conditions) {
|
|
85
|
+
for (const conjunct of splitTop(unwrap(condition), 'AND')) {
|
|
86
|
+
const alternatives = splitTop(conjunct, 'OR');
|
|
87
|
+
if (alternatives.length > 1) {
|
|
88
|
+
const each = alternatives.map((arm) => {
|
|
89
|
+
const found = columnsIn([arm], scan, columns);
|
|
90
|
+
return [...found.equality, ...found.range];
|
|
91
|
+
});
|
|
92
|
+
if (arms === undefined && each.every((arm) => arm.length > 0)) arms = each;
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
const into = EQUALITY.test(conjunct) || IS_NULL.test(conjunct) ? equality : range;
|
|
96
|
+
for (const name of namesIn(conjunct, scan, columns)) {
|
|
97
|
+
if (!equality.includes(name) && !range.includes(name)) into.push(name);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return {
|
|
102
|
+
equality,
|
|
103
|
+
range: range.filter((name) => !equality.includes(name)),
|
|
104
|
+
...(arms === undefined ? {} : { arms }),
|
|
105
|
+
};
|
|
106
|
+
}
|
package/src/plan-oracle.ts
CHANGED
|
@@ -3,6 +3,10 @@
|
|
|
3
3
|
// a scan or a sort only when nothing else answers, so the verdict reads the schema, not the rows.
|
|
4
4
|
// Pure: JSON in, verdicts out (plan 101, 40-next-query-plans).
|
|
5
5
|
|
|
6
|
+
import { columnsIn } from './plan-conditions';
|
|
7
|
+
import type { PlanUnservable, SortKeyRef } from './plan-sort-key';
|
|
8
|
+
import { sortKeyPart, sortUnservable } from './plan-sort-key';
|
|
9
|
+
|
|
6
10
|
/** What a red node is: a whole-table read, or rows sorted for a limited read. */
|
|
7
11
|
export type PlanVerdictCode = 'X_QUERY_UNINDEXED' | 'X_QUERY_SORT_UNSERVED';
|
|
8
12
|
|
|
@@ -16,8 +20,14 @@ export interface PlanVerdict {
|
|
|
16
20
|
readonly equality: readonly string[];
|
|
17
21
|
/** Physical columns it compares any other way (a range, a pattern). */
|
|
18
22
|
readonly range: readonly string[];
|
|
19
|
-
/** The sort's columns on that table, in order. Empty for a scan. */
|
|
23
|
+
/** The sort's columns on that table, in order. Empty for a scan, or a sort no index serves. */
|
|
20
24
|
readonly order: readonly string[];
|
|
25
|
+
/** A sort no declarable index serves: why (`plan-sort-key.ts`). */
|
|
26
|
+
readonly unservable?: PlanUnservable;
|
|
27
|
+
/** The arms of an OR the read is bounded by, each its columns: one index answers each arm. */
|
|
28
|
+
readonly arms?: readonly (readonly string[])[];
|
|
29
|
+
/** The alias the plan reads the table by, when it is not the table's own name. */
|
|
30
|
+
readonly alias?: string;
|
|
21
31
|
}
|
|
22
32
|
|
|
23
33
|
interface PlanNode {
|
|
@@ -70,46 +80,63 @@ export function planRoot(answer: unknown): unknown {
|
|
|
70
80
|
return record(first)?.['Plan'] ?? first;
|
|
71
81
|
}
|
|
72
82
|
|
|
73
|
-
|
|
74
|
-
const
|
|
75
|
-
|
|
76
|
-
|
|
83
|
+
/** Every scan beneath `node` (the node itself included), of any table. */
|
|
84
|
+
const scansBeneath = (node: PlanNode): readonly PlanNode[] => [
|
|
85
|
+
...(node.table !== undefined && SCANS.has(node.type) ? [node] : []),
|
|
86
|
+
...node.children.flatMap(scansBeneath),
|
|
87
|
+
];
|
|
88
|
+
|
|
89
|
+
/** Nodes that compute what is sorted: past one, the sort key is no table's stored order. */
|
|
90
|
+
const COMPUTED: ReadonlySet<string> = new Set(['Aggregate', 'WindowAgg', 'Group']);
|
|
91
|
+
const computedBeneath = (node: PlanNode): boolean =>
|
|
92
|
+
node.children.some((child) => COMPUTED.has(child.type) || computedBeneath(child));
|
|
77
93
|
|
|
78
94
|
/**
|
|
79
|
-
* The
|
|
80
|
-
*
|
|
81
|
-
*
|
|
95
|
+
* The verdict of a `Sort` under a `Limit`. It belongs to the table its key names — a join lists
|
|
96
|
+
* its other side first, and that side's columns are not what is sorted. A key no index can hold
|
|
97
|
+
* still names a table, with no order to index; a key on a table the sweep does not judge (a
|
|
98
|
+
* declared `scan:`, a framework table) is that table's, and no verdict.
|
|
82
99
|
*/
|
|
83
|
-
function
|
|
100
|
+
function sortVerdict(
|
|
101
|
+
node: PlanNode,
|
|
84
102
|
conditions: readonly string[],
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
const
|
|
89
|
-
const
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
103
|
+
tables: PlanTableColumns,
|
|
104
|
+
): PlanVerdict | undefined {
|
|
105
|
+
const scans = scansBeneath(node);
|
|
106
|
+
const judged = scans.filter((scan) => scan.table !== undefined && tables.has(scan.table));
|
|
107
|
+
const scanOf = (ref: SortKeyRef): PlanNode | undefined =>
|
|
108
|
+
scans.find((scan) =>
|
|
109
|
+
ref.qualifier === undefined
|
|
110
|
+
? scan.table !== undefined && tables.get(scan.table)?.has(ref.name) === true
|
|
111
|
+
: ref.qualifier === scan.alias || ref.qualifier === scan.table,
|
|
112
|
+
);
|
|
113
|
+
const parts = node.sortKey.map(sortKeyPart);
|
|
114
|
+
const owners = parts.flatMap((part) => part.refs).flatMap((ref) => scanOf(ref) ?? []);
|
|
115
|
+
const owner = owners[0];
|
|
116
|
+
if (owner?.table !== undefined && !tables.has(owner.table)) return undefined;
|
|
117
|
+
const scan = owner ?? judged[0];
|
|
118
|
+
const columns = scan?.table === undefined ? undefined : tables.get(scan.table);
|
|
119
|
+
if (scan?.table === undefined || columns === undefined) return undefined;
|
|
120
|
+
const plainUnowned = parts.some((part) => part.column && scanOf(part.column) === undefined);
|
|
121
|
+
const unservable: PlanUnservable | undefined =
|
|
122
|
+
parts.some((part) => part.column === undefined) && computedBeneath(node)
|
|
123
|
+
? 'aggregate'
|
|
124
|
+
: plainUnowned
|
|
125
|
+
? 'derived'
|
|
126
|
+
: sortUnservable(parts, (ref) => scanOf(ref)?.table);
|
|
127
|
+
const found = columnsIn([...conditions, ...scan.conditions], scan, columns);
|
|
128
|
+
const order = columnsIn(node.sortKey, scan, columns);
|
|
129
|
+
return {
|
|
130
|
+
code: 'X_QUERY_SORT_UNSERVED',
|
|
131
|
+
node: node.type,
|
|
132
|
+
table: scan.table,
|
|
133
|
+
equality: found.equality,
|
|
134
|
+
range: [],
|
|
135
|
+
order: unservable === undefined ? [...order.equality, ...order.range] : [],
|
|
136
|
+
...(unservable === undefined ? {} : { unservable }),
|
|
137
|
+
};
|
|
102
138
|
}
|
|
103
139
|
|
|
104
|
-
/** The first scan beneath `node` (the node itself included) that reads one of `tables`. */
|
|
105
|
-
const scanBeneath = (node: PlanNode, tables: PlanTableColumns): PlanNode | undefined =>
|
|
106
|
-
node.table !== undefined && SCANS.has(node.type) && tables.has(node.table)
|
|
107
|
-
? node
|
|
108
|
-
: node.children.reduce<PlanNode | undefined>(
|
|
109
|
-
(found, child) => found ?? scanBeneath(child, tables),
|
|
110
|
-
undefined,
|
|
111
|
-
);
|
|
112
|
-
|
|
113
140
|
/** Physical table → its physical columns: only these tables are judged. */
|
|
114
141
|
export type PlanTableColumns = ReadonlyMap<string, ReadonlySet<string>>;
|
|
115
142
|
|
|
@@ -135,24 +162,13 @@ export function judgePlan(answer: unknown, tables: PlanTableColumns): readonly P
|
|
|
135
162
|
table: node.table,
|
|
136
163
|
...found,
|
|
137
164
|
order: [],
|
|
165
|
+
...(node.alias === undefined || node.alias === node.table ? {} : { alias: node.alias }),
|
|
138
166
|
});
|
|
139
167
|
}
|
|
140
168
|
}
|
|
141
169
|
if (node.type === 'Sort' && limited) {
|
|
142
|
-
const
|
|
143
|
-
|
|
144
|
-
if (scan?.table !== undefined && scanColumns !== undefined) {
|
|
145
|
-
const found = columnsIn([...conditions, ...scan.conditions], scan, scanColumns);
|
|
146
|
-
const order = columnsIn(node.sortKey, scan, scanColumns);
|
|
147
|
-
verdicts.push({
|
|
148
|
-
code: 'X_QUERY_SORT_UNSERVED',
|
|
149
|
-
node: node.type,
|
|
150
|
-
table: scan.table,
|
|
151
|
-
equality: found.equality,
|
|
152
|
-
range: [],
|
|
153
|
-
order: [...order.equality, ...order.range],
|
|
154
|
-
});
|
|
155
|
-
}
|
|
170
|
+
const sorted = sortVerdict(node, conditions, tables);
|
|
171
|
+
if (sorted !== undefined) verdicts.push(sorted);
|
|
156
172
|
}
|
|
157
173
|
const below = limited || node.type === 'Limit';
|
|
158
174
|
// A join's conditions name both sides: each child keeps only the conjuncts about itself.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// A Sort node's key, read as an index would have to hold it: which table each part belongs to,
|
|
2
|
+
// whether it is a plain column, and which way it runs. An index serves a sort only when every part
|
|
3
|
+
// is a plain column of ONE table, all running one way — anything else no `indexes:` entry ends.
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Why no index an entity declares can serve a sort: its key is an expression (`entity()` declares
|
|
7
|
+
* no expression index), a value an aggregate or window computed, a value a subquery derived, the
|
|
8
|
+
* columns of more than one table, or columns running in different directions.
|
|
9
|
+
*/
|
|
10
|
+
export type PlanUnservable = 'expression' | 'aggregate' | 'derived' | 'tables' | 'directions';
|
|
11
|
+
|
|
12
|
+
/** A column a sort key names, as the plan prints it: `p.created_at`, or `title` alone. */
|
|
13
|
+
export interface SortKeyRef {
|
|
14
|
+
readonly qualifier: string | undefined;
|
|
15
|
+
readonly name: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface SortKeyPart {
|
|
19
|
+
/** The column, when the part is one column and nothing else. */
|
|
20
|
+
readonly column: SortKeyRef | undefined;
|
|
21
|
+
/** Every column the part names: the expression's too. */
|
|
22
|
+
readonly refs: readonly SortKeyRef[];
|
|
23
|
+
/** `up` is an ascending walk (nulls last), `down` the same walk backwards; `mixed` is neither. */
|
|
24
|
+
readonly way: 'up' | 'down' | 'mixed';
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const NAME = '"?([A-Za-z_][\\w$]*)"?';
|
|
28
|
+
const PLAIN = new RegExp(`^(?:${NAME}\\.)?${NAME}$`);
|
|
29
|
+
const REF = new RegExp(`(?:${NAME}\\.)?${NAME}`, 'g');
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* One part of a `Sort Key`. Postgres prints `DESC` and a `NULLS` that differs from the direction's
|
|
33
|
+
* default; ascending nulls last and descending nulls first are the two walks of one btree.
|
|
34
|
+
*/
|
|
35
|
+
export function sortKeyPart(key: string): SortKeyPart {
|
|
36
|
+
let rest = key.trim();
|
|
37
|
+
const nulls = /\s+NULLS\s+(FIRST|LAST)$/i.exec(rest);
|
|
38
|
+
if (nulls !== null) rest = rest.slice(0, nulls.index);
|
|
39
|
+
const desc = /\s+DESC$/i.test(rest);
|
|
40
|
+
rest = rest.replace(/\s+(?:DESC|ASC)$/i, '');
|
|
41
|
+
const nullsFirst = nulls === null ? desc : nulls[1]?.toUpperCase() === 'FIRST';
|
|
42
|
+
const way = desc === nullsFirst ? (desc ? 'down' : 'up') : 'mixed';
|
|
43
|
+
const plain = PLAIN.exec(rest);
|
|
44
|
+
const refs = [...rest.replace(/'[^']*'/g, '').matchAll(REF)].flatMap(([, qualifier, name]) =>
|
|
45
|
+
name === undefined ? [] : [{ qualifier, name }],
|
|
46
|
+
);
|
|
47
|
+
const column = plain?.[2] === undefined ? undefined : { qualifier: plain[1], name: plain[2] };
|
|
48
|
+
return { column, refs, way };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Why an index cannot serve these parts, given the table each column belongs to (`undefined` for
|
|
53
|
+
* one the judged tables do not hold), or `undefined` when one index on one table could.
|
|
54
|
+
*/
|
|
55
|
+
export function sortUnservable(
|
|
56
|
+
parts: readonly SortKeyPart[],
|
|
57
|
+
tableOf: (ref: SortKeyRef) => string | undefined,
|
|
58
|
+
): PlanUnservable | undefined {
|
|
59
|
+
if (parts.some((part) => part.column === undefined)) return 'expression';
|
|
60
|
+
const tables = new Set(parts.flatMap((part) => (part.column ? [tableOf(part.column)] : [])));
|
|
61
|
+
if (tables.size > 1) return 'tables';
|
|
62
|
+
const ways = new Set(parts.map((part) => part.way));
|
|
63
|
+
if (ways.has('mixed') || ways.size > 1) return 'directions';
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|
package/src/plan-sweep-fix.ts
CHANGED
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
// lookup that pins a unique key reads one row whatever the planner walked to find it.
|
|
4
4
|
|
|
5
5
|
import type { PlanVerdict, PlanVerdictCode } from './plan-oracle';
|
|
6
|
+
import type { PlanUnservable } from './plan-sort-key';
|
|
7
|
+
import { sortKeyPart } from './plan-sort-key';
|
|
6
8
|
import type { SeedEntity } from './seed-graph-shape';
|
|
7
9
|
|
|
8
10
|
export interface PlanFinding {
|
|
@@ -15,6 +17,12 @@ export interface PlanFinding {
|
|
|
15
17
|
readonly table: string;
|
|
16
18
|
/** The `indexes: [{ on }]` entry, by property: tenant, equality, order, range. */
|
|
17
19
|
readonly index: readonly string[];
|
|
20
|
+
/**
|
|
21
|
+
* Why no index is the fix: a sort no declarable index serves, or `declared` — the index the node
|
|
22
|
+
* asks for is one the entity already holds, so declaring it again changes no plan. `index` is
|
|
23
|
+
* empty with it.
|
|
24
|
+
*/
|
|
25
|
+
readonly unserved?: PlanUnservable | 'declared';
|
|
18
26
|
}
|
|
19
27
|
|
|
20
28
|
/** Every column set the table holds unique: the primary key, unique columns, total unique rules. */
|
|
@@ -110,24 +118,74 @@ function servedByPartial(entity: SeedEntity, verdict: PlanVerdict, order: readon
|
|
|
110
118
|
|
|
111
119
|
const ORDER_BY = /\border by\s+(.+?)(?:\s+limit\b|\s+offset\b|\s+for\b|$)/gi;
|
|
112
120
|
|
|
121
|
+
/** `a, f(b, c), d` → its three items: commas outside parentheses only. */
|
|
122
|
+
function items(list: string): string[] {
|
|
123
|
+
const found: string[] = [];
|
|
124
|
+
let depth = 0;
|
|
125
|
+
let start = 0;
|
|
126
|
+
for (let at = 0; at < list.length; at += 1) {
|
|
127
|
+
if (list[at] === '(') depth += 1;
|
|
128
|
+
else if (list[at] === ')') depth -= 1;
|
|
129
|
+
else if (list[at] === ',' && depth === 0) {
|
|
130
|
+
found.push(list.slice(start, at));
|
|
131
|
+
start = at + 1;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return [...found, list.slice(start)];
|
|
135
|
+
}
|
|
136
|
+
|
|
113
137
|
/**
|
|
114
|
-
* The outermost `order by`'s columns, read off the text: an index walk
|
|
115
|
-
* sort key of its own, yet the index that ends the walk has to hold
|
|
138
|
+
* The outermost `order by`'s leading columns of the walked table, read off the text: an index walk
|
|
139
|
+
* avoids a sort and carries no sort key of its own, yet the index that ends the walk has to hold
|
|
140
|
+
* the order too. Only as far as one index can: plain columns of this table, one direction — past
|
|
141
|
+
* them the rows arrive presorted and Postgres finishes with an incremental sort.
|
|
116
142
|
*/
|
|
117
|
-
function orderOf(statement: string, columns: ReadonlySet<string
|
|
143
|
+
function orderOf(statement: string, columns: ReadonlySet<string>, verdict: PlanVerdict) {
|
|
118
144
|
const clauses = [...statement.matchAll(ORDER_BY)];
|
|
119
|
-
const
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
145
|
+
const order: string[] = [];
|
|
146
|
+
let way: string | undefined;
|
|
147
|
+
for (const item of items(clauses[clauses.length - 1]?.[1] ?? '')) {
|
|
148
|
+
const part = sortKeyPart(item);
|
|
149
|
+
const own =
|
|
150
|
+
part.column !== undefined &&
|
|
151
|
+
columns.has(part.column.name) &&
|
|
152
|
+
[undefined, verdict.table, verdict.alias].includes(part.column.qualifier);
|
|
153
|
+
if (!own || part.way === 'mixed' || (way !== undefined && part.way !== way)) break;
|
|
154
|
+
way = part.way;
|
|
155
|
+
if (part.column !== undefined) order.push(part.column.name);
|
|
156
|
+
}
|
|
157
|
+
return order;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* An OR the read is bounded by and nothing else: one index per arm answers it, as a BitmapOr. An
|
|
162
|
+
* arm whose column leads a declared index is answered already — the sweep's settings discourage
|
|
163
|
+
* the BitmapOr, so the plan alone cannot say; credited, never re-proven.
|
|
164
|
+
*/
|
|
165
|
+
function armFindings(
|
|
166
|
+
verdict: PlanVerdict,
|
|
167
|
+
declared: readonly (readonly string[])[],
|
|
168
|
+
property: (column: string) => string | undefined,
|
|
169
|
+
): readonly (readonly string[])[] {
|
|
170
|
+
const arms = new Map((verdict.arms ?? []).map((arm) => [arm.join('\u0000'), arm]));
|
|
171
|
+
return [...arms.values()]
|
|
172
|
+
.filter((arm) => !declared.some((key) => key[0] !== undefined && arm.includes(key[0])))
|
|
173
|
+
.map((arm) => arm.flatMap((column) => property(column) ?? []));
|
|
124
174
|
}
|
|
125
175
|
|
|
176
|
+
/**
|
|
177
|
+
* The plan was taken with every declared index in place: one that leads with exactly these
|
|
178
|
+
* columns is already there, and printing it as the fix would change nothing.
|
|
179
|
+
*/
|
|
180
|
+
const held = (keys: readonly (readonly string[])[], columns: readonly string[]): boolean =>
|
|
181
|
+
columns.length > 0 && keys.some((key) => columns.every((column, at) => key[at] === column));
|
|
182
|
+
|
|
126
183
|
/**
|
|
127
184
|
* The findings one statement's verdicts make. A verdict whose equality columns pin a unique key is
|
|
128
185
|
* dropped: one row at most. A table's sort verdict makes its scan verdict redundant — the index it
|
|
129
186
|
* names leads with the same equality columns. The tenant column leads the index only when the
|
|
130
|
-
* statement compares it: a read across tenants is served by no index that leads with one.
|
|
187
|
+
* statement compares it: a read across tenants is served by no index that leads with one. A sort
|
|
188
|
+
* no declarable index serves, or an index the entity already holds, is red with no index to paste.
|
|
131
189
|
*/
|
|
132
190
|
export function planFindings(
|
|
133
191
|
statement: string,
|
|
@@ -145,31 +203,39 @@ export function planFindings(
|
|
|
145
203
|
key.every((column) => verdict.equality.includes(column)),
|
|
146
204
|
);
|
|
147
205
|
if (pinned) return [];
|
|
206
|
+
const finding = {
|
|
207
|
+
code: verdict.code,
|
|
208
|
+
statement,
|
|
209
|
+
node: verdict.node,
|
|
210
|
+
entity: entity.$name,
|
|
211
|
+
table: verdict.table,
|
|
212
|
+
};
|
|
213
|
+
if (verdict.unservable !== undefined) {
|
|
214
|
+
return [{ ...finding, index: [], unserved: verdict.unservable }];
|
|
215
|
+
}
|
|
148
216
|
const description = entity.$describe();
|
|
149
217
|
const byColumn = new Map(description.columns.map((c) => [c.column, c.property]));
|
|
150
218
|
const walked = verdict.code === 'X_QUERY_UNINDEXED' && verdict.node !== 'Seq Scan';
|
|
151
219
|
// A walk filtered by a range already has its order: what it lacks is the bound. A btree serves
|
|
152
220
|
// no order after a range column, so the index is the equality columns, then the range.
|
|
153
221
|
const ranged = walked && verdict.range.length > 0;
|
|
154
|
-
|
|
222
|
+
const declared = declaredIndexes(entity, statement);
|
|
223
|
+
const bounded = verdict.equality.length > 0 || verdict.range.length > 0;
|
|
224
|
+
if (!bounded && verdict.arms !== undefined && verdict.code === 'X_QUERY_UNINDEXED') {
|
|
225
|
+
const arms = armFindings(verdict, declared, (column) => byColumn.get(column));
|
|
226
|
+
return arms.map((index) => ({ ...finding, index }));
|
|
227
|
+
}
|
|
228
|
+
if (ranged && servedByRange(declared, verdict)) return [];
|
|
155
229
|
const order = ranged
|
|
156
230
|
? []
|
|
157
231
|
: walked
|
|
158
|
-
? orderOf(statement, new Set(description.columns.map((c) => c.column)))
|
|
232
|
+
? orderOf(statement, new Set(description.columns.map((c) => c.column)), verdict)
|
|
159
233
|
: verdict.order;
|
|
160
234
|
if (servedByPartial(entity, verdict, order)) return [];
|
|
161
235
|
const tenant = description.columns.find((c) => c.property === entity.$tenantColumn)?.column;
|
|
162
236
|
const lead = tenant !== undefined && verdict.equality.includes(tenant) ? [tenant] : [];
|
|
163
237
|
const columns = [...new Set([...lead, ...verdict.equality, ...order, ...verdict.range])];
|
|
164
|
-
return [
|
|
165
|
-
|
|
166
|
-
code: verdict.code,
|
|
167
|
-
statement,
|
|
168
|
-
node: verdict.node,
|
|
169
|
-
entity: entity.$name,
|
|
170
|
-
table: verdict.table,
|
|
171
|
-
index: columns.flatMap((column) => byColumn.get(column) ?? []),
|
|
172
|
-
},
|
|
173
|
-
];
|
|
238
|
+
if (held(declared, columns)) return [{ ...finding, index: [], unserved: 'declared' }];
|
|
239
|
+
return [{ ...finding, index: columns.flatMap((column) => byColumn.get(column) ?? []) }];
|
|
174
240
|
});
|
|
175
241
|
}
|
package/src/plan-sweep-report.ts
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import { renderCauseValue, renderFixLiteral, renderFixShellArg } from '@ultimat3/core';
|
|
6
6
|
import type { ExerciseRecord } from './exercise-report';
|
|
7
|
+
import type { PlanUnservable } from './plan-sort-key';
|
|
7
8
|
import type { PlanFinding } from './plan-sweep-fix';
|
|
8
9
|
|
|
9
10
|
export interface SweepFinding {
|
|
@@ -41,7 +42,47 @@ function wholeRead(finding: PlanFinding, senders: readonly ExerciseRecord[]): Sw
|
|
|
41
42
|
};
|
|
42
43
|
}
|
|
43
44
|
|
|
45
|
+
/** Why no index serves a sort, as the cause says it. */
|
|
46
|
+
const UNSERVED_SORT: Readonly<Record<PlanUnservable, string>> = {
|
|
47
|
+
expression: 'an expression over its columns, and entity() declares no expression index',
|
|
48
|
+
aggregate: 'a value an aggregate or window computes, which no stored order holds',
|
|
49
|
+
derived: 'a value a subquery or CTE derives, which no index on a table holds',
|
|
50
|
+
tables: 'columns of more than one table, and one index holds one table',
|
|
51
|
+
directions: 'columns in different directions, and an index declares one order for all of them',
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* A red node no `indexes:` entry ends: a sort no declarable index holds, or one whose index the
|
|
56
|
+
* entity already declares. Printing an index would change no plan, so the fix is the rewrite that
|
|
57
|
+
* lets one apply, or the stated reason the rows stay few.
|
|
58
|
+
*/
|
|
59
|
+
function unservedRead(
|
|
60
|
+
finding: PlanFinding,
|
|
61
|
+
unserved: PlanUnservable | 'declared',
|
|
62
|
+
senders: readonly ExerciseRecord[],
|
|
63
|
+
): SweepFinding {
|
|
64
|
+
const by = senders.map(primitive).join(', ');
|
|
65
|
+
const entity = renderFixLiteral(finding.entity, '<entity>');
|
|
66
|
+
const rerun = renderFixShellArg(senders[0]?.name ?? '', '<primitive>');
|
|
67
|
+
const sent = `${by} sent ${renderCauseValue(excerpt(finding.statement))}`;
|
|
68
|
+
const cause =
|
|
69
|
+
unserved === 'declared'
|
|
70
|
+
? `${sent}, which reads ${finding.table} unbounded (${finding.node}) though entity(${entity}) already declares the index that would bound it: declaring it again changes no plan, the statement as written cannot use it`
|
|
71
|
+
: `${sent}, which sorts its rows to answer a limited read (${finding.node} under Limit) by ${UNSERVED_SORT[unserved]}: no index on ${finding.table} serves that sort, so it slows with every row the table gains`;
|
|
72
|
+
const rewrite =
|
|
73
|
+
unserved === 'declared'
|
|
74
|
+
? "the read compares the index's leading column by = or a range, no function around it"
|
|
75
|
+
: `an order by of plain ${finding.table} columns, all one direction (the rerun names their index)`;
|
|
76
|
+
return {
|
|
77
|
+
code: finding.code,
|
|
78
|
+
cause,
|
|
79
|
+
fix: `x exercise ${rerun} --json # after ${rewrite}, or expectedSeqScan('<why the rows stay few>', fn) around the read`,
|
|
80
|
+
at: senders[0] === undefined ? finding.entity : primitive(senders[0]),
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
44
84
|
function planFinding(finding: PlanFinding, senders: readonly ExerciseRecord[]): SweepFinding {
|
|
85
|
+
if (finding.unserved !== undefined) return unservedRead(finding, finding.unserved, senders);
|
|
45
86
|
if (finding.index.length === 0) return wholeRead(finding, senders);
|
|
46
87
|
const by = senders.map(primitive).join(', ');
|
|
47
88
|
const entity = renderFixLiteral(finding.entity, '<entity>');
|
|
@@ -67,7 +108,8 @@ export function planSweepFindings(records: readonly ExerciseRecord[]): readonly
|
|
|
67
108
|
const groups = new Map<string, { finding: PlanFinding; senders: ExerciseRecord[] }>();
|
|
68
109
|
for (const record of records) {
|
|
69
110
|
for (const finding of record.plans?.findings ?? []) {
|
|
70
|
-
|
|
111
|
+
// An OR's arms are one finding each, on one statement: the index is part of what is red.
|
|
112
|
+
const key = [finding.code, finding.table, finding.statement, ...finding.index].join('\u0000');
|
|
71
113
|
const group = groups.get(key) ?? { finding, senders: [] };
|
|
72
114
|
group.senders.push(record);
|
|
73
115
|
groups.set(key, group);
|
|
@@ -91,9 +133,11 @@ export function planSweepFindings(records: readonly ExerciseRecord[]): readonly
|
|
|
91
133
|
export function planSweepLines(records: readonly ExerciseRecord[]): readonly string[] {
|
|
92
134
|
return records.flatMap((record) => [
|
|
93
135
|
...(record.plans?.findings ?? []).map((finding) =>
|
|
94
|
-
finding.
|
|
95
|
-
? `${primitive(record)} · ${finding.code} ${finding.table},
|
|
96
|
-
:
|
|
136
|
+
finding.unserved !== undefined
|
|
137
|
+
? `${primitive(record)} · ${finding.code} ${finding.table}, no index serves it (${finding.unserved})`
|
|
138
|
+
: finding.index.length === 0
|
|
139
|
+
? `${primitive(record)} · ${finding.code} ${finding.table}, every row (no column to index)`
|
|
140
|
+
: `${primitive(record)} · ${finding.code} ${finding.table} on [${finding.index.join(', ')}]`,
|
|
97
141
|
),
|
|
98
142
|
...(record.scale?.verdict === 'grows' ? record.scale.shapes : []).map(
|
|
99
143
|
(shape) =>
|
package/src/seed-graph-values.ts
CHANGED
|
@@ -6,7 +6,6 @@
|
|
|
6
6
|
import { fnv1a } from '@ultimat3/core';
|
|
7
7
|
import type { ColumnMeta } from '@ultimat3/entity';
|
|
8
8
|
import type * as Sample from '@ultimat3/schema/sample';
|
|
9
|
-
import { patternSamples } from './seed-pattern-sample';
|
|
10
9
|
|
|
11
10
|
/** A column as `entity()` declares it, structurally: this barrel never loads `@ultimat3/entity`. */
|
|
12
11
|
export interface SeedColumn {
|
|
@@ -72,19 +71,11 @@ const textCandidates = (property: string, meta: ColumnMeta, ctx: ValueContext):
|
|
|
72
71
|
return [...sized, ...fromPatterns(property, ctx)];
|
|
73
72
|
};
|
|
74
73
|
|
|
75
|
-
/** Strings drawn from each declared pattern, kept only where every other pattern agrees. */
|
|
76
|
-
const fromPatterns = (property: string, ctx: ValueContext): string[] => {
|
|
77
|
-
const patterns = ctx.patterns ?? [];
|
|
78
|
-
return patterns
|
|
79
|
-
.flatMap((pattern) => patternSamples(pattern, `${property}:${ctx.canary}`))
|
|
80
|
-
.filter((sample) => patterns.every((pattern) => pattern.test(sample)));
|
|
81
|
-
};
|
|
82
|
-
|
|
83
74
|
let loadedSampler: typeof Sample | undefined;
|
|
84
75
|
|
|
85
76
|
/**
|
|
86
|
-
* `@ultimat3/schema/sample`, loaded on the first `json(schema)` column
|
|
87
|
-
* module is on the barrel, and the walker is a dynamic-only dependency there (`CLAUDE.md`).
|
|
77
|
+
* `@ultimat3/schema/sample`, loaded on the first pattern or `json(schema)` column, never at import:
|
|
78
|
+
* this module is on the barrel, and the walker is a dynamic-only dependency there (`CLAUDE.md`).
|
|
88
79
|
* `require` and not `import()` because every caller of `candidatesFor` is synchronous.
|
|
89
80
|
*/
|
|
90
81
|
const sampler = (): typeof Sample => {
|
|
@@ -92,6 +83,24 @@ const sampler = (): typeof Sample => {
|
|
|
92
83
|
return loadedSampler;
|
|
93
84
|
};
|
|
94
85
|
|
|
86
|
+
/**
|
|
87
|
+
* Strings drawn from each declared pattern by schema's one generator (`patternSamples`), kept only
|
|
88
|
+
* where every other pattern agrees. The text seed hashes with the pattern, so two columns differ.
|
|
89
|
+
*/
|
|
90
|
+
const fromPatterns = (property: string, ctx: ValueContext): string[] => {
|
|
91
|
+
const patterns = ctx.patterns ?? [];
|
|
92
|
+
if (patterns.length === 0) return [];
|
|
93
|
+
const { patternSamples } = sampler();
|
|
94
|
+
return patterns
|
|
95
|
+
.flatMap((pattern) =>
|
|
96
|
+
patternSamples(
|
|
97
|
+
pattern,
|
|
98
|
+
fnv1a(`${pattern.source}/${pattern.flags}:${property}:${ctx.canary}`) || 1,
|
|
99
|
+
),
|
|
100
|
+
)
|
|
101
|
+
.filter((sample) => patterns.every((pattern) => pattern.test(sample)));
|
|
102
|
+
};
|
|
103
|
+
|
|
95
104
|
/**
|
|
96
105
|
* Values the column's own schema accepts, seeded by the row so two rows differ and a rerun does
|
|
97
106
|
* not; then the bare `{}`/`[]`, which a schema-less `jsonb` column stores.
|