@quereus/store 4.12.1 → 4.13.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/README.md +30 -0
  2. package/dist/src/common/cost-profile.d.ts +85 -0
  3. package/dist/src/common/cost-profile.d.ts.map +1 -0
  4. package/dist/src/common/cost-profile.js +56 -0
  5. package/dist/src/common/cost-profile.js.map +1 -0
  6. package/dist/src/common/index.d.ts +2 -1
  7. package/dist/src/common/index.d.ts.map +1 -1
  8. package/dist/src/common/index.js +3 -1
  9. package/dist/src/common/index.js.map +1 -1
  10. package/dist/src/common/kv-store.d.ts +16 -0
  11. package/dist/src/common/kv-store.d.ts.map +1 -1
  12. package/dist/src/common/kv-store.js.map +1 -1
  13. package/dist/src/common/serialization.d.ts +62 -3
  14. package/dist/src/common/serialization.d.ts.map +1 -1
  15. package/dist/src/common/serialization.js +67 -3
  16. package/dist/src/common/serialization.js.map +1 -1
  17. package/dist/src/common/store-module-access-plan.d.ts +6 -1
  18. package/dist/src/common/store-module-access-plan.d.ts.map +1 -1
  19. package/dist/src/common/store-module-access-plan.js +357 -136
  20. package/dist/src/common/store-module-access-plan.js.map +1 -1
  21. package/dist/src/common/store-module-base.d.ts +13 -0
  22. package/dist/src/common/store-module-base.d.ts.map +1 -1
  23. package/dist/src/common/store-module-base.js +14 -0
  24. package/dist/src/common/store-module-base.js.map +1 -1
  25. package/dist/src/common/store-module-schema-sync.d.ts.map +1 -1
  26. package/dist/src/common/store-module-schema-sync.js +21 -0
  27. package/dist/src/common/store-module-schema-sync.js.map +1 -1
  28. package/dist/src/common/store-module.d.ts.map +1 -1
  29. package/dist/src/common/store-module.js +1 -1
  30. package/dist/src/common/store-module.js.map +1 -1
  31. package/dist/src/common/store-table-base.d.ts +88 -7
  32. package/dist/src/common/store-table-base.d.ts.map +1 -1
  33. package/dist/src/common/store-table-base.js +135 -17
  34. package/dist/src/common/store-table-base.js.map +1 -1
  35. package/package.json +5 -5
@@ -16,7 +16,8 @@
16
16
  * Free functions rather than a layer of the store-module chain — access planning reads
17
17
  * no module state beyond the table's configured key collation, which the caller passes in.
18
18
  */
19
- import { AccessPlanBuilder, equalitySeekKeyCount, hasSemanticOrdering, isMultiValueEquality } from '@quereus/quereus';
19
+ import { AccessPlanBuilder, combineConjunctive, equalitySeekKeyCount, hasSemanticOrdering, isMultiValueEquality, selectivityFromHistogram, } from '@quereus/quereus';
20
+ import { PARITY_COST_PROFILE } from './cost-profile.js';
20
21
  import { indexPrefixSeekIsCollationExact, indexRangeAtPositionIsOrderSafe, pkOrderPreservingPrefixLength, resolveIndexKeyCollations, resolvePkKeyCollations, } from './pk-key-resolution.js';
21
22
  /**
22
23
  * Planner-side constraint operator groups, as `BestAccessPlanRequest.filters` spells
@@ -45,71 +46,190 @@ const RANGE_OPS = [...LOWER_BOUND_OPS, ...UPPER_BOUND_OPS];
45
46
  * explosion.
46
47
  */
47
48
  const MAX_MULTI_SEEK_KEYS = 1000;
48
- /** Cost of positioning one index seek, in the same unit as the per-row fetch cost. */
49
- const INDEX_SEEK_COST = 0.5;
50
49
  /**
51
- * Cost of resolving ONE secondary-index entry to its row, in the same unit as the full
52
- * scan's 1.0-per-row sequential read.
50
+ * FALLBACK rows each arm returns, as a fraction of the request's `estimatedRows` — shape
51
+ * constants, reached only when {@link resolveArmEstimate} cannot produce a per-predicate
52
+ * estimate from `tableSchema.statistics`. Concretely, a constant is still the estimate
53
+ * when:
53
54
  *
54
- * A secondary-index path reads TWO stores per matched row — the index entry, then the row
55
- * it names — where a sequential scan reads one row per row. `AccessPlanBuilder.eqMatch` /
56
- * `.rangeScan` charge only the first (0.3 / 0.5 per row), so the `eq`, `prefixRange` and
57
- * `range` seek arms add this term on top of the shape's own cost. The MULTI-SEEK arm does
58
- * not; the measured reason is at that arm in {@link tryIndexAccessPlan}.
55
+ * - the table has never been `ANALYZE`d (`tableSchema.statistics` absent) — the common
56
+ * case, which must plan byte-identically to the pre-statistics module;
57
+ * - the snapshot was taken while the table was EMPTY (`rowCount` 0), so it describes
58
+ * nothing and every `distinctCount` in it is 0 — see {@link resolveArmEstimate};
59
+ * - any column filling an EQUALITY role in the arm has no per-column statistics (a
60
+ * column added or renamed after the last `ANALYZE` — the arm falls back WHOLESALE
61
+ * rather than mixing a measured factor with a shape constant);
62
+ * - a pure `range` arm's bound cannot be answered by the column's histogram (no
63
+ * histogram was built, the bound's value is a parameter unknown at plan time, or the
64
+ * histogram declines). A `prefixRange` arm in the same position stays per-query — its
65
+ * measured prefix factors carry the estimate and only the bound's factor falls back
66
+ * to this constant.
59
67
  *
60
- * Why 1.0: `StoreTableScan` resolves entries in `ROW_RESOLUTION_BATCH`-sized `getMany`
61
- * batches (`store-table-scan.ts`: one store round trip per 256 entries, not per row), so a resolved row
62
- * costs about what a sequentially-iterated row costs — the index path simply pays it ON TOP
63
- * OF the entry read. Nothing here has been measured against real storage; it is a shape
64
- * constant like {@link ARM_SELECTIVITY}, chosen for parity rather than from a benchmark.
68
+ * `prefixRange` sits between the two other factors, as its window does: narrower than a
69
+ * bare leading-column range (the prefix pins every column ahead of the bound) and wider
70
+ * than the pure-equality estimate, which stands for a prefix pinning every column the
71
+ * predicate names.
65
72
  *
66
- * The constant is not freely tunable upward. Because {@link ARM_SELECTIVITY} is a fixed
67
- * FRACTION of the table, every arm's cost is linear in `estimatedRows` and so is the full
68
- * scan's — an arm either always prices above a sequential scan or never does, for a given
69
- * constant. The switch-over points: the `range` arm (0.3 × N × (0.5 + R) > N) flips at
70
- * R > 2.83, `prefixRange` at R > 6.17, `eq` at R > 9.7. So raising this past ~2.8 does not
71
- * make range seeks "more expensive", it DISABLES them outright. Real per-column statistics
72
- * (see the NOTE on {@link ARM_SELECTIVITY}) are what would make the comparison
73
- * discriminate per query rather than per arm.
74
- */
75
- const ROW_RESOLUTION_COST = 1.0;
76
- /**
77
- * Estimated rows each arm returns, as a fraction of the request's `estimatedRows`. The
78
- * store keeps no per-column histograms, so these are shape constants, not statistics.
79
- *
80
- * `prefixRange` sits between the two pre-existing factors, as its window does: it is
81
- * narrower than a bare leading-column range (the prefix pins every column ahead of the
82
- * bound) and wider than the pure-equality estimate, which stands for a prefix pinning
83
- * every column the predicate names.
84
- *
85
- * NOTE: these are the store's biggest cost-model gap, and no per-row cost term closes it.
86
- * They are shape constants, so `where col = ?` is modelled at 10% of the table whatever
87
- * the predicate actually matches, and `request.estimatedRows` is the table's row count,
88
- * not a selectivity-adjusted estimate. A predicate matching MOST of the table therefore
89
- * still prices as an index seek returning 10% of it, and the full-scan comparison in
90
- * {@link computeBestAccessPlan} cannot rescue it — the fix is real per-column statistics,
91
- * not another constant here. Two things have to land for that: the store has to KEEP a
92
- * value distribution (`StoreTableBase.getStatistics` reports the row count only — a
93
- * distinct count or histogram would cost a scan, which is why `ANALYZE` collects the
94
- * per-column half itself), and `BestAccessPlanRequest` has to CARRY it, which it does not
95
- * today — the module sees `estimatedRows` and the filters, nothing else, so even an
96
- * analyzed table plans identically. Until then every cost decision this file makes is
97
- * per-ARM, never per-PREDICATE.
98
- *
99
- * NOTE: the arms are exclusive per index, but two INDEXES compete on cost — and an index
100
- * on the equality prefix ALONE prices its `eq` arm cheaper (0.1) than the composite index
101
- * prices its `prefixRange` arm (0.15), so a schema carrying both `(a)` and `(a, b)` picks
102
- * `(a)` for `a = ? and b > ?` and leaves the `b` bound residual. Answers are unaffected.
103
- * Fine while redundant prefix indexes are rare; if one shows up as a slow plan, either
104
- * scale the `eq` factor by how many index columns the prefix actually pins, or drop
105
- * `prefixRange` below `eq` — do not just swap the two constants, which would then mis-rank
106
- * a genuine leading-column range.
73
+ * NOTE: the arms are exclusive per index, but two INDEXES compete on cost — and on an
74
+ * un-analyzed table an index on the equality prefix ALONE prices its `eq` arm cheaper
75
+ * (0.1) than the composite index prices its `prefixRange` arm (0.15), so a schema
76
+ * carrying both `(a)` and `(a, b)` picks `(a)` for `a = ? and b > ?` and leaves the `b`
77
+ * bound residual. Answers are unaffected, and `ANALYZE` largely resolves it: under real
78
+ * statistics the composite arm's estimate is the prefix's factor damped by the bound's,
79
+ * so it under-prices the single-column arm whenever the bound is at all selective. If
80
+ * the un-analyzed ranking ever shows up as a slow plan, the fix is `ANALYZE`, not a
81
+ * constant swap here.
107
82
  */
108
83
  const ARM_SELECTIVITY = {
109
84
  eq: 0.1,
110
85
  prefixRange: 0.15,
111
86
  range: 0.3,
112
87
  };
88
+ /**
89
+ * The `ANALYZE`-collected statistics for the table column at `colIdx`, or undefined when
90
+ * the table has none or the column is not covered.
91
+ *
92
+ * `columnStats` is keyed by LOWERCASE COLUMN NAME while this file works in column index,
93
+ * so the lookup goes index → current column name → stats. That direction is what keeps a
94
+ * post-`ANALYZE` `ALTER TABLE` safe: a RENAMED column's current name is absent from the
95
+ * map (falls back to the shape constant), and a DROPPED column shifts later indexes onto
96
+ * their own current names — a miss or the right entry, never a neighbour's numbers.
97
+ */
98
+ function columnStatsFor(tableInfo, stats, colIdx) {
99
+ const name = tableInfo.columns[colIdx]?.name;
100
+ return name === undefined ? undefined : stats.columnStats.get(name.toLowerCase());
101
+ }
102
+ /**
103
+ * The histogram-answered selectivity of the range bound(s) on `colIdx`, or undefined
104
+ * when the histogram cannot answer them (then the caller falls back to a shape
105
+ * constant).
106
+ *
107
+ * Reads the FIRST lower and FIRST upper bound in `filters` order — the same positional
108
+ * pick {@link claimFirstPerRole} claims and `rule-select-access-path` seeks on, so the
109
+ * bound being priced is the bound being served. Each bound needs a plan-time value
110
+ * (`f.value !== undefined`; a parameter binding has none) and a histogram verdict; a
111
+ * present bound that cannot be answered makes the whole factor undefined rather than
112
+ * half-measured. Both formulas are the engine's: a single bound is
113
+ * `selectivityFromHistogram` verbatim, and a two-sided range combines as
114
+ * `max(0, lowSel + highSel - 1)` — `CatalogStatsProvider.estimateLeaf`'s BETWEEN
115
+ * arithmetic, which models the two bounds as the anti-correlated pair they are rather
116
+ * than damped-independent conjuncts.
117
+ *
118
+ * NOTE: that is the engine's number for `v between 10 and 20` but not for the same range
119
+ * spelled `v > 10 and v < 20` — the engine reaches this arithmetic only from a `Between`
120
+ * node and folds two separate comparisons through `combineConjunctive` instead (roughly 2x
121
+ * looser; `estimateConjunction` carries its own note saying same-column pairing is what
122
+ * would fix it). The tighter number is kept here deliberately: it is the more accurate of
123
+ * the two, and a claimed range leaves no residual `Filter` carrying the engine's competing
124
+ * estimate for the optimizer to compare it against. If the engine ever pairs same-column
125
+ * bounds the two spellings converge with no change on this side; if a plan is measured
126
+ * going wrong on the gap before then, it is the ENGINE's conjunction that should move.
127
+ */
128
+ function rangeBoundSelectivity(stats, colStats, filters, colIdx) {
129
+ const histogram = colStats.histogram;
130
+ if (!histogram)
131
+ return undefined;
132
+ const boundSel = (ops) => {
133
+ const bound = filters.find(f => f.columnIndex === colIdx && ops.includes(f.op));
134
+ if (!bound)
135
+ return { present: false, sel: undefined };
136
+ return {
137
+ present: true,
138
+ sel: bound.value === undefined
139
+ ? undefined
140
+ : selectivityFromHistogram(histogram, bound.op, bound.value, stats.rowCount),
141
+ };
142
+ };
143
+ const low = boundSel(LOWER_BOUND_OPS);
144
+ const high = boundSel(UPPER_BOUND_OPS);
145
+ if ((low.present && low.sel === undefined) || (high.present && high.sel === undefined))
146
+ return undefined;
147
+ if (low.sel !== undefined && high.sel !== undefined)
148
+ return Math.max(0, low.sel + high.sel - 1);
149
+ return low.sel ?? high.sel;
150
+ }
151
+ /**
152
+ * The per-predicate row estimate for one arm: equality columns in `eqCols`, plus — for
153
+ * the `range` / `prefixRange` arms — the bound on `rangeCol`.
154
+ *
155
+ * **The design rule this implements: the estimate must be the number the engine's
156
+ * `CatalogStatsProvider` would produce for the same predicate.** A seek's advertised
157
+ * `rows` and the estimate the residual `Filter` above it carries describe the same row
158
+ * set; two different numbers would have the optimizer comparing two different worlds. So
159
+ * every formula here is `estimateLeaf`'s — equality is `1 / max(distinctCount, 1)`, a
160
+ * bound is the histogram's verdict ({@link rangeBoundSelectivity}) — and the factors
161
+ * combine through the engine's own `combineConjunctive` (damped independence), never a
162
+ * restated product.
163
+ *
164
+ * `statsBacked` — the flag the veto, the multi-seek's resolution charge, and `vetoCost`
165
+ * all key off — means the estimate is per-QUERY rather than a shape constant: every
166
+ * equality column had real statistics, and at least one factor was measured. An arm that
167
+ * is not statistics-backed returns exactly {@link ARM_SELECTIVITY}'s constant, so an
168
+ * un-analyzed table plans byte-identically to the pre-statistics module. A missing
169
+ * equality column falls back WHOLESALE (no mixing a measured factor with a constant);
170
+ * only a `prefixRange` arm's unanswerable BOUND degrades softly, contributing the arm
171
+ * constant as its factor while the measured prefix keeps the estimate per-query.
172
+ *
173
+ * A fraction of `request.estimatedRows`, deliberately, even though the histogram/NDV
174
+ * numbers are ratios of `statistics.rowCount`: `estimatedRows` is the figure the rest of
175
+ * the plan was costed with, and both snapshots come from the same `ANALYZE` unless the
176
+ * planner overrode the size — in which case following the override is the engine-wide
177
+ * rule (see `sizeRequestFromLiveCount`).
178
+ */
179
+ function resolveArmEstimate(tableInfo, filters, arm, eqCols, rangeCol) {
180
+ const fallback = { selectivity: ARM_SELECTIVITY[arm], statsBacked: false };
181
+ const stats = tableInfo.statistics;
182
+ // A snapshot taken while the table was EMPTY describes nothing — every `distinctCount`
183
+ // is 0, no histogram was built — so it is treated as no statistics at all rather than
184
+ // applied. Both halves of the design rule demand it: the engine short-circuits the same
185
+ // case (`estimatePredicateSelectivity` returns 0 outright on `rowCount === 0` and never
186
+ // reaches `estimateLeaf`'s formulas), and applying the snapshot anyway reads `1 / max(0,
187
+ // 1)` as "this equality matches EVERY row" — the opposite extreme — which prices the arm
188
+ // above a scan and hands the veto a table it should never have judged. That is reachable
189
+ // in ordinary use: `analyze` in a bootstrap script that runs before the data load leaves
190
+ // the request sized from the LIVE row count while these numbers still say zero, so every
191
+ // equality on the table would scan until someone re-analyzed it.
192
+ if (!stats || stats.rowCount <= 0)
193
+ return fallback;
194
+ const factors = [];
195
+ for (const colIdx of eqCols) {
196
+ const colStats = columnStatsFor(tableInfo, stats, colIdx);
197
+ if (!colStats)
198
+ return fallback;
199
+ // NOTE: `1/D` assumes uniformity, so an equality on a skewed column (a 99/1
200
+ // two-valued flag) is still mispriced for the common value. The histogram carries
201
+ // per-bucket distinct counts and could answer equality too, but the engine's
202
+ // `CatalogStatsProvider.estimateLeaf` prices `=` as `1/D`, and this file's design
203
+ // rule is to match it — if a skewed equality is ever measured planning wrong, move
204
+ // BOTH the engine and this factor to the histogram together.
205
+ //
206
+ // NOTE: `distinctCount` counts distinct NON-NULL values but the factor is applied
207
+ // against the full row count, so a mostly-NULL column over-estimates its matches
208
+ // (NULL rows never match an equality). Same treatment as skew: the engine prices it
209
+ // this way, so this file does too — fix both together or neither.
210
+ factors.push(1 / Math.max(colStats.distinctCount, 1));
211
+ }
212
+ if (rangeCol !== undefined) {
213
+ const colStats = columnStatsFor(tableInfo, stats, rangeCol);
214
+ const boundFactor = colStats
215
+ ? rangeBoundSelectivity(stats, colStats, filters, rangeCol)
216
+ : undefined;
217
+ if (boundFactor !== undefined) {
218
+ factors.push(boundFactor);
219
+ }
220
+ else if (eqCols.length === 0) {
221
+ // A pure `range` arm with no histogram answer has no measured factor at all —
222
+ // that IS the fallback case, not a degraded estimate.
223
+ return fallback;
224
+ }
225
+ else {
226
+ // `prefixRange` with a measured prefix: the bound alone degrades to the arm
227
+ // constant as its factor; the estimate stays per-query on the prefix's strength.
228
+ factors.push(ARM_SELECTIVITY[arm]);
229
+ }
230
+ }
231
+ return { selectivity: combineConjunctive(factors), statsBacked: true };
232
+ }
113
233
  /**
114
234
  * Build `handledFilters` by claiming ONLY the constraints `rule-select-access-path`
115
235
  * actually consumes: per seek column the FIRST '=' (equality seek), or the FIRST lower
@@ -215,9 +335,18 @@ function resolvePrimaryKeyPins(filters, pkColumns) {
215
335
  * K for an undecorated text PK member. The secondary-index arm never sees it: index key
216
336
  * bytes encode under each index column's own collation, so {@link tryIndexAccessPlan}
217
337
  * judges its filters against that instead.
338
+ *
339
+ * `costProfile` is the BACKEND's declared price for a random point read and a seek key,
340
+ * resolved once per module (`StoreModuleBase.costProfile`) and passed in for the same
341
+ * reason `tableKeyCollation` is: this function reads no module state of its own.
218
342
  */
219
- export function computeBestAccessPlan(db, tableInfo, request, tableKeyCollation) {
343
+ export function computeBestAccessPlan(db, tableInfo, request, tableKeyCollation, costProfile) {
220
344
  const estimatedRows = request.estimatedRows ?? 1000;
345
+ // Whether ANY pushed filter is a runtime-valued set. Detected once, on the REQUEST:
346
+ // such a request is engine-synthesized (`rule-key-set-seek`'s probes, or the key-set
347
+ // semi join itself) and must never be answered with this module's own scan verdict,
348
+ // whichever arm ends up serving it — see the exemption at the seek-vs-scan comparison.
349
+ const requestCarriesRuntimeSet = request.filters.some(f => f.runtimeSet !== undefined);
221
350
  const pkColumns = tableInfo.primaryKeyDefinition.map(pk => pk.index);
222
351
  // Hoisted ABOVE both primary-key arms: the multi-seek arm's ordering advertisement
223
352
  // needs it, and it is cheap (a short loop over the PK members). Its other consumer is
@@ -279,7 +408,7 @@ export function computeBestAccessPlan(db, tableInfo, request, tableKeyCollation)
279
408
  .build();
280
409
  }
281
410
  if (pkPins) {
282
- const plan = primaryKeyMultiSeekPlan(tableInfo, request, pkColumns, pkPins, estimatedRows, pkOrderPreservingPrefix);
411
+ const plan = primaryKeyMultiSeekPlan(tableInfo, request, pkColumns, pkPins, estimatedRows, pkOrderPreservingPrefix, costProfile);
283
412
  // A gate decline FALLS THROUGH to the arms below rather than returning cost-only.
284
413
  // That is safe: an `IN` is not in {@link RANGE_OPS}, so the leading-PK-range arm
285
414
  // cannot grab it, and it lands on the secondary-index arms / full scan exactly as
@@ -342,7 +471,7 @@ export function computeBestAccessPlan(db, tableInfo, request, tableKeyCollation)
342
471
  for (const index of indexes) {
343
472
  if (index.columns.length === 0)
344
473
  continue;
345
- const candidate = tryIndexAccessPlan(db, tableInfo, request, index, estimatedRows);
474
+ const candidate = tryIndexAccessPlan(db, tableInfo, request, index, estimatedRows, costProfile);
346
475
  if (!candidate)
347
476
  continue;
348
477
  const { plan } = candidate;
@@ -360,38 +489,54 @@ export function computeBestAccessPlan(db, tableInfo, request, tableKeyCollation)
360
489
  costOnlyFallback = plan;
361
490
  }
362
491
  if (bestSeekPlan) {
492
+ // Two narrow exemptions skip the seek-vs-scan comparison outright:
493
+ //
494
+ // - **A request carrying a runtime-valued set** (`filter.runtimeSet`). Such a
495
+ // request is only ever built by the engine — `rule-key-set-seek`'s synthesized
496
+ // probes and the key-set semi join itself — and the engine reads a probe answer
497
+ // that names no index as "the module declined", abandoning the whole rewrite.
498
+ // So this module must NEVER substitute its own scan verdict there: the engine
499
+ // makes the scan comparison itself, interpolating a break-even from the module's
500
+ // costs at 2 and 1000 keys. Detected on the REQUEST rather than on the winning
501
+ // arm, because the probe must survive whichever arm this module happens to pick
502
+ // for it. (Substituting a scan verdict at 1000 keys was MEASURED to fail 11
503
+ // tests in `key-set-seek-store.spec.ts` on 200-and-300-row tables where the seek
504
+ // is unambiguously right at the key counts actually used.)
505
+ // - **A multi-seek whose row estimate is not statistics-backed.** Its
506
+ // `min(N, inCount × 0.1N)` union reaches the whole table at ten seek keys, so
507
+ // beyond that the figure is a CLAMP rather than an estimate — vetoing on it
508
+ // would judge the estimator's artifact. A statistics-backed multi-seek clamps
509
+ // only at `inCount ≈ distinctCount`, which is the honest saturation point, so it
510
+ // faces the comparison like every other arm — a literal `col in (…900 values…)`
511
+ // over an analyzed 100-row table now correctly loses to the scan.
363
512
  //
364
- // The MULTI-SEEK arm is exempt, and that is a measured decision rather than an
365
- // oversight. `rule-key-set-seek` (engine side) does not consume this arm as a PLAN —
366
- // it probes `getBestAccessPlan` with SYNTHESIZED runtime-set filters at 2 and 1000
367
- // keys, reads the two costs as a straight line, and interpolates the key count at
368
- // which the seek would overtake a scan. A probe answer that stops naming an index is
369
- // read as "the module declined" and the whole rewrite is abandoned. Since nothing
370
- // here distinguishes a probe from a user's own `col in (…)`, substituting this
371
- // module's own scan verdict at 1000 keys silently switches key-set semi joins off:
372
- // MEASURED, it fails 11 tests in `key-set-seek-store.spec.ts` on tables of 200 and
373
- // 300 rows where the seek is unambiguously the right plan at the key counts actually
374
- // used. The arm's brakes stay `inCount × INDEX_SEEK_COST` (which is what the engine
375
- // interpolates) and {@link MAX_MULTI_SEEK_KEYS}; the engine makes the scan comparison
376
- // this exemption skips, off the same numbers.
513
+ // **The comparison judges the candidate at `vetoCost`, which is `plan.cost` itself
514
+ // exactly when the arm's row estimate is statistics-backed.** A per-query estimate
515
+ // lets the backend's declared profile decide the arm's fate per query: on
516
+ // IndexedDB (`pointRead: 3.0`) the `range` arm's break-even is
517
+ // 1/(0.5 + 3.0) ≈ 0.286 of the table, so a range the histogram puts at 25% seeks
518
+ // and one it puts at 35% scans — the discrimination the declared profile exists to
519
+ // buy. An arm still priced by an {@link ARM_SELECTIVITY} shape constant is instead
520
+ // judged at PARITY `pointRead`, settled when the IndexedDB profile was measured:
521
+ // a fixed fraction makes the veto arm-DISABLING rather than arm-tuning (the
522
+ // flip-point arithmetic is at the top of this file, where the profile's terms are
523
+ // introduced), IndexedDB's measured band straddles the `range` arm's flip point
524
+ // exactly, and the error is wildly asymmetric — disabling the arm costs up to 25×
525
+ // when the real predicate is selective, keeping it costs ~10% in the case the
526
+ // constant describes. So the declared profile decides per QUERY once the estimate
527
+ // is real, and falls back to the parity price where the estimate is still a shape
528
+ // constant. `cost-profile.spec.ts` pins both halves of that policy.
377
529
  //
378
530
  // Ties keep the seek: it returns fewer rows for the rest of the plan to carry.
379
- if (bestSeekPlan.isMultiSeek || bestSeekPlan.plan.cost <= scanPlan.cost)
531
+ const exemptFromVeto = requestCarriesRuntimeSet
532
+ || (bestSeekPlan.isMultiSeek && !bestSeekPlan.statsBacked);
533
+ if (exemptFromVeto || bestSeekPlan.vetoCost <= scanPlan.cost)
380
534
  return bestSeekPlan.plan;
381
535
  // Losing the seek is SAFE, never a wrong answer: the scan claims no filters, so the
382
536
  // engine keeps every one of them as a residual Filter and the row set is identical.
383
537
  // Deliberately returns the scan rather than falling through to `costOnlyFallback`
384
538
  // below — a cost-only plan performs this same sequential scan while advertising an
385
539
  // index arm's (cheaper) cost, so falling through would undo the comparison.
386
- //
387
- // NOTE: a vetoed seek is indistinguishable from a DECLINE to the engine, and
388
- // `rule-key-set-seek` reads a probe answer that names no index as "the module
389
- // declined" and abandons its rewrite. Harmless while the veto is reachable only
390
- // where an arm's estimate is the whole table (a one-row table, per the arithmetic on
391
- // {@link ROW_RESOLUTION_COST}) — a key-set seek there is worthless anyway. If real
392
- // per-column statistics ever make the veto fire on tables worth seeking, the
393
- // comparison has to exempt any request carrying a runtime-valued set, not just the
394
- // arm this module happens to pick for it.
395
540
  return { ...scanPlan, explains: 'Store full table scan (cheaper than the best index seek)' };
396
541
  }
397
542
  // NOTE: a cost-only plan carries no PK-order advertisement even though the store still
@@ -425,7 +570,7 @@ export function computeBestAccessPlan(db, tableInfo, request, tableKeyCollation)
425
570
  * batches. Both gates below mirror one of its `multiSeekMalformed` throws, which stay in
426
571
  * place as the assertion that the plan never produced a shape it cannot serve.
427
572
  */
428
- function primaryKeyMultiSeekPlan(tableInfo, request, pkColumns, pins, estimatedRows, pkOrderPreservingPrefix) {
573
+ function primaryKeyMultiSeekPlan(tableInfo, request, pkColumns, pins, estimatedRows, pkOrderPreservingPrefix, profile) {
429
574
  if (pins.seekKeyCount > MAX_MULTI_SEEK_KEYS)
430
575
  return null;
431
576
  // Mirrors `StoreTableScan.pkHasSemanticOrderingMember`, which `scanMultiSeekPrimary`
@@ -437,8 +582,16 @@ function primaryKeyMultiSeekPlan(tableInfo, request, pkColumns, pins, estimatedR
437
582
  if (pkColumns.some(colIdx => hasSemanticOrdering(tableInfo.columns[colIdx]?.logicalType)))
438
583
  return null;
439
584
  // The PK is unique, so each seek key matches at most one row — no `multiRows` clamp
440
- // artifact (contrast the secondary arm's), and no ROW_RESOLUTION_COST: the point read
441
- // IS the row read, with no index-entry → row indirection to charge for.
585
+ // artifact (contrast the secondary arm's), and no separate `profile.pointRead` term: the
586
+ // point read IS the row read, with no index-entry → row indirection to charge for.
587
+ //
588
+ // NOTE: this arm is therefore slightly OVER-charged on a backend that declares an
589
+ // expensive profile — `profile.seekPositioning` prices "position an index window AND
590
+ // read the row it names", and here there is no index window (IndexedDB: ≈ 3 units of
591
+ // real cost, charged 5). Accepted rather than split into a third knob: a second
592
+ // multi-seek term would double the tuning surface to model a bias whose only effect is
593
+ // that a very large `where pk in (…)` prefers a scan slightly sooner than it should.
594
+ // Revisit only if a primary-key IN is ever measured planning wrong because of it.
442
595
  //
443
596
  // `Math.max(1, …)` is LOAD-BEARING, not defensive. `rows: 0` on a plan that claims
444
597
  // every filter makes `rule-select-access-path` replace the whole table access with an
@@ -449,7 +602,7 @@ function primaryKeyMultiSeekPlan(tableInfo, request, pkColumns, pins, estimatedR
449
602
  const plan = AccessPlanBuilder
450
603
  // `setIsSet(false)` is likewise load-bearing: `eqMatch` defaults `isSet` to
451
604
  // `rows <= 1`, which a one-key runtime set would satisfy.
452
- .eqMatch(rows, pins.seekKeyCount * INDEX_SEEK_COST)
605
+ .eqMatch(rows, pins.seekKeyCount * profile.seekPositioning)
453
606
  .setIsSet(false)
454
607
  .setIndexName('_primary_')
455
608
  // EVERY primary-key column, in `primaryKeyDefinition` order: `scanMultiSeekPrimary`
@@ -459,12 +612,20 @@ function primaryKeyMultiSeekPlan(tableInfo, request, pkColumns, pins, estimatedR
459
612
  .setHandledFilters(claimFirstPerRole(request.filters, equalityRoles(pkColumns)))
460
613
  .setExplanation(`Store primary key multi-seek(${pins.seekKeyCount})`)
461
614
  .build();
462
- // This arm returns straight out of the PK branch, so it never reaches the
463
- // seek-versus-scan comparison further down `computeBestAccessPlan` — the same
464
- // exemption the secondary multi-seek arm gets explicitly, for the same reason:
465
- // `rule-key-set-seek` reads this cost as a straight line at 2 and 1000 keys and
466
- // abandons its rewrite if either probe stops naming an index. Do not "fix" this by
467
- // adding the comparison.
615
+ // This arm returns straight out of the PK branch, so it never reaches the seek-versus-scan
616
+ // comparison further down `computeBestAccessPlan` — structurally what the secondary arms
617
+ // state as `requestCarriesRuntimeSet`, and ONE rule for both: a request the engine
618
+ // synthesized to probe this module must never be answered with the module's own scan
619
+ // verdict. `rule-key-set-seek` reads this cost as a straight line at 2 and 1000 keys and
620
+ // abandons its whole rewrite if either probe stops naming an index; the engine makes the
621
+ // scan comparison itself, off the break-even it interpolates from those two costs. Do not
622
+ // "fix" this by adding the comparison.
623
+ //
624
+ // The PK arm is exempt unconditionally where the secondary arms are exempt per request,
625
+ // and that is not a divergence: a literal `where pk in (…)` is priced honestly here with
626
+ // or without statistics (the PK is unique, so `D = N` and `min(estimatedRows, K)` already
627
+ // IS the per-predicate estimate — nothing for `ANALYZE` to sharpen), so there is no
628
+ // clamp artifact for a veto to correct.
468
629
  //
469
630
  // `scanMultiSeekPrimary` sorts its points ascending by encoded data key, which IS
470
631
  // primary-key order (per-column DESC inversion is baked into the bytes), so the plan
@@ -502,14 +663,22 @@ function primaryKeyMultiSeekPlan(tableInfo, request, pkColumns, pins, estimatedR
502
663
  * handled" and "build a window" decisions cannot disagree. A decline returns a cost-only
503
664
  * plan: cheaper cost, filters unhandled, residual retained; correct, just not sped up.
504
665
  *
505
- * **Cost.** Each single-window seek arm pays {@link ROW_RESOLUTION_COST} per row it expects
506
- * to return, on top of the `AccessPlanBuilder` shape's own per-row term: the shape prices
507
- * reading the index ENTRY, and the store must then read the ROW that entry names out of the
508
- * data store. Two arms deliberately do NOT pay it — a cost-only decline (it resolves
509
- * nothing; the scan reads rows directly) and the multi-seek, for the measured reason
510
- * recorded at that arm below.
666
+ * **Rows.** Each arm's row estimate comes from {@link resolveArmEstimate}: per-predicate
667
+ * when the table's `ANALYZE`-collected statistics cover the arm's columns, the
668
+ * {@link ARM_SELECTIVITY} shape constant otherwise — with `statsBacked` recording which.
669
+ *
670
+ * **Cost.** Each single-window seek arm pays the backend's declared `profile.pointRead`
671
+ * per row it expects to return, on top of the `AccessPlanBuilder` shape's own per-row term:
672
+ * the shape prices reading the index ENTRY, and the store must then read the ROW that entry
673
+ * names out of the data store. A cost-only decline deliberately does not pay it (it
674
+ * resolves nothing; the scan reads rows directly), and the multi-seek pays it exactly when
675
+ * its row estimate is statistics-backed — the reason is recorded at that arm below. Every
676
+ * candidate also carries a `vetoCost`, the price the seek-vs-scan comparison in
677
+ * {@link computeBestAccessPlan} judges it at: the declared cost itself for a
678
+ * statistics-backed arm, the arm repriced at PARITY `pointRead` otherwise (reasoning at
679
+ * that site).
511
680
  */
512
- function tryIndexAccessPlan(db, tableInfo, request, index, estimatedRows) {
681
+ function tryIndexAccessPlan(db, tableInfo, request, index, estimatedRows, profile) {
513
682
  // Exclude PARTIAL indexes from access planning: neither the engine nor this
514
683
  // module checks that the query's WHERE implies the index predicate, so seeking
515
684
  // a partial index for a query it doesn't cover would silently drop the rows the
@@ -574,20 +743,31 @@ function tryIndexAccessPlan(db, tableInfo, request, index, estimatedRows) {
574
743
  seekCols = eqCols;
575
744
  }
576
745
  const isRange = arm !== 'eq';
577
- const rows = Math.max(1, Math.floor(estimatedRows * ARM_SELECTIVITY[arm]));
746
+ // Resolved AFTER the prefixRange→eq degradation above, so the estimate prices the arm
747
+ // actually advertised (a degraded arm's unclaimed bound survives in the residual and
748
+ // must not narrow this estimate). For a multi-seek, `eq`'s estimate is PER SEEK KEY —
749
+ // one full equality tuple — which is exactly what the union arithmetic below wants.
750
+ const { selectivity, statsBacked } = resolveArmEstimate(tableInfo, request.filters, arm, arm === 'range' ? [] : eqCols, arm === 'eq' ? undefined : (arm === 'range' ? leadingCol : trailingRangeCol));
751
+ const rows = Math.max(1, Math.floor(estimatedRows * selectivity));
578
752
  /** The arm's shape — `AccessPlanBuilder`'s per-row index-entry term, no resolution. */
579
753
  const armShape = () => isRange ? AccessPlanBuilder.rangeScan(rows, 0.2) : AccessPlanBuilder.eqMatch(rows, 0.3);
580
- /** The arm's shape plus the per-fetched-row {@link ROW_RESOLUTION_COST}. */
581
- const seekingArm = () => armShape().addCost(rows * ROW_RESOLUTION_COST);
582
- const costOnly = (why) => ({
583
- plan: armShape()
754
+ /** The arm's shape plus one per-fetched-row resolution term at the given price. */
755
+ const seekingArm = (pointRead) => armShape().addCost(rows * pointRead);
756
+ const costOnly = (why) => {
757
+ const plan = armShape()
584
758
  .setHandledFilters(new Array(request.filters.length).fill(false))
585
759
  .setExplanation(`Store index scan on ${index.name} (${why})`)
586
- .build(),
587
- // Cost-only never reaches the comparison (it claims no seek columns), so the flag
588
- // is immaterial; false states the fact — this plan seeks nothing at all.
589
- isMultiSeek: false,
590
- });
760
+ .build();
761
+ return {
762
+ plan,
763
+ // Cost-only resolves nothing, so its cost carries no `pointRead` term to reprice —
764
+ // and it never reaches the comparison anyway (it claims no seek columns).
765
+ vetoCost: plan.cost,
766
+ // Likewise immaterial; false states the fact — this plan seeks nothing at all.
767
+ isMultiSeek: false,
768
+ statsBacked,
769
+ };
770
+ };
591
771
  if (arm === 'range') {
592
772
  if (!indexRangeAtPositionIsOrderSafe(db, tableInfo.columns, index, indexKeyCollations, 0)) {
593
773
  return costOnly('cost-only; index range needs an order-preserving key collation');
@@ -631,13 +811,16 @@ function tryIndexAccessPlan(db, tableInfo, request, index, estimatedRows) {
631
811
  // MemoryTableModule.evaluateIndexAccess's setIsSet(!isMultiSeek). No ordering is
632
812
  // advertised — window emission order is encoded-key order, not any column order.
633
813
  //
634
- // NOTE: this arm alone does NOT pay {@link ROW_RESOLUTION_COST}, though it resolves
635
- // every entry it matches exactly like the single-window arms do. The reason is
636
- // `multiRows`, not the physics: `inCount × rows` reaches `estimatedRows` at ten seek
637
- // keys (`rows` is the fixed 0.1 × N equality shape constant) and at two or three on a
638
- // handful-of-rows table, so beyond that the figure is a CLAMP — "the whole table" —
639
- // rather than an estimate of what the keys match. Charging a per-row term against a
640
- // clamp prices the estimator's artifact:
814
+ // **This arm pays the per-row `profile.pointRead` exactly when `statsBacked`.** It
815
+ // resolves every entry it matches, like the single-window arms do, so the physics
816
+ // always says charge it; what varies is whether `multiRows` is an ESTIMATE worth
817
+ // charging against — and that is precisely what `statsBacked` records.
818
+ //
819
+ // UNBACKED, `rows` is the fixed 0.1 × N equality shape constant, so
820
+ // `min(N, inCount × rows)` reaches the whole table at ten seek keys, and at two or
821
+ // three on a handful-of-rows table. Beyond that the figure is a CLAMP — "the whole
822
+ // table" — rather than an estimate of what the keys match, and charging a per-row
823
+ // term against a clamp prices the estimator's artifact:
641
824
  // - MEASURED: it fails 16 tests in `key-set-seek-store.spec.ts`, every one a
642
825
  // runtime key-set semi join over a 3-to-4-row table that stops seeking.
643
826
  // - The engine's `rule-key-set-seek` interpolates a break-even from THIS cost at 2
@@ -645,36 +828,74 @@ function tryIndexAccessPlan(db, tableInfo, request, index, estimatedRows) {
645
828
  // `N` for every N, so the break-even can never reach the engine's 1000-key ceiling
646
829
  // — key sets above ~710 keys would stop seeking on a table of ANY size, including
647
830
  // the 10M-row tables where an index seek is the entire point.
648
- // The price of leaving it off is a narrower mis-ranking: against another index's `eq`
649
- // arm (which does pay the term) a small-key IN now looks relatively cheaper than it
650
- // is — `where a in (x, y) and b = ?` over a 1000-row table prices ix_a at 61 and ix_b
651
- // at 130 and picks ix_a, though ix_a's own estimate says it fetches 200 rows to ix_b's
652
- // 100. Charging the term everywhere fixes that ranking and breaks the feature above;
653
- // the union estimate has to stop clamping before both can hold. See
654
- // backlog/debt-store-multi-seek-union-row-estimate.
831
+ //
832
+ // BACKED, `rows` is `N/D` for the seek column, so `inCount × rows` only reaches N as
833
+ // `inCount` approaches `D` — the honest saturation point (at `inCount = D` the IN list
834
+ // really does name every value the column holds). The two failure modes above both
835
+ // dissolve there: the 3-to-4-row key-set tables are covered by the `runtimeSet`
836
+ // exemption at the veto site regardless of statistics, and the probe cost stays a
837
+ // straight line in K (`K·seekPositioning + min(N, K·N/D)·(0.3 + pointRead)`) until the
838
+ // clamp bites at `K ≈ D`, which is where a scan genuinely is cheaper.
839
+ //
840
+ // NOTE: charging it DOES change which key-set semi joins the engine rewrites. The
841
+ // engine fits a chord through this cost at 2 and 1000 keys and solves for a break-even;
842
+ // the honest cost is higher, so a rewrite stops firing once the seek keys approach the
843
+ // table's own row count — 1000 keys over a 1000-row table now scans, correctly. The
844
+ // answer never moves (`column-statistics-plan.spec.ts` pins that), and large tables are
845
+ // untouched (10k keys into 10M rows is still nowhere near the clamp). If an ANALYZE is
846
+ // ever measured making a real key-set join SLOWER, the thing to look at is the chord,
847
+ // not this term: the true cost is concave (linear, then flat once `multiRows` clamps at
848
+ // `K = D`), so a two-point interpolation under-reads it in between.
849
+ //
850
+ // Charging it also fixes the mis-ranking the unbacked case still carries: against
851
+ // another index's `eq` arm (which always pays the term) a small-key IN looks cheaper
852
+ // than it is — `where a in (x, y) and b = ?` over an un-analyzed 1000-row table prices
853
+ // ix_a at 61 and ix_b at 130 and picks ix_a, though ix_a's own estimate says it fetches
854
+ // 200 rows to ix_b's 100. `ANALYZE` is now the fix for that; the residual unbacked case
855
+ // is backlog/debt-store-multi-seek-union-row-estimate.
655
856
  const multiRows = Math.min(estimatedRows, inCount * rows);
857
+ const multiSeekShape = AccessPlanBuilder.eqMatch(multiRows, inCount * profile.seekPositioning);
858
+ if (statsBacked)
859
+ multiSeekShape.addCost(multiRows * profile.pointRead);
860
+ const plan = multiSeekShape
861
+ .setIsSet(false)
862
+ .setHandledFilters(handledFilters)
863
+ .setIndexName(index.name)
864
+ .setSeekColumns(seekCols)
865
+ .setExplanation(`Store index multi-seek(${inCount}) on ${index.name}`)
866
+ .build();
656
867
  return {
657
- plan: AccessPlanBuilder.eqMatch(multiRows, inCount * INDEX_SEEK_COST)
658
- .setIsSet(false)
659
- .setHandledFilters(handledFilters)
660
- .setIndexName(index.name)
661
- .setSeekColumns(seekCols)
662
- .setExplanation(`Store index multi-seek(${inCount}) on ${index.name}`)
663
- .build(),
868
+ plan,
869
+ // Statistics-backed: the declared profile decides this arm's fate per query, so the
870
+ // veto price IS the declared price. Unbacked: no `pointRead` term was charged, so
871
+ // there is nothing to reprice at parity — and the arm is exempt from the veto
872
+ // anyway. `plan.cost` is right either way.
873
+ vetoCost: plan.cost,
664
874
  isMultiSeek: true,
875
+ statsBacked,
665
876
  };
666
877
  }
667
878
  const armLabel = arm === 'prefixRange'
668
879
  ? `prefix-range seek(prefix=${eqCols.length})`
669
880
  : arm === 'range' ? 'range scan' : 'seek';
881
+ const plan = seekingArm(profile.pointRead)
882
+ .setHandledFilters(handledFilters)
883
+ .setIndexName(index.name)
884
+ .setSeekColumns(seekCols)
885
+ .setExplanation(`Store index ${armLabel} on ${index.name}`)
886
+ .build();
670
887
  return {
671
- plan: seekingArm()
672
- .setHandledFilters(handledFilters)
673
- .setIndexName(index.name)
674
- .setSeekColumns(seekCols)
675
- .setExplanation(`Store index ${armLabel} on ${index.name}`)
676
- .build(),
888
+ plan,
889
+ // Statistics-backed: `rows` discriminates per query, so the backend's DECLARED price
890
+ // decides this arm's fate per query — that discrimination is what declaring a profile
891
+ // buys. Otherwise judged at parity, because a shape constant makes the veto
892
+ // arm-disabling rather than arm-tuning (full reasoning at the veto site). The parity
893
+ // price is re-derived from `armShape()` rather than adjusted off `plan.cost`, so it is
894
+ // the exact number the pre-profile module produced rather than a float round trip
895
+ // through the declared one.
896
+ vetoCost: statsBacked ? plan.cost : seekingArm(PARITY_COST_PROFILE.pointRead).build().cost,
677
897
  isMultiSeek: false,
898
+ statsBacked,
678
899
  };
679
900
  }
680
901
  /**