@minnowdb/core 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/engine/artifact-cache.d.ts +5 -0
- package/dist/engine/artifact-cache.js +64 -13
- package/dist/engine/database.js +294 -12
- package/dist/engine/optimizer.d.ts +7 -0
- package/dist/engine/optimizer.js +855 -84
- package/dist/engine/point-read.d.ts +57 -0
- package/dist/engine/point-read.js +189 -0
- package/dist/engine/query.d.ts +6 -0
- package/dist/engine/query.js +101 -18
- package/dist/engine/vector.js +69 -5
- package/dist/plan/model.d.ts +7 -1
- package/dist/storage/toolkit/record-core.js +104 -75
- package/dist/storage/types.js +34 -6
- package/dist/transactions/index.js +9 -1
- package/package.json +1 -1
- package/postgres-feature-profile.json +5 -0
- package/sql-feature-matrix.json +78 -10
package/README.md
CHANGED
|
@@ -8,8 +8,9 @@ npm install @minnowdb/core
|
|
|
8
8
|
```
|
|
9
9
|
|
|
10
10
|
- Direct SQL through `MinnowDatabase.query()` and `execute()`.
|
|
11
|
-
- Joins, CTEs, window functions, grouping sets,
|
|
12
|
-
|
|
11
|
+
- Joins, CTEs, window functions, grouping sets, nested correlated subqueries, subquery-backed
|
|
12
|
+
mutations, upserts, `RETURNING`, triggers, exact decimals, nested JSON/JSONB, stored generated
|
|
13
|
+
columns, zoneless DATE, arrays, enums, sequences, and savepoints.
|
|
13
14
|
- Compressed column storage, secondary indexes, full-text search, and snapshot reads.
|
|
14
15
|
- Atomic writes across tabs through IndexedDB or OPFS, strict durability by default, and explicit
|
|
15
16
|
origin-eviction persistence policy.
|
|
@@ -10,6 +10,11 @@ export interface ArtifactCacheStats {
|
|
|
10
10
|
/**
|
|
11
11
|
* Byte-bounded LRU for decoded blocks, vectors, derived results, and memoized query results.
|
|
12
12
|
* Payloads are immutable by convention; callers own the byte estimates for their artifact type.
|
|
13
|
+
*
|
|
14
|
+
* Recency lives on an intrusive doubly-linked list rather than Map insertion order: a hit
|
|
15
|
+
* relinks four pointers instead of deleting and re-inserting the key, which re-hashed the
|
|
16
|
+
* long block-identity keys twice per touch and was the single hottest frame of a cached point
|
|
17
|
+
* lookup.
|
|
13
18
|
*/
|
|
14
19
|
export declare class ArtifactCache {
|
|
15
20
|
#private;
|
|
@@ -1,10 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Byte-bounded LRU for decoded blocks, vectors, derived results, and memoized query results.
|
|
3
3
|
* Payloads are immutable by convention; callers own the byte estimates for their artifact type.
|
|
4
|
+
*
|
|
5
|
+
* Recency lives on an intrusive doubly-linked list rather than Map insertion order: a hit
|
|
6
|
+
* relinks four pointers instead of deleting and re-inserting the key, which re-hashed the
|
|
7
|
+
* long block-identity keys twice per touch and was the single hottest frame of a cached point
|
|
8
|
+
* lookup.
|
|
4
9
|
*/
|
|
5
10
|
export class ArtifactCache {
|
|
6
11
|
#limitBytes;
|
|
7
12
|
#entries = new Map();
|
|
13
|
+
/** Least recently used end of the recency list. */
|
|
14
|
+
#oldest;
|
|
15
|
+
/** Most recently used end of the recency list. */
|
|
16
|
+
#newest;
|
|
8
17
|
#usedBytes = 0;
|
|
9
18
|
#hits = 0;
|
|
10
19
|
#misses = 0;
|
|
@@ -18,6 +27,26 @@ export class ArtifactCache {
|
|
|
18
27
|
get enabled() {
|
|
19
28
|
return this.#limitBytes > 0;
|
|
20
29
|
}
|
|
30
|
+
#unlink(entry) {
|
|
31
|
+
if (entry.previous !== undefined)
|
|
32
|
+
entry.previous.next = entry.next;
|
|
33
|
+
else
|
|
34
|
+
this.#oldest = entry.next;
|
|
35
|
+
if (entry.next !== undefined)
|
|
36
|
+
entry.next.previous = entry.previous;
|
|
37
|
+
else
|
|
38
|
+
this.#newest = entry.previous;
|
|
39
|
+
entry.previous = undefined;
|
|
40
|
+
entry.next = undefined;
|
|
41
|
+
}
|
|
42
|
+
#appendNewest(entry) {
|
|
43
|
+
entry.previous = this.#newest;
|
|
44
|
+
entry.next = undefined;
|
|
45
|
+
if (this.#newest !== undefined)
|
|
46
|
+
this.#newest.next = entry;
|
|
47
|
+
this.#newest = entry;
|
|
48
|
+
this.#oldest ??= entry;
|
|
49
|
+
}
|
|
21
50
|
get(key) {
|
|
22
51
|
if (!this.enabled)
|
|
23
52
|
return undefined;
|
|
@@ -27,8 +56,10 @@ export class ArtifactCache {
|
|
|
27
56
|
return undefined;
|
|
28
57
|
}
|
|
29
58
|
this.#hits += 1;
|
|
30
|
-
this.#
|
|
31
|
-
|
|
59
|
+
if (this.#newest !== entry) {
|
|
60
|
+
this.#unlink(entry);
|
|
61
|
+
this.#appendNewest(entry);
|
|
62
|
+
}
|
|
32
63
|
return entry.payload;
|
|
33
64
|
}
|
|
34
65
|
put(key, payload, bytes) {
|
|
@@ -39,24 +70,44 @@ export class ArtifactCache {
|
|
|
39
70
|
}
|
|
40
71
|
const existing = this.#entries.get(key);
|
|
41
72
|
if (existing !== undefined) {
|
|
42
|
-
this.#entries.delete(key);
|
|
43
73
|
this.#usedBytes -= existing.bytes;
|
|
74
|
+
existing.payload = payload;
|
|
75
|
+
existing.bytes = bytes;
|
|
76
|
+
this.#usedBytes += bytes;
|
|
77
|
+
if (this.#newest !== existing) {
|
|
78
|
+
this.#unlink(existing);
|
|
79
|
+
this.#appendNewest(existing);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
const entry = {
|
|
84
|
+
key,
|
|
85
|
+
payload,
|
|
86
|
+
bytes,
|
|
87
|
+
previous: undefined,
|
|
88
|
+
next: undefined,
|
|
89
|
+
};
|
|
90
|
+
this.#entries.set(key, entry);
|
|
91
|
+
this.#appendNewest(entry);
|
|
92
|
+
this.#usedBytes += bytes;
|
|
44
93
|
}
|
|
45
|
-
|
|
46
|
-
this.#usedBytes
|
|
47
|
-
|
|
48
|
-
if (
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
94
|
+
let oldest = this.#oldest;
|
|
95
|
+
while (this.#usedBytes > this.#limitBytes && oldest !== undefined) {
|
|
96
|
+
const next = oldest.next;
|
|
97
|
+
if (oldest.key !== key) {
|
|
98
|
+
this.#unlink(oldest);
|
|
99
|
+
this.#entries.delete(oldest.key);
|
|
100
|
+
this.#usedBytes -= oldest.bytes;
|
|
101
|
+
this.#evictions += 1;
|
|
102
|
+
}
|
|
103
|
+
oldest = next;
|
|
55
104
|
}
|
|
56
105
|
}
|
|
57
106
|
/** Releases every retained payload while preserving lifetime counters for final diagnostics. */
|
|
58
107
|
clear() {
|
|
59
108
|
this.#entries.clear();
|
|
109
|
+
this.#oldest = undefined;
|
|
110
|
+
this.#newest = undefined;
|
|
60
111
|
this.#usedBytes = 0;
|
|
61
112
|
}
|
|
62
113
|
stats() {
|
package/dist/engine/database.js
CHANGED
|
@@ -18,12 +18,13 @@ import { applyWindowFunctions, bindPlanParameters, bindStatementParameters, DUAL
|
|
|
18
18
|
import { copyQueryResult, planMemoKey, queryResultMemoKey, queryResultRetainedBytes, RESULT_MEMO_MAX_BYTES, } from "./query-cache.js";
|
|
19
19
|
import { QueryMemoryBudgetError, QueryMemoryContext, DEFAULT_QUERY_MEMORY_BUDGET_BYTES, } from "./memory.js";
|
|
20
20
|
import { LiveQueryLimitError, LiveQuerySet, MAX_LIVE_QUERY_SETS_PER_DATABASE, } from "./live.js";
|
|
21
|
-
import { chooseJoinOrder, renderPlan } from "./optimizer.js";
|
|
21
|
+
import { chooseJoinOrder, optimizePlan, qualifyCorrelatedReferences, renderPlan, } from "./optimizer.js";
|
|
22
22
|
import { encodeSqlEqualityValue } from "./sql-semantics.js";
|
|
23
23
|
import { externalSqlDomainValue, externalSqlTextValue, isDateDomainValue, isSqlDomainValue, normalizeSqlDomainValue, protectedSqlTextValue, } from "./sql-domains.js";
|
|
24
24
|
import { toCatalog } from "./catalog.js";
|
|
25
25
|
import { applyColumnSteps, assertColumnDroppable, compileGeneratedColumnExpression, declaredForeignKeys, isDestructiveStep, planMigration, } from "./schema.js";
|
|
26
26
|
import { columnarTableFromRows, createColumnarTable, vectorValue, } from "./vector.js";
|
|
27
|
+
import { cachedPointReadTemplate, equalRunRange, pointReadTestHooks, resolvePointReadShape, valuesAreAscending, } from "./point-read.js";
|
|
27
28
|
// The on-disk format is little-endian; the bulk Float64 copy reads platform order.
|
|
28
29
|
const PLATFORM_LITTLE_ENDIAN = new Uint8Array(new Uint32Array([1]).buffer)[0] === 1;
|
|
29
30
|
const vectorTextDecoder = new TextDecoder("utf-8", { fatal: true });
|
|
@@ -33,6 +34,11 @@ const NULL_STRING_VECTOR_CODE = 0xffffffff;
|
|
|
33
34
|
const COMPOSITE_KEY_COLUMN_NAME = "\u0000minnow_primary_key";
|
|
34
35
|
const ENUM_TYPE_PREFIX = "\u0000minnow_enum_type:";
|
|
35
36
|
const SEQUENCE_PREFIX = "\u0000minnow_sequence:";
|
|
37
|
+
/**
|
|
38
|
+
* Candidate rows the point-read fast path will verify per statement before conceding that the
|
|
39
|
+
* shape is not selective enough and handing the scan back to the vectorized executor.
|
|
40
|
+
*/
|
|
41
|
+
const MAX_POINT_READ_CANDIDATES = 1_024;
|
|
36
42
|
/** Delta-chunk tail length past which a search schedules a fold-by-rebuild of the base. */
|
|
37
43
|
const FTS_FOLD_DELTA_CHUNKS = 16;
|
|
38
44
|
/** Persistent rebuild failure cannot let an accelerator append one durable delta per commit. */
|
|
@@ -1901,10 +1907,10 @@ export class MinnowDatabase {
|
|
|
1901
1907
|
const plannedColumn = writeColumnValues(keyColumn.type, values);
|
|
1902
1908
|
const ranges = writeBlockRanges([plannedColumn], values.length, this.#rowsPerBlock, this.#targetBlockBytes);
|
|
1903
1909
|
for (const [part, { start, end }] of ranges.entries()) {
|
|
1904
|
-
await blockStager.prepare(maximumWriteBlockStoredBytes(plannedColumn, start, end,
|
|
1910
|
+
await blockStager.prepare(maximumWriteBlockStoredBytes(plannedColumn, start, end, "raw"));
|
|
1905
1911
|
const slice = values.slice(start, end);
|
|
1906
1912
|
const encodeStarted = performance.now();
|
|
1907
|
-
const bytes = await this.#encodeColumnBlock(keyColumn.id, asColumnInput(keyColumn.type, slice));
|
|
1913
|
+
const bytes = await this.#encodeColumnBlock(keyColumn.id, asColumnInput(keyColumn.type, slice), "raw");
|
|
1908
1914
|
encodeMs += performance.now() - encodeStarted;
|
|
1909
1915
|
const blockId = [
|
|
1910
1916
|
"table",
|
|
@@ -3139,7 +3145,30 @@ export class MinnowDatabase {
|
|
|
3139
3145
|
// this transaction has staged, and nothing another writer published in between.
|
|
3140
3146
|
return externalizeQueryResult(await this.#duringTransaction(open, () => open.session.query(sql, options)));
|
|
3141
3147
|
}
|
|
3142
|
-
const
|
|
3148
|
+
const compiled = this.#compileCached(sql);
|
|
3149
|
+
// The keyed point-read fast path answers an eligible statement before parameter binding
|
|
3150
|
+
// ever clones the plan. A shape, parameter, catalog, or physical-history condition it
|
|
3151
|
+
// cannot prove falls through to the ordinary executor below, which owns all error
|
|
3152
|
+
// reporting; the count check mirrors bind-time validation so a parameter-arity mistake
|
|
3153
|
+
// still reports through the canonical path.
|
|
3154
|
+
if (!pointReadTestHooks.disabled &&
|
|
3155
|
+
options.version === undefined &&
|
|
3156
|
+
options.spillToStorage === undefined &&
|
|
3157
|
+
options.spillPageRows === undefined &&
|
|
3158
|
+
options.executionMemoryBudgetBytes === undefined &&
|
|
3159
|
+
(compiled.parameterCount ?? 0) === (options.params?.length ?? 0)) {
|
|
3160
|
+
const template = cachedPointReadTemplate(compiled);
|
|
3161
|
+
if (template !== null) {
|
|
3162
|
+
const shape = resolvePointReadShape(template, options.params ?? []);
|
|
3163
|
+
if (shape !== undefined) {
|
|
3164
|
+
const point = await this.#pointReadResult(shape, options);
|
|
3165
|
+
throwIfAborted(options.signal);
|
|
3166
|
+
if (point !== undefined)
|
|
3167
|
+
return externalizeQueryResult(point);
|
|
3168
|
+
}
|
|
3169
|
+
}
|
|
3170
|
+
}
|
|
3171
|
+
const plan = bindPlanParameters(compiled, options.params);
|
|
3143
3172
|
// Result memoization is a pure cache over the freshness probe: the catalog epoch is part
|
|
3144
3173
|
// of the key, so any commit or DDL changes the key and a hit can never be stale. Only
|
|
3145
3174
|
// plain current-version queries memoize — explicit versions, budgets, and spill options
|
|
@@ -3300,7 +3329,7 @@ export class MinnowDatabase {
|
|
|
3300
3329
|
// read has to ask the catalog. It asks by epoch — an O(1) probe the store already serves for
|
|
3301
3330
|
// result memoization — and only re-reads the view set when the catalog has actually moved.
|
|
3302
3331
|
// A database with no views therefore pays one probe, not a catalog scan per query.
|
|
3303
|
-
const { views, domains } = await this.#catalogFacts(probe);
|
|
3332
|
+
const { views, domains, columns: catalogColumns } = await this.#catalogFacts(probe);
|
|
3304
3333
|
let rewritten = plan.usesSequenceCalls === true ? await this.#resolveSequenceCalls(plan) : plan;
|
|
3305
3334
|
if (views.size > 0 && planReadsViews(plan, (name) => views.has(name))) {
|
|
3306
3335
|
const bodies = new Map();
|
|
@@ -3319,6 +3348,9 @@ export class MinnowDatabase {
|
|
|
3319
3348
|
if (domains.size > 0 && planReadsTable(rewritten, (name) => domains.has(name))) {
|
|
3320
3349
|
rewritten = normalizePlanDomainLiterals(rewritten, domains);
|
|
3321
3350
|
}
|
|
3351
|
+
const qualified = qualifyCorrelatedReferences(rewritten, catalogColumns);
|
|
3352
|
+
if (qualified !== rewritten)
|
|
3353
|
+
rewritten = optimizePlan(qualified);
|
|
3322
3354
|
if (!aliased && !natural)
|
|
3323
3355
|
return rewritten;
|
|
3324
3356
|
// These two need column *order*, which only the records carry; both are rare enough that
|
|
@@ -3431,7 +3463,9 @@ export class MinnowDatabase {
|
|
|
3431
3463
|
const views = new Map();
|
|
3432
3464
|
const childKeys = new Map();
|
|
3433
3465
|
const domains = new Map();
|
|
3466
|
+
const columns = new Map();
|
|
3434
3467
|
for (const table of await this.store.listTables()) {
|
|
3468
|
+
columns.set(table.name, table.columns.filter(({ hidden }) => hidden !== true).map(({ name }) => name));
|
|
3435
3469
|
const tableDomains = new Map(table.columns.flatMap((column) => column.sqlDomain === undefined ? [] : [[column.name, column.sqlDomain]]));
|
|
3436
3470
|
if (tableDomains.size > 0)
|
|
3437
3471
|
domains.set(table.name, tableDomains);
|
|
@@ -3445,7 +3479,7 @@ export class MinnowDatabase {
|
|
|
3445
3479
|
existing.push({ table, key });
|
|
3446
3480
|
}
|
|
3447
3481
|
}
|
|
3448
|
-
const facts = { views, childKeys, domains };
|
|
3482
|
+
const facts = { views, childKeys, domains, columns };
|
|
3449
3483
|
this.#catalogCache = { epoch, facts };
|
|
3450
3484
|
return facts;
|
|
3451
3485
|
}
|
|
@@ -3521,6 +3555,231 @@ export class MinnowDatabase {
|
|
|
3521
3555
|
prepared.close();
|
|
3522
3556
|
}
|
|
3523
3557
|
}
|
|
3558
|
+
/**
|
|
3559
|
+
* The keyed point-read fast path: a single-table conjunction of column-equals-literal
|
|
3560
|
+
* predicates covering the unique key names at most one row, so it is answered directly from
|
|
3561
|
+
* cached decoded blocks — no plan binding, streamed view, or vector pipeline. Undefined means
|
|
3562
|
+
* the statement, catalog, or physical history is not provably eligible and the ordinary
|
|
3563
|
+
* executor (which owns all error reporting and general semantics) must run. Every literal
|
|
3564
|
+
* must already match its column's storage type exactly; any coercion case falls back, so the
|
|
3565
|
+
* fast path only ever reproduces what the ordinary path would answer.
|
|
3566
|
+
*/
|
|
3567
|
+
async #pointReadResult(shape, options) {
|
|
3568
|
+
pointReadTestHooks.attempted += 1;
|
|
3569
|
+
const result = await this.#withSharedCatalogSnapshot([shape.table], (snapshot, realTables, visibility) => this.#pointReadAtSnapshot(shape, snapshot, realTables, visibility, options));
|
|
3570
|
+
if (result !== undefined) {
|
|
3571
|
+
pointReadTestHooks.served += 1;
|
|
3572
|
+
options.onStats?.({ peakMemoryBytes: 0 });
|
|
3573
|
+
}
|
|
3574
|
+
return result;
|
|
3575
|
+
}
|
|
3576
|
+
async #pointReadAtSnapshot(shape, snapshot, realTables, visibility, options) {
|
|
3577
|
+
const table = realTables.get(shape.table);
|
|
3578
|
+
if (table === undefined || table.view !== undefined)
|
|
3579
|
+
return undefined;
|
|
3580
|
+
const columnByName = new Map(table.columns.map((column) => [column.name, column]));
|
|
3581
|
+
const matchesStorageType = (column, value) => column.sqlDomain === undefined &&
|
|
3582
|
+
column.hidden !== true &&
|
|
3583
|
+
((column.type === "number" && typeof value === "number") ||
|
|
3584
|
+
(column.type === "string" && typeof value === "string") ||
|
|
3585
|
+
(column.type === "boolean" && typeof value === "boolean") ||
|
|
3586
|
+
(column.type === "datetime" && value instanceof Date));
|
|
3587
|
+
const equalityColumns = new Map();
|
|
3588
|
+
for (const equality of shape.equalities) {
|
|
3589
|
+
const column = columnByName.get(equality.column);
|
|
3590
|
+
if (column === undefined || !matchesStorageType(column, equality.value))
|
|
3591
|
+
return undefined;
|
|
3592
|
+
equalityColumns.set(column.name, column);
|
|
3593
|
+
}
|
|
3594
|
+
// Key coverage is the selectivity proof: the conjunction addresses at most one row, so
|
|
3595
|
+
// per-row candidate verification cannot degrade into an unvectorized table scan.
|
|
3596
|
+
const keyColumn = getUniqueKeyColumn(table);
|
|
3597
|
+
if (keyColumn === undefined)
|
|
3598
|
+
return undefined;
|
|
3599
|
+
const keyComponents = keyColumn.hidden === true ? primaryKeyColumns(table) : [keyColumn];
|
|
3600
|
+
if (keyComponents.length === 0)
|
|
3601
|
+
return undefined;
|
|
3602
|
+
if (!keyComponents.every((component) => equalityColumns.has(component.name))) {
|
|
3603
|
+
return undefined;
|
|
3604
|
+
}
|
|
3605
|
+
const projected = [];
|
|
3606
|
+
for (const item of shape.select) {
|
|
3607
|
+
const column = columnByName.get(item.column);
|
|
3608
|
+
if (column === undefined || column.hidden === true || column.sqlDomain !== undefined) {
|
|
3609
|
+
return undefined;
|
|
3610
|
+
}
|
|
3611
|
+
projected.push({ column, alias: item.alias });
|
|
3612
|
+
}
|
|
3613
|
+
const segments = await this.#visibleSegmentRecords(table, snapshot, visibility);
|
|
3614
|
+
if (segments.some((segment) => {
|
|
3615
|
+
const kind = segment.kind;
|
|
3616
|
+
return kind !== "insert" && kind !== "base";
|
|
3617
|
+
})) {
|
|
3618
|
+
return undefined;
|
|
3619
|
+
}
|
|
3620
|
+
// The most selective searchable component wins nothing provable without statistics, so
|
|
3621
|
+
// prefer a numeric component (zone-map pruning plus binary search) over a string one
|
|
3622
|
+
// (per-block reverse dictionary), and fall back to any equality column on the key.
|
|
3623
|
+
const searchColumn = keyComponents.find((component) => component.type === "number" || component.type === "datetime") ??
|
|
3624
|
+
keyComponents.find((component) => component.type === "string") ??
|
|
3625
|
+
keyComponents[0];
|
|
3626
|
+
if (searchColumn === undefined)
|
|
3627
|
+
return undefined;
|
|
3628
|
+
const searchEquality = shape.equalities.find((equality) => equality.column === searchColumn.name);
|
|
3629
|
+
if (searchEquality === undefined)
|
|
3630
|
+
return undefined;
|
|
3631
|
+
const searchTarget = searchEquality.value instanceof Date
|
|
3632
|
+
? dateMilliseconds(searchEquality.value)
|
|
3633
|
+
: searchEquality.value;
|
|
3634
|
+
const neededColumns = [
|
|
3635
|
+
...new Set([...equalityColumns.values(), ...projected.map((item) => item.column)]),
|
|
3636
|
+
];
|
|
3637
|
+
const rows = [];
|
|
3638
|
+
let candidateBudget = MAX_POINT_READ_CANDIDATES;
|
|
3639
|
+
for (const segment of segments) {
|
|
3640
|
+
throwIfAborted(options.signal);
|
|
3641
|
+
if (segment.rowCount === 0)
|
|
3642
|
+
continue;
|
|
3643
|
+
const searchBlockIds = segment.columnBlockIds[searchColumn.id] ?? [];
|
|
3644
|
+
if (searchBlockIds.length === 0)
|
|
3645
|
+
return undefined;
|
|
3646
|
+
for (const column of neededColumns) {
|
|
3647
|
+
if ((segment.columnBlockIds[column.id]?.length ?? 0) !== searchBlockIds.length) {
|
|
3648
|
+
return undefined;
|
|
3649
|
+
}
|
|
3650
|
+
}
|
|
3651
|
+
let blockIndexes = searchBlockIds.map((_, index) => index);
|
|
3652
|
+
// A single block cannot be pruned, so skip the zone lookup entirely for the hot
|
|
3653
|
+
// one-block OLTP table shape.
|
|
3654
|
+
if (typeof searchTarget === "number" && searchBlockIds.length > 1) {
|
|
3655
|
+
const descriptions = await this.#zoneDescriptions(searchBlockIds, snapshot);
|
|
3656
|
+
blockIndexes = blockIndexes.filter((blockIndex) => {
|
|
3657
|
+
const zone = descriptions.get(searchBlockIds[blockIndex] ?? "")?.metadata.zoneMap;
|
|
3658
|
+
return zone === undefined || (zone.min <= searchTarget && searchTarget <= zone.max);
|
|
3659
|
+
});
|
|
3660
|
+
}
|
|
3661
|
+
if (blockIndexes.length === 0)
|
|
3662
|
+
continue;
|
|
3663
|
+
const searchBlocks = await this.#decodedBlocksThroughCache(blockIndexes.map((blockIndex) => searchBlockIds[blockIndex] ?? ""), snapshot);
|
|
3664
|
+
for (const [position, blockIndex] of blockIndexes.entries()) {
|
|
3665
|
+
throwIfAborted(options.signal);
|
|
3666
|
+
const decoded = searchBlocks[position];
|
|
3667
|
+
if (decoded === undefined)
|
|
3668
|
+
return undefined;
|
|
3669
|
+
const blockId = searchBlockIds[blockIndex] ?? "";
|
|
3670
|
+
const vector = this.#blockColumnVector(blockId, decoded);
|
|
3671
|
+
if (vector.kind !== searchColumn.type)
|
|
3672
|
+
return undefined;
|
|
3673
|
+
const candidates = [];
|
|
3674
|
+
if ((vector.kind === "number" || vector.kind === "datetime") &&
|
|
3675
|
+
typeof searchTarget === "number") {
|
|
3676
|
+
if (decoded.description.nullCount === 0 && valuesAreAscending(vector.values)) {
|
|
3677
|
+
const run = equalRunRange(vector.values, searchTarget);
|
|
3678
|
+
for (let slot = run.begin; slot < run.end; slot += 1)
|
|
3679
|
+
candidates.push(slot);
|
|
3680
|
+
}
|
|
3681
|
+
else {
|
|
3682
|
+
for (let slot = 0; slot < vector.length; slot += 1) {
|
|
3683
|
+
if (vector.values[slot] === searchTarget) {
|
|
3684
|
+
// A null slot's backing value is unspecified; confirm through validity.
|
|
3685
|
+
const value = vectorValue(vector, slot);
|
|
3686
|
+
if (value !== null)
|
|
3687
|
+
candidates.push(slot);
|
|
3688
|
+
}
|
|
3689
|
+
}
|
|
3690
|
+
}
|
|
3691
|
+
}
|
|
3692
|
+
else if (vector.kind === "string" && typeof searchTarget === "string") {
|
|
3693
|
+
const code = this.#dictionaryCode(blockId, vector, searchTarget);
|
|
3694
|
+
if (code === undefined)
|
|
3695
|
+
continue;
|
|
3696
|
+
for (let slot = 0; slot < vector.length; slot += 1) {
|
|
3697
|
+
if (vector.codes[slot] === code)
|
|
3698
|
+
candidates.push(slot);
|
|
3699
|
+
}
|
|
3700
|
+
}
|
|
3701
|
+
else if (vector.kind === "boolean" && typeof searchTarget === "boolean") {
|
|
3702
|
+
for (let slot = 0; slot < vector.length; slot += 1) {
|
|
3703
|
+
if (vectorValue(vector, slot) === searchTarget)
|
|
3704
|
+
candidates.push(slot);
|
|
3705
|
+
}
|
|
3706
|
+
}
|
|
3707
|
+
else {
|
|
3708
|
+
return undefined;
|
|
3709
|
+
}
|
|
3710
|
+
if (candidates.length === 0)
|
|
3711
|
+
continue;
|
|
3712
|
+
candidateBudget -= candidates.length;
|
|
3713
|
+
if (candidateBudget < 0)
|
|
3714
|
+
return undefined;
|
|
3715
|
+
const remainingColumns = neededColumns.filter((column) => column.id !== searchColumn.id);
|
|
3716
|
+
const rowBlockIds = remainingColumns.map((column) => segment.columnBlockIds[column.id]?.[blockIndex] ?? "");
|
|
3717
|
+
const rowBlocks = await this.#decodedBlocksThroughCache(rowBlockIds, snapshot);
|
|
3718
|
+
const vectors = new Map([[searchColumn.name, vector]]);
|
|
3719
|
+
for (const [index, column] of remainingColumns.entries()) {
|
|
3720
|
+
const block = rowBlocks[index];
|
|
3721
|
+
if (block === undefined)
|
|
3722
|
+
return undefined;
|
|
3723
|
+
const columnVector = this.#blockColumnVector(rowBlockIds[index] ?? "", block);
|
|
3724
|
+
if (columnVector.kind !== column.type)
|
|
3725
|
+
return undefined;
|
|
3726
|
+
vectors.set(column.name, columnVector);
|
|
3727
|
+
}
|
|
3728
|
+
for (const slot of candidates) {
|
|
3729
|
+
let matches = true;
|
|
3730
|
+
for (const equality of shape.equalities) {
|
|
3731
|
+
const columnVector = vectors.get(equality.column);
|
|
3732
|
+
if (columnVector === undefined)
|
|
3733
|
+
return undefined;
|
|
3734
|
+
const stored = vectorValue(columnVector, slot);
|
|
3735
|
+
const wanted = equality.value;
|
|
3736
|
+
const equal = wanted instanceof Date
|
|
3737
|
+
? stored instanceof Date && dateMilliseconds(stored) === dateMilliseconds(wanted)
|
|
3738
|
+
: stored === wanted;
|
|
3739
|
+
if (!equal) {
|
|
3740
|
+
matches = false;
|
|
3741
|
+
break;
|
|
3742
|
+
}
|
|
3743
|
+
}
|
|
3744
|
+
if (!matches)
|
|
3745
|
+
continue;
|
|
3746
|
+
const row = {};
|
|
3747
|
+
for (const item of projected) {
|
|
3748
|
+
const columnVector = vectors.get(item.column.name);
|
|
3749
|
+
if (columnVector === undefined)
|
|
3750
|
+
return undefined;
|
|
3751
|
+
const value = vectorValue(columnVector, slot);
|
|
3752
|
+
// A stored plain-text value in the protected NUL namespace crosses the result
|
|
3753
|
+
// boundary through the ordinary executor's wrapping rules; reproducing them here
|
|
3754
|
+
// is not worth the risk, so the whole statement falls back.
|
|
3755
|
+
if (typeof value === "string" && value.charCodeAt(0) === 0)
|
|
3756
|
+
return undefined;
|
|
3757
|
+
row[item.alias] = value;
|
|
3758
|
+
}
|
|
3759
|
+
rows.push(row);
|
|
3760
|
+
}
|
|
3761
|
+
}
|
|
3762
|
+
}
|
|
3763
|
+
return {
|
|
3764
|
+
columns: shape.select.map((item) => item.alias),
|
|
3765
|
+
columnDomains: shape.select.map(() => null),
|
|
3766
|
+
rows,
|
|
3767
|
+
};
|
|
3768
|
+
}
|
|
3769
|
+
/** The dictionary code for a value in one block's string vector, reverse-mapped and cached. */
|
|
3770
|
+
#dictionaryCode(blockId, vector, value) {
|
|
3771
|
+
let reverse = this.#cacheGet(`drd ${blockId}`);
|
|
3772
|
+
if (reverse === undefined) {
|
|
3773
|
+
reverse = new Map();
|
|
3774
|
+
for (const [code, entry] of vector.dictionary.entries())
|
|
3775
|
+
reverse.set(entry, code);
|
|
3776
|
+
let bytes = 64;
|
|
3777
|
+
for (const entry of vector.dictionary)
|
|
3778
|
+
bytes += 32 + entry.length;
|
|
3779
|
+
this.#cachePut(`drd ${blockId}`, reverse, bytes);
|
|
3780
|
+
}
|
|
3781
|
+
return reverse.get(value);
|
|
3782
|
+
}
|
|
3524
3783
|
/**
|
|
3525
3784
|
* Streams a one-column snapshot projection into bounded pages. DELETE uses this before its
|
|
3526
3785
|
* first staged write: selection keeps all ordinary pruning/predicate semantics without
|
|
@@ -3530,6 +3789,8 @@ export class MinnowDatabase {
|
|
|
3530
3789
|
async #queryCompiledFirstColumn(plan, probe) {
|
|
3531
3790
|
const options = this.#effectiveQueryOptions({ memoize: false });
|
|
3532
3791
|
plan = await this.#applyCatalogRewrites(plan, probe);
|
|
3792
|
+
if (!this.#canStreamPlanShape(plan, options))
|
|
3793
|
+
return undefined;
|
|
3533
3794
|
const values = [];
|
|
3534
3795
|
const streamed = await this.#queryStreamed(plan, options, undefined, probe, {
|
|
3535
3796
|
batchRows: 2_048,
|
|
@@ -5315,9 +5576,9 @@ export class MinnowDatabase {
|
|
|
5315
5576
|
const plannedColumn = writeColumnValues(keyColumn.type, values);
|
|
5316
5577
|
const ranges = writeBlockRanges([plannedColumn], values.length, this.#rowsPerBlock, this.#targetBlockBytes);
|
|
5317
5578
|
for (const [part, { start, end }] of ranges.entries()) {
|
|
5318
|
-
await blockStager.prepare(maximumWriteBlockStoredBytes(plannedColumn, start, end,
|
|
5579
|
+
await blockStager.prepare(maximumWriteBlockStoredBytes(plannedColumn, start, end, "raw"));
|
|
5319
5580
|
const slice = values.slice(start, end);
|
|
5320
|
-
const bytes = await this.#encodeColumnBlock(keyColumn.id, asColumnInput(keyColumn.type, slice));
|
|
5581
|
+
const bytes = await this.#encodeColumnBlock(keyColumn.id, asColumnInput(keyColumn.type, slice), "raw");
|
|
5321
5582
|
const blockId = [
|
|
5322
5583
|
"table",
|
|
5323
5584
|
table.id,
|
|
@@ -5810,6 +6071,20 @@ export class MinnowDatabase {
|
|
|
5810
6071
|
const table = reported.base.derived ?? reported.base.union ?? reported.base.windowed;
|
|
5811
6072
|
if (table === undefined && reported.joins.length === 0) {
|
|
5812
6073
|
const record = await this.#findTable(reported.base.table);
|
|
6074
|
+
const pointTemplate = record.view === undefined ? cachedPointReadTemplate(reported) : null;
|
|
6075
|
+
if (pointTemplate !== null) {
|
|
6076
|
+
const keyColumn = getUniqueKeyColumn(record);
|
|
6077
|
+
const keyComponents = keyColumn === undefined
|
|
6078
|
+
? []
|
|
6079
|
+
: keyColumn.hidden === true
|
|
6080
|
+
? primaryKeyColumns(record)
|
|
6081
|
+
: [keyColumn];
|
|
6082
|
+
const covered = new Set(pointTemplate.equalities.map((equality) => equality.column));
|
|
6083
|
+
if (keyComponents.length > 0 &&
|
|
6084
|
+
keyComponents.every((component) => covered.has(component.name))) {
|
|
6085
|
+
notes.push("key-covering equality answers as a point read when the visible history allows");
|
|
6086
|
+
}
|
|
6087
|
+
}
|
|
5813
6088
|
if (zonePredicates(reported, record).length > 0) {
|
|
5814
6089
|
notes.push("zone-map pruning applies to the unbudgeted scan");
|
|
5815
6090
|
}
|
|
@@ -7291,7 +7566,7 @@ export class MinnowDatabase {
|
|
|
7291
7566
|
}
|
|
7292
7567
|
}
|
|
7293
7568
|
}
|
|
7294
|
-
const plan = {
|
|
7569
|
+
const plan = optimizePlan({
|
|
7295
7570
|
sql: `(${statement.kind})`,
|
|
7296
7571
|
base: { table: table.name, alias: table.name },
|
|
7297
7572
|
joins: [],
|
|
@@ -7303,7 +7578,7 @@ export class MinnowDatabase {
|
|
|
7303
7578
|
groupBy: [],
|
|
7304
7579
|
having: [],
|
|
7305
7580
|
orderBy: [],
|
|
7306
|
-
};
|
|
7581
|
+
});
|
|
7307
7582
|
if (statement.kind === "delete" && returningColumns === undefined) {
|
|
7308
7583
|
const internalWriter = options.writer;
|
|
7309
7584
|
if (internalWriter.queryFirstColumn !== undefined) {
|
|
@@ -14176,8 +14451,15 @@ export class MinnowDatabase {
|
|
|
14176
14451
|
* the codec each block actually used is recorded in the block itself, so a wrong guess costs
|
|
14177
14452
|
* bytes and never correctness.
|
|
14178
14453
|
*/
|
|
14179
|
-
async #encodeColumnBlock(columnId, input
|
|
14180
|
-
|
|
14454
|
+
async #encodeColumnBlock(columnId, input,
|
|
14455
|
+
/**
|
|
14456
|
+
* "raw" for delete-key blocks: they hold only the doomed keys, compaction folds them away,
|
|
14457
|
+
* and a 50k-key range delete otherwise spends more time in gzip than in the whole rest of
|
|
14458
|
+
* the statement. The stored format carries the codec per block, so nothing changes on disk
|
|
14459
|
+
* beyond the byte count.
|
|
14460
|
+
*/
|
|
14461
|
+
preferred = this.#compression) {
|
|
14462
|
+
return this.#encodePreferredBlock(columnId, preferred, columnInputBytesBelow(input, GZIP_MINIMUM_INPUT_BYTES), (compression) => encodeBlock(input, compression));
|
|
14181
14463
|
}
|
|
14182
14464
|
/**
|
|
14183
14465
|
* Applies the same adaptive gzip rule to ordinary writes and compaction output. `gzip` is a
|
|
@@ -6,6 +6,13 @@ import { type CompiledQuery } from "./query.js";
|
|
|
6
6
|
* predicates containing subqueries stay self-contained when they move.
|
|
7
7
|
*/
|
|
8
8
|
export declare function optimizePlan(plan: CompiledQuery): CompiledQuery;
|
|
9
|
+
/**
|
|
10
|
+
* Resolves an unqualified column in a nested block against the nearest enclosing query scope
|
|
11
|
+
* when that name does not exist in the nested block itself. Compilation has no catalog, so this
|
|
12
|
+
* SQL name-resolution step runs later for database-backed plans and returns the original plan
|
|
13
|
+
* unchanged when it finds nothing to qualify.
|
|
14
|
+
*/
|
|
15
|
+
export declare function qualifyCorrelatedReferences(plan: CompiledQuery, tableColumns: ReadonlyMap<string, readonly string[]>): CompiledQuery;
|
|
9
16
|
/**
|
|
10
17
|
* Cost-based build-side selection using prepared or catalog-derived row counts. A single inner
|
|
11
18
|
* equi-join probes from the base into an index over the joined table, so a joined input more than
|