@minnowdb/core 0.5.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/engine/cancellation.d.ts +2 -0
- package/dist/engine/cancellation.js +4 -0
- package/dist/engine/catalog.d.ts +3 -1
- package/dist/engine/catalog.js +1 -0
- package/dist/engine/client.d.ts +32 -4
- package/dist/engine/client.js +82 -15
- package/dist/engine/database.d.ts +23 -14
- package/dist/engine/database.js +528 -79
- package/dist/engine/defaults.js +11 -0
- package/dist/engine/errors.d.ts +13 -0
- package/dist/engine/errors.js +22 -0
- package/dist/engine/fts.d.ts +2 -15
- package/dist/engine/live.d.ts +1 -7
- package/dist/engine/live.js +2 -12
- package/dist/engine/optimizer.d.ts +7 -0
- package/dist/engine/optimizer.js +1349 -76
- package/dist/engine/query.d.ts +11 -278
- package/dist/engine/query.js +178 -49
- package/dist/engine/schema-wire.d.ts +7 -1
- package/dist/engine/schema-wire.js +4 -0
- package/dist/engine/schema.d.ts +67 -33
- package/dist/engine/schema.js +138 -7
- package/dist/engine/sql-domains.d.ts +8 -0
- package/dist/engine/sql-domains.js +25 -0
- package/dist/engine/sql-json.js +22 -3
- package/dist/engine/vector.d.ts +2 -2
- package/dist/engine/vector.js +369 -43
- package/dist/engine/worker-host.js +119 -44
- package/dist/plan/index.d.ts +5 -4
- package/dist/plan/index.js +3 -3
- package/dist/plan/model.d.ts +224 -0
- package/dist/plan/model.js +1 -0
- package/dist/storage/types.d.ts +7 -0
- package/dist/storage/types.js +16 -0
- package/dist/transactions/index.d.ts +5 -3
- package/dist/transactions/index.js +58 -8
- package/dist/worker-protocol/index.d.ts +6 -1
- package/dist/worker-protocol/index.js +5 -2
- package/package.json +1 -1
- package/postgres-feature-profile.json +5 -0
- package/sql-feature-matrix.json +89 -21
package/dist/engine/database.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { toColumnarBatch, } from "./batch.js";
|
|
2
2
|
import { ArtifactCache } from "./artifact-cache.js";
|
|
3
|
+
import { throwIfAborted } from "./cancellation.js";
|
|
3
4
|
import { BufferedTableWriter } from "./buffered-writer.js";
|
|
4
5
|
export { attachLifecycleFlush, BufferedTableWriter, MAX_BUFFERED_WRITER_PENDING_ADDS, } from "./buffered-writer.js";
|
|
5
|
-
import { CompactionJobCancelledError, CompactionMemoryBudgetError, CompactionWriteAmplificationError, MaintenanceBacklogError, MissingKeyError, SqlCompileError, UnknownTableError, UniqueConstraintError, VisibleSegmentCursorStaleError, } from "./errors.js";
|
|
6
|
+
import { CompactionJobCancelledError, CompactionMemoryBudgetError, CompactionWriteAmplificationError, DatabaseReadBacklogError, MaintenanceBacklogError, MissingKeyError, SqlCompileError, UnknownTableError, UniqueConstraintError, VisibleSegmentCursorStaleError, } from "./errors.js";
|
|
7
|
+
export { DatabaseReadBacklogError } from "./errors.js";
|
|
6
8
|
import { BLOCK_HEADER_LENGTH, buildPhysicalColumnFromRanges, decodeBlock, decodePhysicalBlock, encodeBlock, encodePhysicalBlock, getCompressionMemoryBound, inspectBlock, maximumPhysicalBlockByteLength, MAX_BLOCK_METADATA_BYTE_LENGTH, MAX_BLOCK_ROW_COUNT, MAX_PHYSICAL_COLUMN_BYTE_LENGTH, MAX_STORED_BLOCK_BYTE_LENGTH, measurePhysicalColumnRanges, physicalColumnByteLength, slicePhysicalColumn, StoredBlockPayloadTooLargeError, wellFormedUtf8ByteLength, } from "../block-format/index.js";
|
|
7
9
|
import { dateIsoString, dateMilliseconds } from "../date-value.js";
|
|
8
10
|
import { estimateCompactionRowsPerOutput, planAlignedWriteBlockRanges as writeBlockRanges, } from "./write-block-planner.js";
|
|
@@ -16,11 +18,11 @@ import { applyWindowFunctions, bindPlanParameters, bindStatementParameters, DUAL
|
|
|
16
18
|
import { copyQueryResult, planMemoKey, queryResultMemoKey, queryResultRetainedBytes, RESULT_MEMO_MAX_BYTES, } from "./query-cache.js";
|
|
17
19
|
import { QueryMemoryBudgetError, QueryMemoryContext, DEFAULT_QUERY_MEMORY_BUDGET_BYTES, } from "./memory.js";
|
|
18
20
|
import { LiveQueryLimitError, LiveQuerySet, MAX_LIVE_QUERY_SETS_PER_DATABASE, } from "./live.js";
|
|
19
|
-
import { chooseJoinOrder, renderPlan } from "./optimizer.js";
|
|
21
|
+
import { chooseJoinOrder, optimizePlan, qualifyCorrelatedReferences, renderPlan, } from "./optimizer.js";
|
|
20
22
|
import { encodeSqlEqualityValue } from "./sql-semantics.js";
|
|
21
23
|
import { externalSqlDomainValue, externalSqlTextValue, isDateDomainValue, isSqlDomainValue, normalizeSqlDomainValue, protectedSqlTextValue, } from "./sql-domains.js";
|
|
22
24
|
import { toCatalog } from "./catalog.js";
|
|
23
|
-
import { applyColumnSteps, assertColumnDroppable, declaredForeignKeys, isDestructiveStep, planMigration, } from "./schema.js";
|
|
25
|
+
import { applyColumnSteps, assertColumnDroppable, compileGeneratedColumnExpression, declaredForeignKeys, isDestructiveStep, planMigration, } from "./schema.js";
|
|
24
26
|
import { columnarTableFromRows, createColumnarTable, vectorValue, } from "./vector.js";
|
|
25
27
|
// The on-disk format is little-endian; the bulk Float64 copy reads platform order.
|
|
26
28
|
const PLATFORM_LITTLE_ENDIAN = new Uint8Array(new Uint32Array([1]).buffer)[0] === 1;
|
|
@@ -103,14 +105,6 @@ export const MAX_TRANSACTION_SAVEPOINT_BYTES = 8 * 1024 * 1024;
|
|
|
103
105
|
export const MAX_DATABASE_PENDING_WRITES = 64;
|
|
104
106
|
/** Concurrent direct reads retained by one database instance. Reads still execute in parallel. */
|
|
105
107
|
export const MAX_DATABASE_ACTIVE_READS = 256;
|
|
106
|
-
export class DatabaseReadBacklogError extends Error {
|
|
107
|
-
limit;
|
|
108
|
-
name = "DatabaseReadBacklogError";
|
|
109
|
-
constructor(limit = MAX_DATABASE_ACTIVE_READS) {
|
|
110
|
-
super(`A database cannot retain more than ${String(limit)} active reads; await a read`);
|
|
111
|
-
this.limit = limit;
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
108
|
/** Commits tolerated while collection fails before foreground writes assist and backpressure. */
|
|
115
109
|
const AUTO_COLLECT_MAX_DEBT_COMMITS = 4_096;
|
|
116
110
|
/** Failed background passes retry without another write, with bounded exponential backoff. */
|
|
@@ -291,17 +285,29 @@ function isTransactionalStatement(statement) {
|
|
|
291
285
|
statement.kind === "delete" ||
|
|
292
286
|
statement.kind === "select");
|
|
293
287
|
}
|
|
288
|
+
function returningExecuteFields(table, columns, rows) {
|
|
289
|
+
return {
|
|
290
|
+
returnedRows: rows,
|
|
291
|
+
returnedColumns: [...columns],
|
|
292
|
+
returnedColumnDomains: columns.map((name) => table.columns.find((column) => column.name === name)?.sqlDomain ?? null),
|
|
293
|
+
};
|
|
294
|
+
}
|
|
294
295
|
function externalizeExecuteResult(result) {
|
|
295
296
|
if (!("returnedRows" in result) || !Array.isArray(result.returnedRows))
|
|
296
297
|
return result;
|
|
297
|
-
const columns =
|
|
298
|
+
const columns = result.returnedColumns ?? [
|
|
299
|
+
...new Set(result.returnedRows.flatMap((row) => Object.keys(row))),
|
|
300
|
+
];
|
|
301
|
+
const columnDomains = result.returnedColumnDomains ?? unknownColumnDomains(columns);
|
|
298
302
|
return {
|
|
299
303
|
...result,
|
|
300
304
|
returnedRows: externalizeQueryResult({
|
|
301
305
|
columns,
|
|
302
|
-
columnDomains
|
|
306
|
+
columnDomains,
|
|
303
307
|
rows: result.returnedRows,
|
|
304
308
|
}).rows,
|
|
309
|
+
returnedColumns: [...columns],
|
|
310
|
+
returnedColumnDomains: structuredClone(columnDomains),
|
|
305
311
|
};
|
|
306
312
|
}
|
|
307
313
|
/** Rejects an INSERT value that is still an unbound `?`/`$n` slot. */
|
|
@@ -571,6 +577,8 @@ export class MinnowDatabase {
|
|
|
571
577
|
#droppingTables = new Set();
|
|
572
578
|
/** Per table: the visible segment count a failed auto-compaction must see before retrying. */
|
|
573
579
|
#autoCompactionBackoff = new Map();
|
|
580
|
+
/** Exact current-layout counters; local commits advance them without rescanning history. */
|
|
581
|
+
#autoCompactionHints = new Map();
|
|
574
582
|
/** Data commits per table since its last write-path auto-compaction check. */
|
|
575
583
|
#commitsSinceCompactionCheck = new Map();
|
|
576
584
|
/** The compaction step in flight per table, so steps on one table run one at a time. */
|
|
@@ -749,6 +757,7 @@ export class MinnowDatabase {
|
|
|
749
757
|
this.#autoCompactionsRequested.clear();
|
|
750
758
|
this.#idleCompactionTableIds.clear();
|
|
751
759
|
this.#autoCompactionBackoff.clear();
|
|
760
|
+
this.#autoCompactionHints.clear();
|
|
752
761
|
this.#commitsSinceCompactionCheck.clear();
|
|
753
762
|
this.#artifactCache.clear();
|
|
754
763
|
}
|
|
@@ -813,6 +822,9 @@ export class MinnowDatabase {
|
|
|
813
822
|
...(sqlDomain === undefined ? {} : { sqlDomain: structuredClone(sqlDomain) }),
|
|
814
823
|
nullable: column.nullable ?? false,
|
|
815
824
|
...(column.defaultValue === undefined ? {} : { defaultValue: column.defaultValue }),
|
|
825
|
+
...(column.generatedValue === undefined
|
|
826
|
+
? {}
|
|
827
|
+
: { generatedValue: structuredClone(column.generatedValue) }),
|
|
816
828
|
...(column.enumValues === undefined
|
|
817
829
|
? {}
|
|
818
830
|
: { enumValues: validateEnumValues(column.enumValues, columnName) }),
|
|
@@ -871,6 +883,15 @@ export class MinnowDatabase {
|
|
|
871
883
|
throw new TypeError(`Unique key cannot be nullable: ${uniqueKeyColumn.name}`);
|
|
872
884
|
}
|
|
873
885
|
for (const column of columns) {
|
|
886
|
+
if (column.generatedValue !== undefined) {
|
|
887
|
+
if (column.defaultValue !== undefined || column.backfill !== undefined) {
|
|
888
|
+
throw new TypeError(`A generated column cannot also have a default or backfill: ${name}.${column.name}`);
|
|
889
|
+
}
|
|
890
|
+
compileGeneratedColumnExpression(name, column.name, column.generatedValue.sql, columns);
|
|
891
|
+
if (column === uniqueKeyColumn || compositePrimaryColumns.includes(column)) {
|
|
892
|
+
throw new TypeError(`Generated columns cannot be row-addressing keys: ${name}.${column.name}`);
|
|
893
|
+
}
|
|
894
|
+
}
|
|
874
895
|
if (column.defaultValue !== undefined) {
|
|
875
896
|
validateColumnDefault({ ...column, isUniqueKey: column === uniqueKeyColumn }, column.defaultValue);
|
|
876
897
|
if (column.defaultValue.kind === "literal" && column.sqlDomain !== undefined) {
|
|
@@ -1059,6 +1080,9 @@ export class MinnowDatabase {
|
|
|
1059
1080
|
: { sqlDomain: structuredClone(column.sqlDomain) }),
|
|
1060
1081
|
nullable: column.nullable,
|
|
1061
1082
|
...(column.defaultValue === undefined ? {} : { defaultValue: column.defaultValue }),
|
|
1083
|
+
...(column.generatedValue === undefined
|
|
1084
|
+
? {}
|
|
1085
|
+
: { generatedValue: column.generatedValue }),
|
|
1062
1086
|
...(column.enumValues === undefined ? {} : { enumValues: [...column.enumValues] }),
|
|
1063
1087
|
})),
|
|
1064
1088
|
...(uniqueKey === undefined ? {} : { uniqueKey }),
|
|
@@ -1129,7 +1153,7 @@ export class MinnowDatabase {
|
|
|
1129
1153
|
}
|
|
1130
1154
|
async #withReadReservation(run) {
|
|
1131
1155
|
if (this.#activeReadReservations >= MAX_DATABASE_ACTIVE_READS) {
|
|
1132
|
-
throw new DatabaseReadBacklogError();
|
|
1156
|
+
throw new DatabaseReadBacklogError(MAX_DATABASE_ACTIVE_READS);
|
|
1133
1157
|
}
|
|
1134
1158
|
this.#activeReadReservations += 1;
|
|
1135
1159
|
try {
|
|
@@ -1140,7 +1164,9 @@ export class MinnowDatabase {
|
|
|
1140
1164
|
}
|
|
1141
1165
|
}
|
|
1142
1166
|
async #queryWithReadReservation(sql, options) {
|
|
1167
|
+
throwIfAborted(options.signal);
|
|
1143
1168
|
await this.#settleExpiredStatementTransaction();
|
|
1169
|
+
throwIfAborted(options.signal);
|
|
1144
1170
|
// An open statement transaction routes through its session, which owns the reservation for
|
|
1145
1171
|
// the actual execution. Avoid charging the same query twice at this public boundary.
|
|
1146
1172
|
return this.#openTransaction === undefined
|
|
@@ -1569,6 +1595,7 @@ export class MinnowDatabase {
|
|
|
1569
1595
|
for (const column of table.columns)
|
|
1570
1596
|
this.#gzipVerdicts.delete(column.id);
|
|
1571
1597
|
this.#autoCompactionBackoff.delete(table.id);
|
|
1598
|
+
this.#autoCompactionHints.delete(table.id);
|
|
1572
1599
|
this.#commitsSinceCompactionCheck.delete(table.id);
|
|
1573
1600
|
this.#idleCompactionTableIds.delete(table.id);
|
|
1574
1601
|
const postingPrefix = `${table.id}/`;
|
|
@@ -1634,6 +1661,7 @@ export class MinnowDatabase {
|
|
|
1634
1661
|
Object.values(pivoted.columns).find((values) => values.length > 0)?.length ??
|
|
1635
1662
|
0;
|
|
1636
1663
|
normalizeDomainBatch(table, filled.batch);
|
|
1664
|
+
fillStoredGeneratedColumns(table, filled.batch, inputRowCount, filled.generated);
|
|
1637
1665
|
for (const name of filled.generated.keys()) {
|
|
1638
1666
|
const column = table.columns.find((candidate) => candidate.name === name);
|
|
1639
1667
|
const values = filled.batch.columns[name];
|
|
@@ -1720,6 +1748,7 @@ export class MinnowDatabase {
|
|
|
1720
1748
|
throw new TypeError(`Table needs a unique key before rows can be updated: ${table.name}`);
|
|
1721
1749
|
}
|
|
1722
1750
|
const normalizedInput = normalizeDomainUpdate(table, input);
|
|
1751
|
+
rejectGeneratedUpdateAssignments(table, normalizedInput);
|
|
1723
1752
|
const keys = validateUpdateBatch(table, keyColumn, normalizedInput);
|
|
1724
1753
|
const current = await this.store.getTable(table.id);
|
|
1725
1754
|
if (current === undefined)
|
|
@@ -1737,6 +1766,10 @@ export class MinnowDatabase {
|
|
|
1737
1766
|
changes: Object.fromEntries(Object.entries(changes).map(([name, value]) => [name, [value]])),
|
|
1738
1767
|
});
|
|
1739
1768
|
}
|
|
1769
|
+
/** Deletes one row by the table's unique key. */
|
|
1770
|
+
async delete(tableName, key) {
|
|
1771
|
+
return this.deleteBatch(tableName, { keys: [key] });
|
|
1772
|
+
}
|
|
1740
1773
|
deleteBatch(tableName, input) {
|
|
1741
1774
|
return this.#withWriteReservation(() => this.#deleteBatchReserved(tableName, input));
|
|
1742
1775
|
}
|
|
@@ -1972,15 +2005,13 @@ export class MinnowDatabase {
|
|
|
1972
2005
|
async #writeUpdateBatch(table, keyColumn, input, keys) {
|
|
1973
2006
|
await this.#assertCompactionCapacity(table);
|
|
1974
2007
|
const started = performance.now();
|
|
1975
|
-
|
|
1976
|
-
Object.values(input.changes).reduce((total, values) => total + estimateValuesBytes(values), 0);
|
|
2008
|
+
let logicalBytes;
|
|
1977
2009
|
// Deferred: the record rides the single-shot commit below unless trigger rows stage first.
|
|
1978
2010
|
const transaction = await this.#transactions.beginDeferred({ durableSnapshot: false });
|
|
1979
2011
|
transaction.limitLevelZeroSegments(table.id, MAX_LEVEL_ZERO_SEGMENTS);
|
|
1980
2012
|
const blockStager = new BoundedWriteBlockStager(transaction);
|
|
1981
2013
|
const segmentId = this.#createId();
|
|
1982
2014
|
const columnBlockIds = {};
|
|
1983
|
-
const changedColumns = Object.keys(input.changes).sort();
|
|
1984
2015
|
let storedBytes = 0;
|
|
1985
2016
|
let blockCount = 0;
|
|
1986
2017
|
let encodeMs = 0;
|
|
@@ -1989,6 +2020,22 @@ export class MinnowDatabase {
|
|
|
1989
2020
|
let retries = 0;
|
|
1990
2021
|
try {
|
|
1991
2022
|
await this.#assertKeysExist(table, keyColumn, transaction.snapshotVersion, keys);
|
|
2023
|
+
const checks = table.checks ?? [];
|
|
2024
|
+
let changedForeignKey = (table.foreignKeys ?? []).some((key) => key.enforced !== false &&
|
|
2025
|
+
foreignKeyColumns(key).some((column) => input.changes[column] !== undefined));
|
|
2026
|
+
const preImages = await this.#triggerPreImages(table, keyColumn, input.keys.filter((key) => key !== null), "update", (sql, params) => this.#sessionQuery(transaction, sql, { params }), checks.length > 0 ||
|
|
2027
|
+
changedForeignKey ||
|
|
2028
|
+
tableGeneratedExpressions(table).length > 0 ||
|
|
2029
|
+
secondaryIndexUpdateNeedsPreImages(table, input) ||
|
|
2030
|
+
readyUniqueSecondaryIndexes(table).some(({ columns }) => columns.some((column) => input.changes[column.name] !== undefined)));
|
|
2031
|
+
input = applyStoredGeneratedUpdateChanges(table, input, preImages);
|
|
2032
|
+
validateUpdateBatch(table, keyColumn, input);
|
|
2033
|
+
changedForeignKey = (table.foreignKeys ?? []).some((key) => key.enforced !== false &&
|
|
2034
|
+
foreignKeyColumns(key).some((column) => input.changes[column] !== undefined));
|
|
2035
|
+
logicalBytes =
|
|
2036
|
+
estimateValuesBytes(input.keys) +
|
|
2037
|
+
Object.values(input.changes).reduce((total, values) => total + estimateValuesBytes(values), 0);
|
|
2038
|
+
const changedColumns = Object.keys(input.changes).sort();
|
|
1992
2039
|
const columns = [keyColumn, ...changedColumns.map((name) => findColumn(table, name))];
|
|
1993
2040
|
const plannedColumns = columns.map((column) => writeColumnValues(column.type, column.id === keyColumn.id ? input.keys : (input.changes[column.name] ?? [])));
|
|
1994
2041
|
const ranges = writeBlockRanges(plannedColumns, input.keys.length, this.#rowsPerBlock, this.#targetBlockBytes);
|
|
@@ -2022,13 +2069,6 @@ export class MinnowDatabase {
|
|
|
2022
2069
|
}
|
|
2023
2070
|
columnBlockIds[column.id] = blockIds;
|
|
2024
2071
|
}
|
|
2025
|
-
const checks = table.checks ?? [];
|
|
2026
|
-
const changedForeignKey = (table.foreignKeys ?? []).some((key) => key.enforced !== false &&
|
|
2027
|
-
foreignKeyColumns(key).some((column) => input.changes[column] !== undefined));
|
|
2028
|
-
const preImages = await this.#triggerPreImages(table, keyColumn, input.keys.filter((key) => key !== null), "update", (sql, params) => this.#sessionQuery(transaction, sql, { params }), checks.length > 0 ||
|
|
2029
|
-
changedForeignKey ||
|
|
2030
|
-
secondaryIndexUpdateNeedsPreImages(table, input) ||
|
|
2031
|
-
readyUniqueSecondaryIndexes(table).some(({ columns }) => columns.some((column) => input.changes[column.name] !== undefined)));
|
|
2032
2072
|
stageSecondaryUniqueMutationChanges(transaction, table, input, preImages);
|
|
2033
2073
|
const secondaryDeltas = buildSecondaryUpdateDeltas(table, input, preImages);
|
|
2034
2074
|
if (secondaryDeltas.length > 0) {
|
|
@@ -2101,6 +2141,14 @@ export class MinnowDatabase {
|
|
|
2101
2141
|
blockCount,
|
|
2102
2142
|
storedBytes,
|
|
2103
2143
|
version: manifest.version,
|
|
2144
|
+
...(tableGeneratedExpressions(table).length === 0
|
|
2145
|
+
? {}
|
|
2146
|
+
: {
|
|
2147
|
+
generatedColumns: Object.fromEntries(tableGeneratedExpressions(table).map(({ column }) => [
|
|
2148
|
+
column.name,
|
|
2149
|
+
(input.changes[column.name] ?? []).map((value) => externalSqlDomainValue(value)),
|
|
2150
|
+
])),
|
|
2151
|
+
}),
|
|
2104
2152
|
metrics: createWriteMetrics({
|
|
2105
2153
|
logicalBytes,
|
|
2106
2154
|
storedBytes,
|
|
@@ -2554,12 +2602,14 @@ export class MinnowDatabase {
|
|
|
2554
2602
|
}
|
|
2555
2603
|
async #prepareCompiledPlan(plan, options = {}, probe) {
|
|
2556
2604
|
options = this.#effectiveQueryOptions(options);
|
|
2605
|
+
throwIfAborted(options.signal);
|
|
2557
2606
|
// The ORDER-BY-expression desugar's wrapper is projection-only: prepare the inner block
|
|
2558
2607
|
// directly (no derived materialization) and project each result to the visible aliases,
|
|
2559
2608
|
// so `.search()` costs the same whether or not the caller also selects the score.
|
|
2560
2609
|
const wrapper = transparentProjectionSource(plan);
|
|
2561
2610
|
if (wrapper !== undefined) {
|
|
2562
2611
|
const prepared = await this.#prepareCompiledPlan(wrapper.inner, options, probe);
|
|
2612
|
+
throwIfAborted(options.signal);
|
|
2563
2613
|
return {
|
|
2564
2614
|
sql: prepared.sql,
|
|
2565
2615
|
tables: prepared.tables,
|
|
@@ -2580,6 +2630,7 @@ export class MinnowDatabase {
|
|
|
2580
2630
|
let outputNeedsExternalization = true;
|
|
2581
2631
|
let outputColumnDomains = [];
|
|
2582
2632
|
const prepareAtSnapshot = async (snapshot, realTables, visibility) => {
|
|
2633
|
+
throwIfAborted(options.signal);
|
|
2583
2634
|
const typedSchemas = new Map([...realTables.values()].map((table) => [
|
|
2584
2635
|
table.name,
|
|
2585
2636
|
table.columns.map(({ name, type, integer, sqlDomain }) => ({
|
|
@@ -2595,30 +2646,37 @@ export class MinnowDatabase {
|
|
|
2595
2646
|
const expandedPlan = expandFtsColumns(plan, (tableName) => searchableFtsColumns(realTables.get(tableName)));
|
|
2596
2647
|
const resolution = subqueryResolutionSteps(expandedPlan);
|
|
2597
2648
|
for (const step of resolution.steps) {
|
|
2598
|
-
|
|
2649
|
+
throwIfAborted(options.signal);
|
|
2650
|
+
step.substitute(await this.#executeBlockCached(step.block, snapshot, visibility, memory, realTables, typedSchemas, options.memoize !== false, options.spillToStorage !== false, options.spillToStorage === true, options.spillPageRows, options.signal));
|
|
2651
|
+
throwIfAborted(options.signal);
|
|
2599
2652
|
}
|
|
2600
2653
|
resolvedPlan = resolution.plan;
|
|
2601
2654
|
// Index-served BM25 statistics, computed against the same catalog snapshot the pruner
|
|
2602
2655
|
// reads, so a pruned scoring scan always carries exact corpus numbers.
|
|
2603
2656
|
ftsStats = await this.#ftsIndexStats(resolvedPlan, realTables, snapshot, visibility);
|
|
2657
|
+
throwIfAborted(options.signal);
|
|
2604
2658
|
columnarTables = await this.#prepareBlockInputs(resolvedPlan, snapshot, visibility, memory, realTables, typedSchemas, undefined,
|
|
2605
2659
|
// memoize: false means "compute this statement's results" — that covers the
|
|
2606
2660
|
// columnar forms of derived and windowed sources too, not just the result memo.
|
|
2607
|
-
options.memoize !== false, options.spillToStorage !== false, options.spillToStorage === true, options.spillPageRows);
|
|
2661
|
+
options.memoize !== false, options.spillToStorage !== false, options.spillToStorage === true, options.spillPageRows, options.signal);
|
|
2662
|
+
throwIfAborted(options.signal);
|
|
2608
2663
|
outputNeedsExternalization = queryResultNeedsExternalization(resolvedPlan, typedSchemas);
|
|
2609
2664
|
outputColumnDomains = inferResultColumnDomains(resolvedPlan, typedSchemas);
|
|
2610
2665
|
};
|
|
2611
2666
|
if (options.version !== undefined) {
|
|
2612
2667
|
// Explicit time travel keeps the per-call lease and version-anchored reads.
|
|
2613
2668
|
const realTables = await this.#findRealBlockTables(plan);
|
|
2669
|
+
throwIfAborted(options.signal);
|
|
2614
2670
|
await this.#withLeasedSnapshot(options.version, async (snapshot) => {
|
|
2615
2671
|
const visibility = await this.#blockSegmentVisibility(realTables);
|
|
2672
|
+
throwIfAborted(options.signal);
|
|
2616
2673
|
await prepareAtSnapshot(snapshot, realTables, visibility);
|
|
2617
2674
|
});
|
|
2618
2675
|
}
|
|
2619
2676
|
else {
|
|
2620
2677
|
await this.#withSharedCatalogSnapshot(collectRealTableNames(plan), prepareAtSnapshot, probe);
|
|
2621
2678
|
}
|
|
2679
|
+
throwIfAborted(options.signal);
|
|
2622
2680
|
return createPreparedColumnarQuery(chooseJoinOrder(resolvedPlan, columnarTables), columnarTables, memory, {
|
|
2623
2681
|
...(ftsStats === undefined ? {} : { ftsStats }),
|
|
2624
2682
|
outputNeedsExternalization,
|
|
@@ -2909,18 +2967,21 @@ export class MinnowDatabase {
|
|
|
2909
2967
|
* materialize their referenced columns. Each block resolves its own inputs, so one real table
|
|
2910
2968
|
* projected differently by two blocks never collides.
|
|
2911
2969
|
*/
|
|
2912
|
-
async #prepareBlockInputs(block, snapshot, visibility, memory, realTables, typedSchemas, extraInputs, cacheResults = true, allowSpill = true, forceSpill = false, spillPageRows) {
|
|
2970
|
+
async #prepareBlockInputs(block, snapshot, visibility, memory, realTables, typedSchemas, extraInputs, cacheResults = true, allowSpill = true, forceSpill = false, spillPageRows, signal) {
|
|
2971
|
+
throwIfAborted(signal);
|
|
2913
2972
|
const inputs = new Map(extraInputs ?? []);
|
|
2914
2973
|
const sources = [block.base, ...block.joins];
|
|
2915
2974
|
for (const source of sources) {
|
|
2975
|
+
throwIfAborted(signal);
|
|
2916
2976
|
if (inputs.has(source.table))
|
|
2917
2977
|
continue;
|
|
2918
2978
|
if (source.recursive !== undefined) {
|
|
2919
2979
|
const { reference, base, step, all } = source.recursive;
|
|
2920
|
-
const { result: baseResult, schema } = await this.#executeBlockWithSchemaCached(base, snapshot, visibility, memory, realTables, typedSchemas, cacheResults, allowSpill, forceSpill, spillPageRows);
|
|
2980
|
+
const { result: baseResult, schema } = await this.#executeBlockWithSchemaCached(base, snapshot, visibility, memory, realTables, typedSchemas, cacheResults, allowSpill, forceSpill, spillPageRows, signal);
|
|
2921
2981
|
typedSchemas.set(reference, schema);
|
|
2922
2982
|
const state = createRecursiveCteState(baseResult, all);
|
|
2923
2983
|
while (state.frontier.length > 0) {
|
|
2984
|
+
throwIfAborted(signal);
|
|
2924
2985
|
// Each iteration's inputs release with its own context, so only the accumulated rows
|
|
2925
2986
|
// and the final columnar table hold memory across the fixpoint loop.
|
|
2926
2987
|
const iterationMemory = memory.createChild();
|
|
@@ -2930,7 +2991,7 @@ export class MinnowDatabase {
|
|
|
2930
2991
|
columnDomains: schema.map(({ sqlDomain }) => sqlDomain ?? null),
|
|
2931
2992
|
rows: state.frontier,
|
|
2932
2993
|
}, schema);
|
|
2933
|
-
state.absorb(await this.#executeBlock(step, snapshot, visibility, iterationMemory, realTables, typedSchemas, new Map([[reference, frontierTable]]), cacheResults, allowSpill, forceSpill, spillPageRows));
|
|
2994
|
+
state.absorb(await this.#executeBlock(step, snapshot, visibility, iterationMemory, realTables, typedSchemas, new Map([[reference, frontierTable]]), cacheResults, allowSpill, forceSpill, spillPageRows, signal));
|
|
2934
2995
|
}
|
|
2935
2996
|
finally {
|
|
2936
2997
|
iterationMemory.close();
|
|
@@ -2948,7 +3009,8 @@ export class MinnowDatabase {
|
|
|
2948
3009
|
const results = [];
|
|
2949
3010
|
let schema;
|
|
2950
3011
|
for (const member of source.union.blocks) {
|
|
2951
|
-
|
|
3012
|
+
throwIfAborted(signal);
|
|
3013
|
+
const { result, schema: memberSchema } = await this.#executeBlockWithSchemaCached(member, snapshot, visibility, memory, realTables, typedSchemas, cacheResults, allowSpill, forceSpill, spillPageRows, signal);
|
|
2952
3014
|
if (schema === undefined)
|
|
2953
3015
|
schema = memberSchema;
|
|
2954
3016
|
else {
|
|
@@ -2977,6 +3039,7 @@ export class MinnowDatabase {
|
|
|
2977
3039
|
// key plus the window spec, so a warm repeat skips the window pass and the
|
|
2978
3040
|
// rows-to-columnar conversion entirely.
|
|
2979
3041
|
const innerKey = await this.#blockResultCacheKey(source.windowed.block, snapshot, visibility, realTables, cacheResults);
|
|
3042
|
+
throwIfAborted(signal);
|
|
2980
3043
|
const columnarKey = innerKey === undefined
|
|
2981
3044
|
? undefined
|
|
2982
3045
|
: `ctw|${JSON.stringify(source.windowed.windows)}|${innerKey}`;
|
|
@@ -2988,7 +3051,7 @@ export class MinnowDatabase {
|
|
|
2988
3051
|
continue;
|
|
2989
3052
|
}
|
|
2990
3053
|
}
|
|
2991
|
-
const { result: inner, schema: innerSchema } = await this.#executeBlockWithSchemaCached(source.windowed.block, snapshot, visibility, memory, realTables, typedSchemas, cacheResults, allowSpill, forceSpill, spillPageRows);
|
|
3054
|
+
const { result: inner, schema: innerSchema } = await this.#executeBlockWithSchemaCached(source.windowed.block, snapshot, visibility, memory, realTables, typedSchemas, cacheResults, allowSpill, forceSpill, spillPageRows, signal);
|
|
2992
3055
|
const windowed = applyWindowFunctions(inner, source.windowed.windows, {
|
|
2993
3056
|
copyRows: cacheResults,
|
|
2994
3057
|
});
|
|
@@ -3015,6 +3078,7 @@ export class MinnowDatabase {
|
|
|
3015
3078
|
if (derived === undefined)
|
|
3016
3079
|
continue;
|
|
3017
3080
|
const derivedKey = await this.#blockResultCacheKey(derived, snapshot, visibility, realTables, cacheResults);
|
|
3081
|
+
throwIfAborted(signal);
|
|
3018
3082
|
const columnarKey = derivedKey === undefined ? undefined : `ctd|${derivedKey}`;
|
|
3019
3083
|
if (columnarKey !== undefined) {
|
|
3020
3084
|
const hit = this.#cacheGet(columnarKey);
|
|
@@ -3024,7 +3088,7 @@ export class MinnowDatabase {
|
|
|
3024
3088
|
continue;
|
|
3025
3089
|
}
|
|
3026
3090
|
}
|
|
3027
|
-
const { result, schema } = await this.#executeBlockWithSchemaCached(derived, snapshot, visibility, memory, realTables, typedSchemas, cacheResults, allowSpill, forceSpill, spillPageRows);
|
|
3091
|
+
const { result, schema } = await this.#executeBlockWithSchemaCached(derived, snapshot, visibility, memory, realTables, typedSchemas, cacheResults, allowSpill, forceSpill, spillPageRows, signal);
|
|
3028
3092
|
typedSchemas.set(source.table, schema);
|
|
3029
3093
|
const derivedTable = derivedColumnarTable(source.table, result, schema);
|
|
3030
3094
|
if (columnarKey !== undefined) {
|
|
@@ -3039,6 +3103,7 @@ export class MinnowDatabase {
|
|
|
3039
3103
|
const columns = referencedColumns(block, nameSchemas);
|
|
3040
3104
|
const onlyRealSource = sources.length === 1 && sources[0]?.derived === undefined ? block : undefined;
|
|
3041
3105
|
for (const source of sources) {
|
|
3106
|
+
throwIfAborted(signal);
|
|
3042
3107
|
if (source.derived !== undefined || inputs.has(source.table))
|
|
3043
3108
|
continue;
|
|
3044
3109
|
if (source.table === DUAL_TABLE) {
|
|
@@ -3050,6 +3115,7 @@ export class MinnowDatabase {
|
|
|
3050
3115
|
throw new UnknownTableError(source.table);
|
|
3051
3116
|
const requestedColumns = columns.get(table.name) ?? [];
|
|
3052
3117
|
inputs.set(table.name, await this.#materializeColumnarTableAtSnapshot(table, snapshot, requestedColumns.length === 0 ? [] : resolveReadColumns(table, requestedColumns), visibility, onlyRealSource));
|
|
3118
|
+
throwIfAborted(signal);
|
|
3053
3119
|
}
|
|
3054
3120
|
return inputs;
|
|
3055
3121
|
}
|
|
@@ -3066,6 +3132,7 @@ export class MinnowDatabase {
|
|
|
3066
3132
|
return this.#queryWithReadReservation(sql, options);
|
|
3067
3133
|
}
|
|
3068
3134
|
async #queryUnreserved(sql, options) {
|
|
3135
|
+
throwIfAborted(options.signal);
|
|
3069
3136
|
const open = this.#openTransaction;
|
|
3070
3137
|
if (open !== undefined) {
|
|
3071
3138
|
// Read-your-writes: inside BEGIN … COMMIT a read sees the pre-scope snapshot plus what
|
|
@@ -3091,6 +3158,7 @@ export class MinnowDatabase {
|
|
|
3091
3158
|
const result = !memoizable
|
|
3092
3159
|
? await this.#queryCompiled(plan, options)
|
|
3093
3160
|
: await this.#memoizedQuery(plan, `res ${queryResultMemoKey(sql, options.params ?? [])}`, options, probe);
|
|
3161
|
+
throwIfAborted(options.signal);
|
|
3094
3162
|
return externalizeQueryResult(result);
|
|
3095
3163
|
}
|
|
3096
3164
|
/**
|
|
@@ -3155,7 +3223,8 @@ export class MinnowDatabase {
|
|
|
3155
3223
|
};
|
|
3156
3224
|
}
|
|
3157
3225
|
async #queryBatches(sql, options, cursor) {
|
|
3158
|
-
options = this.#effectiveQueryOptions(options);
|
|
3226
|
+
options = this.#effectiveQueryOptions({ ...options, signal: cursor.signal });
|
|
3227
|
+
cursor.signal.throwIfAborted();
|
|
3159
3228
|
// A transaction's staged overlay has its own executor. Keep its read-your-writes semantics
|
|
3160
3229
|
// and page the materialized result; native scan batching is for durable snapshots.
|
|
3161
3230
|
if (this.#openTransaction === undefined) {
|
|
@@ -3196,16 +3265,22 @@ export class MinnowDatabase {
|
|
|
3196
3265
|
* and on IndexedDB every probe is a read transaction, a floor under every small query.
|
|
3197
3266
|
*/
|
|
3198
3267
|
async #memoizedQuery(plan, key, options, probe) {
|
|
3268
|
+
throwIfAborted(options.signal);
|
|
3199
3269
|
const before = await probe();
|
|
3270
|
+
throwIfAborted(options.signal);
|
|
3200
3271
|
const cached = this.#cacheGet(`${key}\u0001${String(before.catalogEpoch)}`);
|
|
3201
|
-
if (cached !== undefined)
|
|
3272
|
+
if (cached !== undefined) {
|
|
3273
|
+
options.onStats?.({ peakMemoryBytes: 0 });
|
|
3202
3274
|
return copyQueryResult(cached);
|
|
3275
|
+
}
|
|
3203
3276
|
const result = await this.#queryCompiled(plan, options, before);
|
|
3277
|
+
throwIfAborted(options.signal);
|
|
3204
3278
|
const bytes = queryResultRetainedBytes(result);
|
|
3205
3279
|
if (bytes <= RESULT_MEMO_MAX_BYTES) {
|
|
3206
3280
|
// Cache only when the epoch did not move during execution: the result is then exactly
|
|
3207
3281
|
// that epoch's answer. A moved epoch simply skips the cache — never mis-files.
|
|
3208
3282
|
const after = await probe();
|
|
3283
|
+
throwIfAborted(options.signal);
|
|
3209
3284
|
if (after.catalogEpoch === before.catalogEpoch) {
|
|
3210
3285
|
this.#cachePut(`${key}\u0001${String(before.catalogEpoch)}`, copyQueryResult(result), bytes);
|
|
3211
3286
|
}
|
|
@@ -3225,7 +3300,7 @@ export class MinnowDatabase {
|
|
|
3225
3300
|
// read has to ask the catalog. It asks by epoch — an O(1) probe the store already serves for
|
|
3226
3301
|
// result memoization — and only re-reads the view set when the catalog has actually moved.
|
|
3227
3302
|
// A database with no views therefore pays one probe, not a catalog scan per query.
|
|
3228
|
-
const { views, domains } = await this.#catalogFacts(probe);
|
|
3303
|
+
const { views, domains, columns: catalogColumns } = await this.#catalogFacts(probe);
|
|
3229
3304
|
let rewritten = plan.usesSequenceCalls === true ? await this.#resolveSequenceCalls(plan) : plan;
|
|
3230
3305
|
if (views.size > 0 && planReadsViews(plan, (name) => views.has(name))) {
|
|
3231
3306
|
const bodies = new Map();
|
|
@@ -3244,6 +3319,9 @@ export class MinnowDatabase {
|
|
|
3244
3319
|
if (domains.size > 0 && planReadsTable(rewritten, (name) => domains.has(name))) {
|
|
3245
3320
|
rewritten = normalizePlanDomainLiterals(rewritten, domains);
|
|
3246
3321
|
}
|
|
3322
|
+
const qualified = qualifyCorrelatedReferences(rewritten, catalogColumns);
|
|
3323
|
+
if (qualified !== rewritten)
|
|
3324
|
+
rewritten = optimizePlan(qualified);
|
|
3247
3325
|
if (!aliased && !natural)
|
|
3248
3326
|
return rewritten;
|
|
3249
3327
|
// These two need column *order*, which only the records carry; both are rare enough that
|
|
@@ -3356,7 +3434,9 @@ export class MinnowDatabase {
|
|
|
3356
3434
|
const views = new Map();
|
|
3357
3435
|
const childKeys = new Map();
|
|
3358
3436
|
const domains = new Map();
|
|
3437
|
+
const columns = new Map();
|
|
3359
3438
|
for (const table of await this.store.listTables()) {
|
|
3439
|
+
columns.set(table.name, table.columns.filter(({ hidden }) => hidden !== true).map(({ name }) => name));
|
|
3360
3440
|
const tableDomains = new Map(table.columns.flatMap((column) => column.sqlDomain === undefined ? [] : [[column.name, column.sqlDomain]]));
|
|
3361
3441
|
if (tableDomains.size > 0)
|
|
3362
3442
|
domains.set(table.name, tableDomains);
|
|
@@ -3370,7 +3450,7 @@ export class MinnowDatabase {
|
|
|
3370
3450
|
existing.push({ table, key });
|
|
3371
3451
|
}
|
|
3372
3452
|
}
|
|
3373
|
-
const facts = { views, childKeys, domains };
|
|
3453
|
+
const facts = { views, childKeys, domains, columns };
|
|
3374
3454
|
this.#catalogCache = { epoch, facts };
|
|
3375
3455
|
return facts;
|
|
3376
3456
|
}
|
|
@@ -3385,10 +3465,13 @@ export class MinnowDatabase {
|
|
|
3385
3465
|
*/
|
|
3386
3466
|
async #queryCompiled(plan, options = {}, probe) {
|
|
3387
3467
|
options = this.#effectiveQueryOptions(options);
|
|
3468
|
+
throwIfAborted(options.signal);
|
|
3388
3469
|
// One freshness probe per query: read here unless the caller already has one, and handed
|
|
3389
3470
|
// to the view lookup and the catalog state below, which would otherwise probe again each.
|
|
3390
3471
|
probe ??= await this.store.getCatalogProbe();
|
|
3472
|
+
throwIfAborted(options.signal);
|
|
3391
3473
|
plan = await this.#applyCatalogRewrites(plan, probe);
|
|
3474
|
+
throwIfAborted(options.signal);
|
|
3392
3475
|
const spillPageRows = options.spillPageRows === undefined
|
|
3393
3476
|
? undefined
|
|
3394
3477
|
: positiveWholeNumber(options.spillPageRows, "Query spill page rows");
|
|
@@ -3409,6 +3492,7 @@ export class MinnowDatabase {
|
|
|
3409
3492
|
}
|
|
3410
3493
|
}
|
|
3411
3494
|
const prepared = await this.#prepareCompiledPlan(plan, options, probe);
|
|
3495
|
+
throwIfAborted(options.signal);
|
|
3412
3496
|
// Read the peak before close(): closing releases the context and zeroes what it tracked.
|
|
3413
3497
|
const report = (result) => {
|
|
3414
3498
|
options.onStats?.({ peakMemoryBytes: prepared.memoryUsage.peakBytes });
|
|
@@ -3416,11 +3500,16 @@ export class MinnowDatabase {
|
|
|
3416
3500
|
};
|
|
3417
3501
|
try {
|
|
3418
3502
|
const spill = options.spillToStorage ?? options.executionMemoryBudgetBytes !== undefined;
|
|
3419
|
-
if (!spill)
|
|
3420
|
-
|
|
3503
|
+
if (!spill) {
|
|
3504
|
+
const result = prepared.execute();
|
|
3505
|
+
throwIfAborted(options.signal);
|
|
3506
|
+
return report(result);
|
|
3507
|
+
}
|
|
3421
3508
|
if (options.spillToStorage !== true) {
|
|
3422
3509
|
try {
|
|
3423
|
-
|
|
3510
|
+
const result = prepared.execute();
|
|
3511
|
+
throwIfAborted(options.signal);
|
|
3512
|
+
return report(result);
|
|
3424
3513
|
}
|
|
3425
3514
|
catch (error) {
|
|
3426
3515
|
if (!(error instanceof QueryMemoryBudgetError))
|
|
@@ -3430,12 +3519,35 @@ export class MinnowDatabase {
|
|
|
3430
3519
|
return report(await prepared.executeAsync({
|
|
3431
3520
|
...(spillPageRows === undefined ? {} : { spillPageRows }),
|
|
3432
3521
|
spillStore: this.#leasedSpillStore(),
|
|
3522
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
3433
3523
|
}));
|
|
3434
3524
|
}
|
|
3435
3525
|
finally {
|
|
3436
3526
|
prepared.close();
|
|
3437
3527
|
}
|
|
3438
3528
|
}
|
|
3529
|
+
/**
|
|
3530
|
+
* Streams a one-column snapshot projection into bounded pages. DELETE uses this before its
|
|
3531
|
+
* first staged write: selection keeps all ordinary pruning/predicate semantics without
|
|
3532
|
+
* retaining every QueryRow object until the key array is complete. Undefined means the shape
|
|
3533
|
+
* or physical history needs the established row-shaped executor.
|
|
3534
|
+
*/
|
|
3535
|
+
async #queryCompiledFirstColumn(plan, probe) {
|
|
3536
|
+
const options = this.#effectiveQueryOptions({ memoize: false });
|
|
3537
|
+
plan = await this.#applyCatalogRewrites(plan, probe);
|
|
3538
|
+
if (!this.#canStreamPlanShape(plan, options))
|
|
3539
|
+
return undefined;
|
|
3540
|
+
const values = [];
|
|
3541
|
+
const streamed = await this.#queryStreamed(plan, options, undefined, probe, {
|
|
3542
|
+
batchRows: 2_048,
|
|
3543
|
+
signal: new AbortController().signal,
|
|
3544
|
+
consumeFirstColumn: async (batch) => {
|
|
3545
|
+
values.push(...batch);
|
|
3546
|
+
},
|
|
3547
|
+
consume: async () => undefined,
|
|
3548
|
+
});
|
|
3549
|
+
return streamed === undefined ? undefined : values;
|
|
3550
|
+
}
|
|
3439
3551
|
/**
|
|
3440
3552
|
* Backs the executor's spill pages with durable owner leases: each owner registers a lease
|
|
3441
3553
|
* before its first page write and renews it while pages are read or written, so an abandoned
|
|
@@ -3823,15 +3935,18 @@ export class MinnowDatabase {
|
|
|
3823
3935
|
* one, so a table that cannot be compacted today costs one attempt, not one per query.
|
|
3824
3936
|
*/
|
|
3825
3937
|
#maybeScheduleAutoCompaction(table, segments) {
|
|
3938
|
+
this.#maybeScheduleAutoCompactionFromHint(table, autoCompactionHint(null, segments));
|
|
3939
|
+
}
|
|
3940
|
+
#maybeScheduleAutoCompactionFromHint(table, hint) {
|
|
3826
3941
|
if (this.#closed)
|
|
3827
3942
|
return;
|
|
3828
3943
|
if (!this.#autoCompact)
|
|
3829
3944
|
return;
|
|
3830
3945
|
if (this.#droppingTables.has(table.id))
|
|
3831
3946
|
return;
|
|
3832
|
-
if (!
|
|
3947
|
+
if (!autoCompactionDueHint(hint))
|
|
3833
3948
|
return;
|
|
3834
|
-
if (
|
|
3949
|
+
if (hint.visible < (this.#autoCompactionBackoff.get(table.id) ?? 0))
|
|
3835
3950
|
return;
|
|
3836
3951
|
if (this.#autoCompactionsInFlight.has(table.id)) {
|
|
3837
3952
|
// A final burst can cross the threshold while the prior fold is still planning or
|
|
@@ -3845,14 +3960,14 @@ export class MinnowDatabase {
|
|
|
3845
3960
|
if (folded)
|
|
3846
3961
|
this.#autoCompactionBackoff.delete(table.id);
|
|
3847
3962
|
else {
|
|
3848
|
-
this.#autoCompactionBackoff.set(table.id, Math.min(AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS, Math.max(2,
|
|
3963
|
+
this.#autoCompactionBackoff.set(table.id, Math.min(AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS, Math.max(2, hint.visible * 2)));
|
|
3849
3964
|
}
|
|
3850
3965
|
})
|
|
3851
3966
|
.catch(() => {
|
|
3852
3967
|
// Back off deterministic failures, but never beyond the maximum L0 prefix a fold can
|
|
3853
3968
|
// consume. A transient conflict near the end of a burst must not strand hundreds of
|
|
3854
3969
|
// segments waiting for a segment count the idle database can never reach.
|
|
3855
|
-
this.#autoCompactionBackoff.set(table.id, Math.min(AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS, Math.max(2,
|
|
3970
|
+
this.#autoCompactionBackoff.set(table.id, Math.min(AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS, Math.max(2, hint.visible * 2)));
|
|
3856
3971
|
})
|
|
3857
3972
|
.finally(() => {
|
|
3858
3973
|
if (this.#autoCompactionsInFlight.get(table.id) === run) {
|
|
@@ -3872,6 +3987,10 @@ export class MinnowDatabase {
|
|
|
3872
3987
|
// every write and let the commit return the identical CompactionBacklogError at the CAS.
|
|
3873
3988
|
if (!this.#autoCompact)
|
|
3874
3989
|
return;
|
|
3990
|
+
const hintVersion = transaction?.snapshotVersion ?? (await this.store.getCurrentManifestVersion());
|
|
3991
|
+
const hint = this.#autoCompactionHints.get(table.id);
|
|
3992
|
+
if (hint?.version === hintVersion && hint.levelZero < MAX_LEVEL_ZERO_SEGMENTS)
|
|
3993
|
+
return;
|
|
3875
3994
|
let segments = await this.#currentVisibleSegments(table);
|
|
3876
3995
|
let levelZero = 0;
|
|
3877
3996
|
for (const segment of segments) {
|
|
@@ -3945,9 +4064,11 @@ export class MinnowDatabase {
|
|
|
3945
4064
|
const version = await this.store.getCurrentManifestVersion();
|
|
3946
4065
|
segments = await this.#visibleSegmentRecords(table, new Snapshot(this.store, version));
|
|
3947
4066
|
if ((await this.store.getCurrentManifestVersion()) === version) {
|
|
4067
|
+
this.#autoCompactionHints.set(table.id, autoCompactionHint(version, segments));
|
|
3948
4068
|
return segments;
|
|
3949
4069
|
}
|
|
3950
4070
|
}
|
|
4071
|
+
this.#autoCompactionHints.delete(table.id);
|
|
3951
4072
|
return segments;
|
|
3952
4073
|
}
|
|
3953
4074
|
/**
|
|
@@ -3958,6 +4079,7 @@ export class MinnowDatabase {
|
|
|
3958
4079
|
if (this.#closed)
|
|
3959
4080
|
return;
|
|
3960
4081
|
const contribution = transaction?.committedCatalogContribution;
|
|
4082
|
+
this.#advanceAutoCompactionHints(manifest, contribution?.segments);
|
|
3961
4083
|
const initial = contribution?.initialCatalogProbe;
|
|
3962
4084
|
const contributedTableIds = new Set(contribution?.segments.map((segment) => segment.tableId) ?? []);
|
|
3963
4085
|
if (contribution !== undefined &&
|
|
@@ -4051,7 +4173,7 @@ export class MinnowDatabase {
|
|
|
4051
4173
|
this.#armIdleCompactionCheck();
|
|
4052
4174
|
}
|
|
4053
4175
|
/** Counts one physical rewrite version once in this engine, including lost-ack recovery. */
|
|
4054
|
-
#afterCompactionCommit(manifest) {
|
|
4176
|
+
#afterCompactionCommit(manifest, tableId) {
|
|
4055
4177
|
if (this.#accountedCompactionVersions.has(manifest.version))
|
|
4056
4178
|
return;
|
|
4057
4179
|
this.#accountedCompactionVersions.add(manifest.version);
|
|
@@ -4060,8 +4182,49 @@ export class MinnowDatabase {
|
|
|
4060
4182
|
if (oldest !== undefined)
|
|
4061
4183
|
this.#accountedCompactionVersions.delete(oldest);
|
|
4062
4184
|
}
|
|
4185
|
+
// A fold replaces source segments with a different physical layout. Its manifest is
|
|
4186
|
+
// logically unchanged and therefore carries no changed-table ID, so invalidate explicitly;
|
|
4187
|
+
// #runAutoCompaction's stable post-fold probe seeds the replacement counters.
|
|
4188
|
+
this.#autoCompactionHints.delete(tableId);
|
|
4063
4189
|
this.#afterCommit(manifest);
|
|
4064
4190
|
}
|
|
4191
|
+
/** Advances exact local hints, or discards them when a commit's provenance is incomplete. */
|
|
4192
|
+
#advanceAutoCompactionHints(manifest, contributedSegments) {
|
|
4193
|
+
for (const [tableId, hint] of this.#autoCompactionHints) {
|
|
4194
|
+
if (hint.version !== manifest.previousVersion) {
|
|
4195
|
+
// Another database committed since this instance last proved the layout.
|
|
4196
|
+
this.#autoCompactionHints.delete(tableId);
|
|
4197
|
+
continue;
|
|
4198
|
+
}
|
|
4199
|
+
this.#autoCompactionHints.set(tableId, { ...hint, version: manifest.version });
|
|
4200
|
+
}
|
|
4201
|
+
const additionsByTable = new Map();
|
|
4202
|
+
for (const segment of contributedSegments ?? []) {
|
|
4203
|
+
const additions = additionsByTable.get(segment.tableId);
|
|
4204
|
+
if (additions === undefined)
|
|
4205
|
+
additionsByTable.set(segment.tableId, [segment]);
|
|
4206
|
+
else
|
|
4207
|
+
additions.push(segment);
|
|
4208
|
+
}
|
|
4209
|
+
for (const tableId of manifest.changedTableIds) {
|
|
4210
|
+
const hint = this.#autoCompactionHints.get(tableId);
|
|
4211
|
+
if (hint === undefined)
|
|
4212
|
+
continue;
|
|
4213
|
+
const additions = additionsByTable.get(tableId);
|
|
4214
|
+
if (additions === undefined) {
|
|
4215
|
+
// DDL and future write paths without an exact segment contribution fail closed.
|
|
4216
|
+
this.#autoCompactionHints.delete(tableId);
|
|
4217
|
+
continue;
|
|
4218
|
+
}
|
|
4219
|
+
const contribution = autoCompactionHint(manifest.version, additions);
|
|
4220
|
+
this.#autoCompactionHints.set(tableId, {
|
|
4221
|
+
version: manifest.version,
|
|
4222
|
+
visible: hint.visible + contribution.visible,
|
|
4223
|
+
levelZero: hint.levelZero + contribution.levelZero,
|
|
4224
|
+
deltas: hint.deltas + contribution.deltas,
|
|
4225
|
+
});
|
|
4226
|
+
}
|
|
4227
|
+
}
|
|
4065
4228
|
/**
|
|
4066
4229
|
* Debounces the final write-path check for a burst. Sampling every few commits keeps the hot
|
|
4067
4230
|
* path cheap, but the last one through seven commits can be the ones that cross a fold
|
|
@@ -4099,6 +4262,12 @@ export class MinnowDatabase {
|
|
|
4099
4262
|
const table = await this.store.getTable(tableId);
|
|
4100
4263
|
if (table === undefined)
|
|
4101
4264
|
return;
|
|
4265
|
+
const version = await this.store.getCurrentManifestVersion();
|
|
4266
|
+
const hint = this.#autoCompactionHints.get(tableId);
|
|
4267
|
+
if (hint?.version === version) {
|
|
4268
|
+
this.#maybeScheduleAutoCompactionFromHint(table, hint);
|
|
4269
|
+
return;
|
|
4270
|
+
}
|
|
4102
4271
|
this.#maybeScheduleAutoCompaction(table, await this.#currentVisibleSegments(table));
|
|
4103
4272
|
}
|
|
4104
4273
|
catch {
|
|
@@ -4379,6 +4548,7 @@ export class MinnowDatabase {
|
|
|
4379
4548
|
const statementNow = this.#now();
|
|
4380
4549
|
const filled = await fillColumnDefaults(target, input, (sql) => this.#evaluateDefaultExpression(sql, statementNow), derivedRows.length);
|
|
4381
4550
|
normalizeDomainBatch(target, filled.batch);
|
|
4551
|
+
fillStoredGeneratedColumns(target, filled.batch, derivedRows.length, filled.generated);
|
|
4382
4552
|
fillCompositePrimaryKey(target, filled.batch, derivedRows.length);
|
|
4383
4553
|
validateBatch(target, filled.batch, filled.autoIncrement?.column.name);
|
|
4384
4554
|
if (filled.autoIncrement !== undefined && filled.autoIncrement.missingIndexes.length > 0) {
|
|
@@ -4651,6 +4821,15 @@ export class MinnowDatabase {
|
|
|
4651
4821
|
? this.#queryCompiled(plan, {}, probe)
|
|
4652
4822
|
: this.#sessionQueryPlan(transaction, await this.#applyCatalogRewrites(plan));
|
|
4653
4823
|
}),
|
|
4824
|
+
queryFirstColumn: (plan) => this.#withReadReservation(async () => {
|
|
4825
|
+
// A staged overlay can add, replace, or remove keys without changing the pinned
|
|
4826
|
+
// manifest. Keep that case on the read-your-writes row executor; before the first
|
|
4827
|
+
// stage, the transaction's exact catalog probe makes the streamed snapshot safe.
|
|
4828
|
+
const probe = transaction.initialCatalogProbe;
|
|
4829
|
+
return transaction.stagedWorkCount === 0 && probe !== undefined
|
|
4830
|
+
? this.#queryCompiledFirstColumn(plan, probe)
|
|
4831
|
+
: undefined;
|
|
4832
|
+
}),
|
|
4654
4833
|
executeStatement: (statement) => {
|
|
4655
4834
|
open();
|
|
4656
4835
|
return guarded(() => this.runStatement(statement, { writer }));
|
|
@@ -4719,21 +4898,26 @@ export class MinnowDatabase {
|
|
|
4719
4898
|
*/
|
|
4720
4899
|
async #sessionQuery(transaction, sql, options = {}) {
|
|
4721
4900
|
options = this.#effectiveQueryOptions(options);
|
|
4901
|
+
throwIfAborted(options.signal);
|
|
4722
4902
|
const bound = bindPlanParameters(this.#compileCached(sql), options.params);
|
|
4723
4903
|
// The same catalog rewrites a read outside a scope gets: a scope that could not see a view
|
|
4724
4904
|
// would make the scope's reads a different language from everyone else's.
|
|
4725
4905
|
const plan = await this.#applyCatalogRewrites(bound);
|
|
4906
|
+
throwIfAborted(options.signal);
|
|
4726
4907
|
return this.#sessionQueryPlan(transaction, plan, options);
|
|
4727
4908
|
}
|
|
4728
4909
|
/** #sessionQuery for an already-bound plan: trigger bodies resolve their reads through it. */
|
|
4729
4910
|
async #sessionQueryPlan(transaction, plan, options = {}) {
|
|
4730
4911
|
options = this.#effectiveQueryOptions(options);
|
|
4912
|
+
throwIfAborted(options.signal);
|
|
4731
4913
|
const names = collectRealTableNames(plan);
|
|
4732
4914
|
const tables = await Promise.all(names.map((name) => this.#findTable(name)));
|
|
4915
|
+
throwIfAborted(options.signal);
|
|
4733
4916
|
const realTables = new Map(tables.map((table) => [table.name, table]));
|
|
4734
4917
|
const pendingIds = new Set(transaction.pendingSegmentIds);
|
|
4735
4918
|
const pendingBlocks = new Set(transaction.pendingBlockIds);
|
|
4736
4919
|
const ourRecord = await this.store.getTransaction(transaction.id);
|
|
4920
|
+
throwIfAborted(options.signal);
|
|
4737
4921
|
// Pinned to the scope's own snapshot, not the current manifest: the documented contract
|
|
4738
4922
|
// is "the pre-scope snapshot plus everything this scope staged", and #assertKeysExist
|
|
4739
4923
|
// already validates against transaction.snapshotVersion — reading the current version
|
|
@@ -4757,6 +4941,7 @@ export class MinnowDatabase {
|
|
|
4757
4941
|
getBlocks: async (blockIds) => {
|
|
4758
4942
|
const blocks = [];
|
|
4759
4943
|
for (let start = 0; start < blockIds.length; start += 64) {
|
|
4944
|
+
throwIfAborted(options.signal);
|
|
4760
4945
|
const window = blockIds.slice(start, start + 64);
|
|
4761
4946
|
const committedIds = [];
|
|
4762
4947
|
const committedIndexes = [];
|
|
@@ -4781,6 +4966,7 @@ export class MinnowDatabase {
|
|
|
4781
4966
|
this.#getBlocksWindowed(committedIds),
|
|
4782
4967
|
Promise.all(stagedIds.map((blockId) => transaction.getBlock(blockId))),
|
|
4783
4968
|
]);
|
|
4969
|
+
throwIfAborted(options.signal);
|
|
4784
4970
|
const ordered = new Array(window.length);
|
|
4785
4971
|
committedIndexes.forEach((index, offset) => {
|
|
4786
4972
|
ordered[index] = committed[offset];
|
|
@@ -4799,10 +4985,12 @@ export class MinnowDatabase {
|
|
|
4799
4985
|
const transactionRecords = new Map();
|
|
4800
4986
|
const deferredSegments = transaction.deferredSegments;
|
|
4801
4987
|
for (const table of tables) {
|
|
4988
|
+
throwIfAborted(options.signal);
|
|
4802
4989
|
const segments = [
|
|
4803
4990
|
...(await listTableSegmentsPaged(this.store, table.id)),
|
|
4804
4991
|
...deferredSegments.filter((segment) => segment.tableId === table.id),
|
|
4805
4992
|
];
|
|
4993
|
+
throwIfAborted(options.signal);
|
|
4806
4994
|
// Staged segments sort after every committed one; commitOrdinal keeps their
|
|
4807
4995
|
// staging order (the journal's id list is sorted, so it cannot).
|
|
4808
4996
|
const doctored = segments.map((segment) => pendingIds.has(segment.id)
|
|
@@ -4812,6 +5000,7 @@ export class MinnowDatabase {
|
|
|
4812
5000
|
for (const record of await this.#transactionRecordsForSegments(doctored)) {
|
|
4813
5001
|
transactionRecords.set(record.id, record);
|
|
4814
5002
|
}
|
|
5003
|
+
throwIfAborted(options.signal);
|
|
4815
5004
|
}
|
|
4816
5005
|
if (ourRecord !== undefined) {
|
|
4817
5006
|
transactionRecords.set(transaction.id, {
|
|
@@ -4831,6 +5020,7 @@ export class MinnowDatabase {
|
|
|
4831
5020
|
/** Executes one plan at an explicit snapshot and visibility, covering every plan shape. */
|
|
4832
5021
|
async #queryAtVisibility(plan, snapshot, visibility, realTables, cacheResults = true, options = {}) {
|
|
4833
5022
|
options = this.#effectiveQueryOptions(options);
|
|
5023
|
+
throwIfAborted(options.signal);
|
|
4834
5024
|
plan = expandFtsColumns(plan, (name) => searchableFtsColumns(realTables.get(name)));
|
|
4835
5025
|
const typedSchemas = new Map([...realTables.values()].map((table) => [
|
|
4836
5026
|
table.name,
|
|
@@ -4847,16 +5037,22 @@ export class MinnowDatabase {
|
|
|
4847
5037
|
const spillPageRows = options.spillPageRows === undefined
|
|
4848
5038
|
? undefined
|
|
4849
5039
|
: positiveWholeNumber(options.spillPageRows, "Query spill page rows");
|
|
5040
|
+
const report = (result) => {
|
|
5041
|
+
throwIfAborted(options.signal);
|
|
5042
|
+
options.onStats?.({ peakMemoryBytes: memory.usage.peakBytes });
|
|
5043
|
+
return result;
|
|
5044
|
+
};
|
|
4850
5045
|
try {
|
|
4851
5046
|
const resolution = subqueryResolutionSteps(plan);
|
|
4852
5047
|
for (const step of resolution.steps) {
|
|
4853
|
-
|
|
5048
|
+
throwIfAborted(options.signal);
|
|
5049
|
+
step.substitute(await this.#executeBlock(step.block, snapshot, visibility, memory, realTables, typedSchemas, undefined, cacheResults, allowSpill, forceSpill, spillPageRows, options.signal));
|
|
4854
5050
|
}
|
|
4855
5051
|
const wrapper = transparentProjectionSource(resolution.plan);
|
|
4856
5052
|
if (wrapper !== undefined) {
|
|
4857
|
-
return projectResultColumns(await this.#executeBlock(wrapper.inner, snapshot, visibility, memory, realTables, typedSchemas, undefined, cacheResults, allowSpill, forceSpill, spillPageRows), wrapper.aliases);
|
|
5053
|
+
return report(projectResultColumns(await this.#executeBlock(wrapper.inner, snapshot, visibility, memory, realTables, typedSchemas, undefined, cacheResults, allowSpill, forceSpill, spillPageRows, options.signal), wrapper.aliases));
|
|
4858
5054
|
}
|
|
4859
|
-
return await this.#executeBlock(resolution.plan, snapshot, visibility, memory, realTables, typedSchemas, undefined, cacheResults, allowSpill, forceSpill, spillPageRows);
|
|
5055
|
+
return report(await this.#executeBlock(resolution.plan, snapshot, visibility, memory, realTables, typedSchemas, undefined, cacheResults, allowSpill, forceSpill, spillPageRows, options.signal));
|
|
4860
5056
|
}
|
|
4861
5057
|
finally {
|
|
4862
5058
|
memory.close();
|
|
@@ -4941,6 +5137,7 @@ export class MinnowDatabase {
|
|
|
4941
5137
|
throw new TypeError(`Table needs a unique key before rows can be updated: ${table.name}`);
|
|
4942
5138
|
}
|
|
4943
5139
|
input = normalizeDomainUpdate(table, input);
|
|
5140
|
+
rejectGeneratedUpdateAssignments(table, input);
|
|
4944
5141
|
const keys = validateUpdateBatch(table, keyColumn, input);
|
|
4945
5142
|
// Read-your-writes membership: keys staged by this scope pass, keys the scope removed
|
|
4946
5143
|
// fail, and everything else checks against the committed snapshot as usual.
|
|
@@ -4955,12 +5152,17 @@ export class MinnowDatabase {
|
|
|
4955
5152
|
await this.#assertKeysExist(table, keyColumn, transaction.snapshotVersion, committedKeys);
|
|
4956
5153
|
}
|
|
4957
5154
|
const sessionChecks = table.checks ?? [];
|
|
4958
|
-
|
|
5155
|
+
let changedForeignKey = (table.foreignKeys ?? []).some((key) => key.enforced !== false &&
|
|
4959
5156
|
foreignKeyColumns(key).some((column) => input.changes[column] !== undefined));
|
|
4960
5157
|
const preImages = await this.#triggerPreImages(table, keyColumn, input.keys.filter((key) => key !== null), "update", (preImageSql, params) => this.#sessionQuery(transaction, preImageSql, { params }), sessionChecks.length > 0 ||
|
|
4961
5158
|
changedForeignKey ||
|
|
5159
|
+
tableGeneratedExpressions(table).length > 0 ||
|
|
4962
5160
|
secondaryIndexUpdateNeedsPreImages(table, input) ||
|
|
4963
5161
|
readyUniqueSecondaryIndexes(table).some(({ columns }) => columns.some((column) => input.changes[column.name] !== undefined)));
|
|
5162
|
+
input = applyStoredGeneratedUpdateChanges(table, input, preImages);
|
|
5163
|
+
validateUpdateBatch(table, keyColumn, input);
|
|
5164
|
+
changedForeignKey = (table.foreignKeys ?? []).some((key) => key.enforced !== false &&
|
|
5165
|
+
foreignKeyColumns(key).some((column) => input.changes[column] !== undefined));
|
|
4964
5166
|
stageSecondaryUniqueMutationChanges(transaction, table, input, preImages);
|
|
4965
5167
|
const secondaryDeltas = buildSecondaryUpdateDeltas(table, input, preImages);
|
|
4966
5168
|
if (secondaryDeltas.length > 0) {
|
|
@@ -5042,7 +5244,19 @@ export class MinnowDatabase {
|
|
|
5042
5244
|
},
|
|
5043
5245
|
]);
|
|
5044
5246
|
await this.#stageTriggerDerivedInserts(transaction, table, "update", input.keys.length, sessionUpdateValueAt, "after", cascadeBudget);
|
|
5045
|
-
return {
|
|
5247
|
+
return {
|
|
5248
|
+
tableName: table.name,
|
|
5249
|
+
segmentId,
|
|
5250
|
+
rowCount: input.keys.length,
|
|
5251
|
+
...(tableGeneratedExpressions(table).length === 0
|
|
5252
|
+
? {}
|
|
5253
|
+
: {
|
|
5254
|
+
generatedColumns: Object.fromEntries(tableGeneratedExpressions(table).map(({ column }) => [
|
|
5255
|
+
column.name,
|
|
5256
|
+
(input.changes[column.name] ?? []).map((value) => externalSqlDomainValue(value)),
|
|
5257
|
+
])),
|
|
5258
|
+
}),
|
|
5259
|
+
};
|
|
5046
5260
|
}
|
|
5047
5261
|
async #sessionDelete(transaction, tableName, input, cascadeBudget = 1, referentialBudget = REFERENTIAL_CASCADES) {
|
|
5048
5262
|
const table = await this.#findTable(tableName);
|
|
@@ -5379,6 +5593,9 @@ export class MinnowDatabase {
|
|
|
5379
5593
|
...(columnDefinition.defaultSpec === undefined
|
|
5380
5594
|
? {}
|
|
5381
5595
|
: { defaultValue: columnDefinition.defaultSpec }),
|
|
5596
|
+
...(columnDefinition.generatedSpec === undefined
|
|
5597
|
+
? {}
|
|
5598
|
+
: { generatedValue: columnDefinition.generatedSpec }),
|
|
5382
5599
|
...(columnDefinition.enumValues === undefined
|
|
5383
5600
|
? {}
|
|
5384
5601
|
: { enumValues: columnDefinition.enumValues }),
|
|
@@ -5516,6 +5733,28 @@ export class MinnowDatabase {
|
|
|
5516
5733
|
}
|
|
5517
5734
|
}
|
|
5518
5735
|
}
|
|
5736
|
+
// Adopting generation on an existing physical column is metadata-only only when every
|
|
5737
|
+
// stored value already equals the declared expression. This supports migrations from an
|
|
5738
|
+
// application-maintained column without silently blessing stale historical rows.
|
|
5739
|
+
for (const step of steps) {
|
|
5740
|
+
if (step.kind !== "alter-generated" || step.generatedValue === null)
|
|
5741
|
+
continue;
|
|
5742
|
+
const target = record.columns.find(({ name }) => name === step.columnName);
|
|
5743
|
+
if (target === undefined)
|
|
5744
|
+
continue;
|
|
5745
|
+
const expression = compileGeneratedColumnExpression(record.name, target.name, step.generatedValue.sql, record.columns);
|
|
5746
|
+
const rows = (await this.query(`SELECT * FROM ${quoteSqlIdentifier(record.name)}`, { memoize: false })).rows;
|
|
5747
|
+
for (const [rowIndex, row] of rows.entries()) {
|
|
5748
|
+
const expected = normalizeGeneratedColumnValue(target, evaluateRowExpression(expression, record.name, generatedEvaluationRow(record, row)), rowIndex);
|
|
5749
|
+
const actual = normalizeGeneratedColumnValue(target, row[target.name] ?? null, rowIndex);
|
|
5750
|
+
const equal = expected instanceof Date && actual instanceof Date
|
|
5751
|
+
? dateMilliseconds(expected) === dateMilliseconds(actual)
|
|
5752
|
+
: Object.is(expected, actual);
|
|
5753
|
+
if (!equal) {
|
|
5754
|
+
throw new TypeError(`Generated column cannot be adopted: ${record.name}.${target.name} differs from its expression at row ${String(rowIndex)}`);
|
|
5755
|
+
}
|
|
5756
|
+
}
|
|
5757
|
+
}
|
|
5519
5758
|
// The remaining metadata-only steps share one catalog/manifest CAS. Physical drops ran
|
|
5520
5759
|
// above because segment rewriting and payload retirement have a stronger atomic boundary.
|
|
5521
5760
|
const columns = applyColumnSteps(record, steps, this.#createId);
|
|
@@ -6067,6 +6306,7 @@ export class MinnowDatabase {
|
|
|
6067
6306
|
const statementNow = this.#now();
|
|
6068
6307
|
const filledProposed = await fillColumnDefaults(table, proposedBatch, (sql) => this.#evaluateDefaultExpression(sql, statementNow), statement.rows.length);
|
|
6069
6308
|
normalizeDomainBatch(table, filledProposed.batch);
|
|
6309
|
+
fillStoredGeneratedColumns(table, filledProposed.batch, statement.rows.length, filledProposed.generated);
|
|
6070
6310
|
fillCompositePrimaryKey(table, filledProposed.batch, statement.rows.length);
|
|
6071
6311
|
const proposed = filledProposed.batch;
|
|
6072
6312
|
// UPSERT only handles a uniqueness conflict. Other INSERT-domain failures are still errors,
|
|
@@ -6137,7 +6377,10 @@ export class MinnowDatabase {
|
|
|
6137
6377
|
// and UUID defaults twice and could even change which row conflicts.
|
|
6138
6378
|
const columns = visibleTableColumns(table).map(({ name }) => name);
|
|
6139
6379
|
const deferredAutoIncrement = new Set(filledProposed.autoIncrement?.missingIndexes ?? []);
|
|
6140
|
-
const
|
|
6380
|
+
const generatedColumns = new Set(table.columns.flatMap((column) => column.generatedValue === undefined ? [] : [column.name]));
|
|
6381
|
+
const rows = Array.from({ length: statement.rows.length }, (_, rowIndex) => columns.map((name) => generatedColumns.has(name) ||
|
|
6382
|
+
(name === filledProposed.autoIncrement?.column.name &&
|
|
6383
|
+
deferredAutoIncrement.has(rowIndex))
|
|
6141
6384
|
? { default: true }
|
|
6142
6385
|
: (proposed.columns[name]?.[rowIndex] ?? null)));
|
|
6143
6386
|
return this.runStatement({
|
|
@@ -6231,6 +6474,8 @@ export class MinnowDatabase {
|
|
|
6231
6474
|
if (freshRows.length > 0) {
|
|
6232
6475
|
const columns = {};
|
|
6233
6476
|
for (const column of table.columns) {
|
|
6477
|
+
if (column.generatedValue !== undefined)
|
|
6478
|
+
continue;
|
|
6234
6479
|
columns[column.name] = freshRows.map((rowIndex) => proposed.columns[column.name]?.[rowIndex] ?? null);
|
|
6235
6480
|
}
|
|
6236
6481
|
await transaction.insertBatch(table.name, { columns });
|
|
@@ -6269,7 +6514,9 @@ export class MinnowDatabase {
|
|
|
6269
6514
|
table: statement.table,
|
|
6270
6515
|
rowCount: outcome.affectedRows.length,
|
|
6271
6516
|
...(version === null || version === undefined ? {} : { version }),
|
|
6272
|
-
...(outcome.returnedRows === undefined
|
|
6517
|
+
...(outcome.returnedRows === undefined || returningColumns === undefined
|
|
6518
|
+
? {}
|
|
6519
|
+
: returningExecuteFields(table, returningColumns, outcome.returnedRows)),
|
|
6273
6520
|
};
|
|
6274
6521
|
}
|
|
6275
6522
|
/** ON CONFLICT DO NOTHING: drops insert rows whose unique key already exists at a snapshot. */
|
|
@@ -6292,10 +6539,13 @@ export class MinnowDatabase {
|
|
|
6292
6539
|
const statementNow = this.#now();
|
|
6293
6540
|
const filled = await fillColumnDefaults(table, proposed, (sql) => this.#evaluateDefaultExpression(sql, statementNow), statement.rows.length);
|
|
6294
6541
|
normalizeDomainBatch(table, filled.batch);
|
|
6542
|
+
fillStoredGeneratedColumns(table, filled.batch, statement.rows.length, filled.generated);
|
|
6295
6543
|
fillCompositePrimaryKey(table, filled.batch, statement.rows.length);
|
|
6296
6544
|
const columns = visibleTableColumns(table).map(({ name }) => name);
|
|
6297
6545
|
const deferredAutoIncrement = new Set(filled.autoIncrement?.missingIndexes ?? []);
|
|
6298
|
-
const
|
|
6546
|
+
const generatedColumns = new Set(table.columns.flatMap((column) => (column.generatedValue === undefined ? [] : [column.name])));
|
|
6547
|
+
const materializedRows = Array.from({ length: statement.rows.length }, (_, rowIndex) => columns.map((name) => generatedColumns.has(name) ||
|
|
6548
|
+
(name === filled.autoIncrement?.column.name && deferredAutoIncrement.has(rowIndex))
|
|
6299
6549
|
? { default: true }
|
|
6300
6550
|
: (filled.batch.columns[name]?.[rowIndex] ?? null)));
|
|
6301
6551
|
const keys = filled.batch.columns[keyColumn.name] ?? [];
|
|
@@ -6754,6 +7004,9 @@ export class MinnowDatabase {
|
|
|
6754
7004
|
if (record.columns.some(({ name }) => name === added.name)) {
|
|
6755
7005
|
throw new TypeError(`Column already exists: ${addColumnStatement.table}.${added.name}`);
|
|
6756
7006
|
}
|
|
7007
|
+
if (added.generatedValue !== undefined) {
|
|
7008
|
+
throw new TypeError("ALTER TABLE cannot add a generated column without rewriting existing rows");
|
|
7009
|
+
}
|
|
6757
7010
|
if (added.defaultValue !== undefined) {
|
|
6758
7011
|
validateColumnDefault({
|
|
6759
7012
|
name: added.name,
|
|
@@ -6912,11 +7165,19 @@ export class MinnowDatabase {
|
|
|
6912
7165
|
return this.#mergeConflictingInsertRows(statement, options);
|
|
6913
7166
|
}
|
|
6914
7167
|
if (statement.kind === "insert" && statement.rows.length === 0) {
|
|
7168
|
+
const table = await this.#findTable(statement.table);
|
|
7169
|
+
const returningColumns = options.returning === undefined
|
|
7170
|
+
? undefined
|
|
7171
|
+
: options.returning === "*"
|
|
7172
|
+
? visibleTableColumns(table).map(({ name }) => name)
|
|
7173
|
+
: [...options.returning];
|
|
6915
7174
|
return {
|
|
6916
7175
|
kind: "insert",
|
|
6917
7176
|
table: statement.table,
|
|
6918
7177
|
rowCount: 0,
|
|
6919
|
-
...(
|
|
7178
|
+
...(returningColumns === undefined
|
|
7179
|
+
? {}
|
|
7180
|
+
: returningExecuteFields(table, returningColumns, [])),
|
|
6920
7181
|
};
|
|
6921
7182
|
}
|
|
6922
7183
|
if (statement.kind === "insert") {
|
|
@@ -6978,7 +7239,7 @@ export class MinnowDatabase {
|
|
|
6978
7239
|
...(returningColumns === undefined
|
|
6979
7240
|
? {}
|
|
6980
7241
|
: {
|
|
6981
|
-
|
|
7242
|
+
...returningExecuteFields(table, returningColumns, statement.rows.map((row, rowIndex) => Object.fromEntries(returningColumns.map((name) => {
|
|
6982
7243
|
const column = table.columns.find((candidate) => candidate.name === name);
|
|
6983
7244
|
const position = statement.columns.indexOf(name);
|
|
6984
7245
|
const inserted = position < 0 ? undefined : row[position];
|
|
@@ -6988,7 +7249,7 @@ export class MinnowDatabase {
|
|
|
6988
7249
|
? null
|
|
6989
7250
|
: boundInsertValue(inserted));
|
|
6990
7251
|
return [name, executionSqlValueFromInput(column, value)];
|
|
6991
|
-
}))),
|
|
7252
|
+
})))),
|
|
6992
7253
|
}),
|
|
6993
7254
|
};
|
|
6994
7255
|
}
|
|
@@ -7037,7 +7298,7 @@ export class MinnowDatabase {
|
|
|
7037
7298
|
}
|
|
7038
7299
|
}
|
|
7039
7300
|
}
|
|
7040
|
-
const plan = {
|
|
7301
|
+
const plan = optimizePlan({
|
|
7041
7302
|
sql: `(${statement.kind})`,
|
|
7042
7303
|
base: { table: table.name, alias: table.name },
|
|
7043
7304
|
joins: [],
|
|
@@ -7049,7 +7310,33 @@ export class MinnowDatabase {
|
|
|
7049
7310
|
groupBy: [],
|
|
7050
7311
|
having: [],
|
|
7051
7312
|
orderBy: [],
|
|
7052
|
-
};
|
|
7313
|
+
});
|
|
7314
|
+
if (statement.kind === "delete" && returningColumns === undefined) {
|
|
7315
|
+
const internalWriter = options.writer;
|
|
7316
|
+
if (internalWriter.queryFirstColumn !== undefined) {
|
|
7317
|
+
const selectedKeys = await internalWriter.queryFirstColumn(plan);
|
|
7318
|
+
if (selectedKeys !== undefined) {
|
|
7319
|
+
const keys = selectedKeys;
|
|
7320
|
+
if (keyColumn.type === "string" && keyColumn.sqlDomain === undefined) {
|
|
7321
|
+
for (let index = 0; index < keys.length; index += 1) {
|
|
7322
|
+
keys[index] = storedSqlValueFromExecution(keyColumn, keys[index] ?? null);
|
|
7323
|
+
}
|
|
7324
|
+
}
|
|
7325
|
+
if (keys.some((key) => key === null)) {
|
|
7326
|
+
throw new TypeError(`Unique key values must not be null: ${keyColumn.name}`);
|
|
7327
|
+
}
|
|
7328
|
+
if (keys.length === 0) {
|
|
7329
|
+
return { kind: "delete", table: table.name, rowCount: 0 };
|
|
7330
|
+
}
|
|
7331
|
+
const deleted = await internalWriter.deleteBatch(table.name, { keys });
|
|
7332
|
+
return {
|
|
7333
|
+
kind: "delete",
|
|
7334
|
+
table: table.name,
|
|
7335
|
+
rowCount: deleted.rowCount,
|
|
7336
|
+
};
|
|
7337
|
+
}
|
|
7338
|
+
}
|
|
7339
|
+
}
|
|
7053
7340
|
let rows;
|
|
7054
7341
|
if (options.writer !== undefined) {
|
|
7055
7342
|
// Inside a scope the rows to touch are the ones the scope can see, which includes what it
|
|
@@ -7077,7 +7364,9 @@ export class MinnowDatabase {
|
|
|
7077
7364
|
kind: "delete",
|
|
7078
7365
|
table: table.name,
|
|
7079
7366
|
rowCount: 0,
|
|
7080
|
-
...(returnedRows === undefined
|
|
7367
|
+
...(returnedRows === undefined || returningColumns === undefined
|
|
7368
|
+
? {}
|
|
7369
|
+
: returningExecuteFields(table, returningColumns, returnedRows)),
|
|
7081
7370
|
};
|
|
7082
7371
|
}
|
|
7083
7372
|
const deleted = options.writer === undefined
|
|
@@ -7090,7 +7379,9 @@ export class MinnowDatabase {
|
|
|
7090
7379
|
...(deleted.version === undefined || deleted.version === null
|
|
7091
7380
|
? {}
|
|
7092
7381
|
: { version: deleted.version }),
|
|
7093
|
-
...(returnedRows === undefined
|
|
7382
|
+
...(returnedRows === undefined || returningColumns === undefined
|
|
7383
|
+
? {}
|
|
7384
|
+
: returningExecuteFields(table, returningColumns, returnedRows)),
|
|
7094
7385
|
};
|
|
7095
7386
|
}
|
|
7096
7387
|
if (keys.length === 0) {
|
|
@@ -7098,7 +7389,9 @@ export class MinnowDatabase {
|
|
|
7098
7389
|
kind: "update",
|
|
7099
7390
|
table: table.name,
|
|
7100
7391
|
rowCount: 0,
|
|
7101
|
-
...(returningColumns === undefined
|
|
7392
|
+
...(returningColumns === undefined
|
|
7393
|
+
? {}
|
|
7394
|
+
: returningExecuteFields(table, returningColumns, [])),
|
|
7102
7395
|
};
|
|
7103
7396
|
}
|
|
7104
7397
|
const changes = {};
|
|
@@ -7131,7 +7424,9 @@ export class MinnowDatabase {
|
|
|
7131
7424
|
name,
|
|
7132
7425
|
returnedChanges !== undefined && name in returnedChanges
|
|
7133
7426
|
? (returnedChanges[name]?.[index] ?? null)
|
|
7134
|
-
:
|
|
7427
|
+
: updated.generatedColumns?.[name] !== undefined
|
|
7428
|
+
? (updated.generatedColumns[name][index] ?? null)
|
|
7429
|
+
: (row[name] ?? null),
|
|
7135
7430
|
])));
|
|
7136
7431
|
return {
|
|
7137
7432
|
kind: "update",
|
|
@@ -7140,7 +7435,9 @@ export class MinnowDatabase {
|
|
|
7140
7435
|
...(updated.version === undefined || updated.version === null
|
|
7141
7436
|
? {}
|
|
7142
7437
|
: { version: updated.version }),
|
|
7143
|
-
...(returnedRows === undefined
|
|
7438
|
+
...(returnedRows === undefined || returningColumns === undefined
|
|
7439
|
+
? {}
|
|
7440
|
+
: returningExecuteFields(table, returningColumns, returnedRows)),
|
|
7144
7441
|
};
|
|
7145
7442
|
}
|
|
7146
7443
|
/**
|
|
@@ -7175,6 +7472,7 @@ export class MinnowDatabase {
|
|
|
7175
7472
|
*/
|
|
7176
7473
|
async #queryStreamed(plan, options, spillPageRows, probe, cursor) {
|
|
7177
7474
|
options = this.#effectiveQueryOptions(options);
|
|
7475
|
+
throwIfAborted(options.signal);
|
|
7178
7476
|
const tableNames = [plan.base.table, ...plan.joins.map((join) => join.table)];
|
|
7179
7477
|
const uniqueTableNames = [...new Set(tableNames)];
|
|
7180
7478
|
if (options.version === undefined) {
|
|
@@ -7204,6 +7502,7 @@ export class MinnowDatabase {
|
|
|
7204
7502
|
});
|
|
7205
7503
|
}
|
|
7206
7504
|
async #queryStreamedAtSnapshot(plan, options, spillPageRows, snapshot, tables, visibility, cursor) {
|
|
7505
|
+
throwIfAborted(options.signal);
|
|
7207
7506
|
// Copy-on-write: expansion clones only full-text plans, leaving the compile cache's copy
|
|
7208
7507
|
// untouched.
|
|
7209
7508
|
plan = expandFtsColumns(plan, (tableName) => searchableFtsColumns(tables.find((table) => table.name === tableName)));
|
|
@@ -7224,7 +7523,9 @@ export class MinnowDatabase {
|
|
|
7224
7523
|
const visibleByTable = new Map();
|
|
7225
7524
|
const rowCounts = new Map();
|
|
7226
7525
|
for (const table of tables) {
|
|
7526
|
+
throwIfAborted(options.signal);
|
|
7227
7527
|
const segments = await this.#visibleSegmentRecords(table, snapshot, visibility);
|
|
7528
|
+
throwIfAborted(options.signal);
|
|
7228
7529
|
visibleByTable.set(table.name, segments);
|
|
7229
7530
|
if (segments.every((segment) => {
|
|
7230
7531
|
const kind = segment.kind;
|
|
@@ -7258,6 +7559,7 @@ export class MinnowDatabase {
|
|
|
7258
7559
|
record.name,
|
|
7259
7560
|
record.id === freshBaseTable.id ? freshBaseTable : record,
|
|
7260
7561
|
])), snapshot, visibility);
|
|
7562
|
+
throwIfAborted(options.signal);
|
|
7261
7563
|
if (planContainsFts(plan, "bm25") && ftsStats === undefined)
|
|
7262
7564
|
return undefined;
|
|
7263
7565
|
// The streamed scan honors the same index pruning as the materialized path; a budgeted
|
|
@@ -7266,11 +7568,14 @@ export class MinnowDatabase {
|
|
|
7266
7568
|
(await this.#visibleSegmentRecords(freshBaseTable, snapshot, visibility));
|
|
7267
7569
|
this.#maybeScheduleAutoCompaction(freshBaseTable, visibleBaseSegments);
|
|
7268
7570
|
const ftsSegments = await this.#ftsPrunedSegments(freshBaseTable, visibleBaseSegments, plan, snapshot);
|
|
7571
|
+
throwIfAborted(options.signal);
|
|
7269
7572
|
const indexed = await this.#secondaryIndexPrunedSegments(freshBaseTable, ftsSegments, plan, snapshot);
|
|
7573
|
+
throwIfAborted(options.signal);
|
|
7270
7574
|
const baseSegments = indexed.segments;
|
|
7271
7575
|
// Zone-map elimination composes after index pruning: whole row groups whose statistics
|
|
7272
7576
|
// reject the plan's predicates never stream at all.
|
|
7273
7577
|
const zonePruned = await this.#zonePrunedStreamSegments(plan, freshBaseTable, projectedBaseColumns, baseSegments, snapshot);
|
|
7578
|
+
throwIfAborted(options.signal);
|
|
7274
7579
|
const baseView = this.#streamedViewFactory(baseTable, projectedBaseColumns, zonePruned?.segments ?? baseSegments, snapshot, zonePruned?.storedBlocks, zonePruned !== undefined || indexed.pruned);
|
|
7275
7580
|
if (baseView === undefined)
|
|
7276
7581
|
return undefined;
|
|
@@ -7292,22 +7597,26 @@ export class MinnowDatabase {
|
|
|
7292
7597
|
const budget = options.executionMemoryBudgetBytes;
|
|
7293
7598
|
const estimate = estimatedColumnarBytes(buildSegments, projectedBuildColumns);
|
|
7294
7599
|
if (buildView !== undefined && estimate > budget / 4) {
|
|
7295
|
-
return this.#runPartitionedJoin(plan, budget, estimate, baseView, buildView, buildTable.name, partitionedShape.buildKeyName);
|
|
7600
|
+
return this.#runPartitionedJoin(plan, budget, estimate, baseView, buildView, buildTable.name, partitionedShape.buildKeyName, options.signal);
|
|
7296
7601
|
}
|
|
7297
7602
|
}
|
|
7298
7603
|
}
|
|
7299
7604
|
}
|
|
7300
7605
|
const runStreamedExecution = async (attempt) => {
|
|
7606
|
+
throwIfAborted(options.signal);
|
|
7301
7607
|
const memory = new QueryMemoryContext(options.executionMemoryBudgetBytes);
|
|
7302
7608
|
let prepared;
|
|
7303
7609
|
try {
|
|
7304
7610
|
const streamed = await baseView.create(memory);
|
|
7611
|
+
throwIfAborted(options.signal);
|
|
7305
7612
|
const inputTables = new Map([[baseTable.name, streamed.table]]);
|
|
7306
7613
|
for (const table of tables) {
|
|
7614
|
+
throwIfAborted(options.signal);
|
|
7307
7615
|
if (table.name === baseTable.name)
|
|
7308
7616
|
continue;
|
|
7309
7617
|
const requestedColumns = columns.get(table.name) ?? [];
|
|
7310
7618
|
inputTables.set(table.name, await this.#materializeColumnarTableAtSnapshot(table, snapshot, requestedColumns.length === 0 ? [] : resolveReadColumns(table, requestedColumns), visibility));
|
|
7619
|
+
throwIfAborted(options.signal);
|
|
7311
7620
|
}
|
|
7312
7621
|
prepared = createPreparedColumnarQuery(plan, inputTables, memory, {
|
|
7313
7622
|
...(ftsStats === undefined ? {} : { ftsStats }),
|
|
@@ -7320,6 +7629,14 @@ export class MinnowDatabase {
|
|
|
7320
7629
|
batchRows: cursor.batchRows,
|
|
7321
7630
|
signal: cursor.signal,
|
|
7322
7631
|
loadScanWindow: streamed.load,
|
|
7632
|
+
...(cursor.consumeFirstColumn === undefined
|
|
7633
|
+
? {}
|
|
7634
|
+
: {
|
|
7635
|
+
consumeFirstColumn: async (values) => {
|
|
7636
|
+
emission.happened = true;
|
|
7637
|
+
await cursor.consumeFirstColumn?.(values);
|
|
7638
|
+
},
|
|
7639
|
+
}),
|
|
7323
7640
|
}, async (batch) => {
|
|
7324
7641
|
emission.happened = true;
|
|
7325
7642
|
await cursor.consume(externalizeQueryResult(batch));
|
|
@@ -7342,6 +7659,7 @@ export class MinnowDatabase {
|
|
|
7342
7659
|
try {
|
|
7343
7660
|
const streamedResult = await prepared.executeAsync({
|
|
7344
7661
|
loadScanWindow: streamed.load,
|
|
7662
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
7345
7663
|
});
|
|
7346
7664
|
options.onStats?.({ peakMemoryBytes: memory.usage.peakBytes });
|
|
7347
7665
|
return streamedResult;
|
|
@@ -7356,6 +7674,7 @@ export class MinnowDatabase {
|
|
|
7356
7674
|
...(spillPageRows === undefined ? {} : { spillPageRows }),
|
|
7357
7675
|
spillStore: this.#leasedSpillStore(),
|
|
7358
7676
|
loadScanWindow: streamed.load,
|
|
7677
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
7359
7678
|
});
|
|
7360
7679
|
options.onStats?.({ peakMemoryBytes: memory.usage.peakBytes });
|
|
7361
7680
|
return spilledResult;
|
|
@@ -7605,7 +7924,8 @@ export class MinnowDatabase {
|
|
|
7605
7924
|
* like any unordered query. The rescans trade time for bounded memory; the decoded-block
|
|
7606
7925
|
* cache keeps repeat decodes cheap.
|
|
7607
7926
|
*/
|
|
7608
|
-
async #runPartitionedJoin(plan, budgetBytes, estimatedBuildBytes, baseView, buildView, buildTableName, buildKeyName) {
|
|
7927
|
+
async #runPartitionedJoin(plan, budgetBytes, estimatedBuildBytes, baseView, buildView, buildTableName, buildKeyName, signal) {
|
|
7928
|
+
throwIfAborted(signal);
|
|
7609
7929
|
// One partition should fit in an eighth of the budget: the resident partition briefly
|
|
7610
7930
|
// exists twice (boxed rows, then their vectors) and shares the budget with both streamed
|
|
7611
7931
|
// windows and the accumulating result.
|
|
@@ -7618,6 +7938,7 @@ export class MinnowDatabase {
|
|
|
7618
7938
|
try {
|
|
7619
7939
|
const combined = [];
|
|
7620
7940
|
for (let partition = 0; partition < partitions; partition += 1) {
|
|
7941
|
+
throwIfAborted(signal);
|
|
7621
7942
|
if (combined.length >= wantedRows)
|
|
7622
7943
|
break;
|
|
7623
7944
|
const passMemory = root.createChild();
|
|
@@ -7628,6 +7949,7 @@ export class MinnowDatabase {
|
|
|
7628
7949
|
const collectMemory = passMemory.createChild();
|
|
7629
7950
|
const buildScanMemory = collectMemory.createChild();
|
|
7630
7951
|
const build = await buildView.create(buildScanMemory);
|
|
7952
|
+
throwIfAborted(signal);
|
|
7631
7953
|
let buildRows = [];
|
|
7632
7954
|
const buildColumns = [...build.table.columns.entries()];
|
|
7633
7955
|
// Small steps keep the collection windows a minor budget term next to the resident
|
|
@@ -7635,8 +7957,10 @@ export class MinnowDatabase {
|
|
|
7635
7957
|
// consumed length clamps to the returned resident end.
|
|
7636
7958
|
const step = 1_024;
|
|
7637
7959
|
for (let start = 0; start < build.table.rowCount;) {
|
|
7960
|
+
throwIfAborted(signal);
|
|
7638
7961
|
let length = Math.min(step, build.table.rowCount - start);
|
|
7639
7962
|
const residentEnd = await build.load(start, length);
|
|
7963
|
+
throwIfAborted(signal);
|
|
7640
7964
|
if (typeof residentEnd === "number" && residentEnd > start) {
|
|
7641
7965
|
length = Math.min(length, residentEnd - start);
|
|
7642
7966
|
}
|
|
@@ -7667,13 +7991,17 @@ export class MinnowDatabase {
|
|
|
7667
7991
|
const buildInput = columnarTableFromRows(buildTableName, buildRows, undefined, false);
|
|
7668
7992
|
buildRows = [];
|
|
7669
7993
|
const streamedBase = await baseView.create(passMemory);
|
|
7994
|
+
throwIfAborted(signal);
|
|
7670
7995
|
const inputTables = new Map([
|
|
7671
7996
|
[buildTableName, buildInput],
|
|
7672
7997
|
[streamedBase.table.name, streamedBase.table],
|
|
7673
7998
|
]);
|
|
7674
7999
|
prepared = createPreparedColumnarQuery(strippedPlan, inputTables, passMemory);
|
|
7675
8000
|
collectMemory.close();
|
|
7676
|
-
const result = await prepared.executeAsync({
|
|
8001
|
+
const result = await prepared.executeAsync({
|
|
8002
|
+
loadScanWindow: streamedBase.load,
|
|
8003
|
+
...(signal === undefined ? {} : { signal }),
|
|
8004
|
+
});
|
|
7677
8005
|
for (const row of result.rows) {
|
|
7678
8006
|
if (combined.length >= wantedRows)
|
|
7679
8007
|
break;
|
|
@@ -8237,17 +8565,24 @@ export class MinnowDatabase {
|
|
|
8237
8565
|
};
|
|
8238
8566
|
}
|
|
8239
8567
|
/** Prepares one block's inputs and executes it, returning the caller-owned result. */
|
|
8240
|
-
async #executeBlock(block, snapshot, visibility, memory, realTables, typedSchemas, extraInputs, cacheResults = true, allowSpill = true, forceSpill = false, spillPageRows) {
|
|
8568
|
+
async #executeBlock(block, snapshot, visibility, memory, realTables, typedSchemas, extraInputs, cacheResults = true, allowSpill = true, forceSpill = false, spillPageRows, signal) {
|
|
8569
|
+
throwIfAborted(signal);
|
|
8241
8570
|
const ftsStats = await this.#ftsIndexStats(block, realTables, snapshot, visibility);
|
|
8242
|
-
|
|
8571
|
+
throwIfAborted(signal);
|
|
8572
|
+
const inputs = await this.#prepareBlockInputs(block, snapshot, visibility, memory, realTables, typedSchemas, extraInputs, cacheResults, allowSpill, forceSpill, spillPageRows, signal);
|
|
8573
|
+
throwIfAborted(signal);
|
|
8243
8574
|
const prepared = createPreparedColumnarQuery(block, inputs, memory.createChild(), ftsStats === undefined ? {} : { ftsStats });
|
|
8244
8575
|
try {
|
|
8245
8576
|
if (!allowSpill || memory.usage.budgetBytes === Number.MAX_SAFE_INTEGER) {
|
|
8246
|
-
|
|
8577
|
+
const result = prepared.execute();
|
|
8578
|
+
throwIfAborted(signal);
|
|
8579
|
+
return result;
|
|
8247
8580
|
}
|
|
8248
8581
|
if (!forceSpill) {
|
|
8249
8582
|
try {
|
|
8250
|
-
|
|
8583
|
+
const result = prepared.execute();
|
|
8584
|
+
throwIfAborted(signal);
|
|
8585
|
+
return result;
|
|
8251
8586
|
}
|
|
8252
8587
|
catch (error) {
|
|
8253
8588
|
if (!(error instanceof QueryMemoryBudgetError))
|
|
@@ -8257,6 +8592,7 @@ export class MinnowDatabase {
|
|
|
8257
8592
|
return await prepared.executeAsync({
|
|
8258
8593
|
spillStore: this.#leasedSpillStore(),
|
|
8259
8594
|
...(spillPageRows === undefined ? {} : { spillPageRows }),
|
|
8595
|
+
...(signal === undefined ? {} : { signal }),
|
|
8260
8596
|
});
|
|
8261
8597
|
}
|
|
8262
8598
|
finally {
|
|
@@ -8270,14 +8606,17 @@ export class MinnowDatabase {
|
|
|
8270
8606
|
* real tables it references, so an identical block over identical table states reuses
|
|
8271
8607
|
* the previous result instead of re-reading and re-executing.
|
|
8272
8608
|
*/
|
|
8273
|
-
async #executeBlockCached(block, snapshot, visibility, memory, realTables, typedSchemas, cacheResults = true, allowSpill = true, forceSpill = false, spillPageRows) {
|
|
8609
|
+
async #executeBlockCached(block, snapshot, visibility, memory, realTables, typedSchemas, cacheResults = true, allowSpill = true, forceSpill = false, spillPageRows, signal) {
|
|
8610
|
+
throwIfAborted(signal);
|
|
8274
8611
|
const key = await this.#blockResultCacheKey(block, snapshot, visibility, realTables, cacheResults);
|
|
8612
|
+
throwIfAborted(signal);
|
|
8275
8613
|
if (key !== undefined) {
|
|
8276
8614
|
const cached = this.#cacheGet(key);
|
|
8277
8615
|
if (cached !== undefined)
|
|
8278
8616
|
return cached;
|
|
8279
8617
|
}
|
|
8280
|
-
const result = await this.#executeBlock(block, snapshot, visibility, memory, realTables, typedSchemas, undefined, cacheResults, allowSpill, forceSpill, spillPageRows);
|
|
8618
|
+
const result = await this.#executeBlock(block, snapshot, visibility, memory, realTables, typedSchemas, undefined, cacheResults, allowSpill, forceSpill, spillPageRows, signal);
|
|
8619
|
+
throwIfAborted(signal);
|
|
8281
8620
|
if (key !== undefined)
|
|
8282
8621
|
this.#cachePut(key, result, queryResultRetainedBytes(result));
|
|
8283
8622
|
return result;
|
|
@@ -8288,15 +8627,18 @@ export class MinnowDatabase {
|
|
|
8288
8627
|
* synthetic sources in typedSchemas, and a later cache hit skips that registration, so
|
|
8289
8628
|
* re-inferring the schema at hit time would find those names missing.
|
|
8290
8629
|
*/
|
|
8291
|
-
async #executeBlockWithSchemaCached(block, snapshot, visibility, memory, realTables, typedSchemas, cacheResults = true, allowSpill = true, forceSpill = false, spillPageRows) {
|
|
8630
|
+
async #executeBlockWithSchemaCached(block, snapshot, visibility, memory, realTables, typedSchemas, cacheResults = true, allowSpill = true, forceSpill = false, spillPageRows, signal) {
|
|
8631
|
+
throwIfAborted(signal);
|
|
8292
8632
|
const key = await this.#blockResultCacheKey(block, snapshot, visibility, realTables, cacheResults);
|
|
8633
|
+
throwIfAborted(signal);
|
|
8293
8634
|
const schemaKey = key === undefined ? undefined : `s${key}`;
|
|
8294
8635
|
if (schemaKey !== undefined) {
|
|
8295
8636
|
const cached = this.#cacheGet(schemaKey);
|
|
8296
8637
|
if (cached !== undefined)
|
|
8297
8638
|
return cached;
|
|
8298
8639
|
}
|
|
8299
|
-
const result = await this.#executeBlock(block, snapshot, visibility, memory, realTables, typedSchemas, undefined, cacheResults, allowSpill, forceSpill, spillPageRows);
|
|
8640
|
+
const result = await this.#executeBlock(block, snapshot, visibility, memory, realTables, typedSchemas, undefined, cacheResults, allowSpill, forceSpill, spillPageRows, signal);
|
|
8641
|
+
throwIfAborted(signal);
|
|
8300
8642
|
const schema = inferBlockSchema(block, typedSchemas);
|
|
8301
8643
|
const entry = { result, schema };
|
|
8302
8644
|
if (schemaKey !== undefined) {
|
|
@@ -10669,7 +11011,7 @@ export class MinnowDatabase {
|
|
|
10669
11011
|
if (recoveredManifest === undefined) {
|
|
10670
11012
|
throw new Error(`Compaction manifest is missing: ${String(linkedTransaction.committedVersion)}`);
|
|
10671
11013
|
}
|
|
10672
|
-
this.#afterCompactionCommit(recoveredManifest);
|
|
11014
|
+
this.#afterCompactionCommit(recoveredManifest, table.id);
|
|
10673
11015
|
return this.#publishedCompactionProgress(table, job);
|
|
10674
11016
|
}
|
|
10675
11017
|
const rewritePlan = job.rewritePlan;
|
|
@@ -10855,7 +11197,7 @@ export class MinnowDatabase {
|
|
|
10855
11197
|
let publicationConflict;
|
|
10856
11198
|
try {
|
|
10857
11199
|
manifest = await transaction.commit();
|
|
10858
|
-
this.#afterCompactionCommit(manifest);
|
|
11200
|
+
this.#afterCompactionCommit(manifest, table.id);
|
|
10859
11201
|
break;
|
|
10860
11202
|
}
|
|
10861
11203
|
catch (error) {
|
|
@@ -14096,13 +14438,27 @@ function assertTriggerBodyTargetSchema(compiled, target) {
|
|
|
14096
14438
|
if (column !== undefined &&
|
|
14097
14439
|
isDefaultInsertValue(value) &&
|
|
14098
14440
|
!column.nullable &&
|
|
14099
|
-
column.defaultValue === undefined
|
|
14441
|
+
column.defaultValue === undefined &&
|
|
14442
|
+
column.generatedValue === undefined) {
|
|
14100
14443
|
throw new TypeError(`Trigger body INSERT uses DEFAULT for a non-nullable column without a default: ${column.name}`);
|
|
14101
14444
|
}
|
|
14102
14445
|
}
|
|
14103
14446
|
}
|
|
14447
|
+
for (const [index, column] of insertColumns.entries()) {
|
|
14448
|
+
if (column.generatedValue === undefined)
|
|
14449
|
+
continue;
|
|
14450
|
+
for (const row of compiled.rows) {
|
|
14451
|
+
const value = row[index];
|
|
14452
|
+
if (value === undefined || !isDefaultInsertValue(value)) {
|
|
14453
|
+
throw new TypeError(`Trigger body INSERT assigns generated column: ${column.name}`);
|
|
14454
|
+
}
|
|
14455
|
+
}
|
|
14456
|
+
}
|
|
14104
14457
|
for (const column of columns) {
|
|
14105
|
-
if (!provided.has(column.name) &&
|
|
14458
|
+
if (!provided.has(column.name) &&
|
|
14459
|
+
!column.nullable &&
|
|
14460
|
+
column.defaultValue === undefined &&
|
|
14461
|
+
column.generatedValue === undefined) {
|
|
14106
14462
|
throw new TypeError(`Trigger body INSERT omits a non-nullable column without a default: ${column.name}`);
|
|
14107
14463
|
}
|
|
14108
14464
|
}
|
|
@@ -14124,6 +14480,9 @@ function assertTriggerBodyTargetSchema(compiled, target) {
|
|
|
14124
14480
|
if (assignedColumn === undefined) {
|
|
14125
14481
|
throw new TypeError(`Trigger body UPDATE assignment column does not exist: ${assignment.column}`);
|
|
14126
14482
|
}
|
|
14483
|
+
if (assignedColumn.generatedValue !== undefined) {
|
|
14484
|
+
throw new TypeError(`Trigger body UPDATE assigns generated column: ${assignment.column}`);
|
|
14485
|
+
}
|
|
14127
14486
|
if (assignedColumn.id === target.uniqueKeyColumnId ||
|
|
14128
14487
|
(target.primaryKeyColumnIds ?? []).includes(assignedColumn.id)) {
|
|
14129
14488
|
throw new TypeError(`Trigger body UPDATE cannot update a key column: ${assignment.column}`);
|
|
@@ -14273,6 +14632,7 @@ function visibleSegmentManifestVersion(value, label) {
|
|
|
14273
14632
|
*/
|
|
14274
14633
|
const compiledChecks = new Map();
|
|
14275
14634
|
const COMPILED_CHECK_CACHE_LIMIT = 64;
|
|
14635
|
+
const compiledGeneratedColumns = new Map();
|
|
14276
14636
|
function tableChecks(table) {
|
|
14277
14637
|
const key = `${table.id}/${String(table.revision)}`;
|
|
14278
14638
|
const cached = compiledChecks.get(key);
|
|
@@ -14295,6 +14655,84 @@ function tableChecks(table) {
|
|
|
14295
14655
|
compiledChecks.set(key, compiled);
|
|
14296
14656
|
return compiled;
|
|
14297
14657
|
}
|
|
14658
|
+
function tableGeneratedExpressions(table) {
|
|
14659
|
+
const key = `${table.id}/${String(table.revision)}`;
|
|
14660
|
+
const cached = compiledGeneratedColumns.get(key);
|
|
14661
|
+
if (cached !== undefined)
|
|
14662
|
+
return cached;
|
|
14663
|
+
const compiled = table.columns.flatMap((column) => column.generatedValue === undefined
|
|
14664
|
+
? []
|
|
14665
|
+
: [
|
|
14666
|
+
{
|
|
14667
|
+
column,
|
|
14668
|
+
expression: compileGeneratedColumnExpression(table.name, column.name, column.generatedValue.sql, table.columns),
|
|
14669
|
+
},
|
|
14670
|
+
]);
|
|
14671
|
+
if (compiledGeneratedColumns.size >= COMPILED_CHECK_CACHE_LIMIT) {
|
|
14672
|
+
const oldest = compiledGeneratedColumns.keys().next().value;
|
|
14673
|
+
if (oldest !== undefined)
|
|
14674
|
+
compiledGeneratedColumns.delete(oldest);
|
|
14675
|
+
}
|
|
14676
|
+
compiledGeneratedColumns.set(key, compiled);
|
|
14677
|
+
return compiled;
|
|
14678
|
+
}
|
|
14679
|
+
function normalizeGeneratedColumnValue(column, value, rowIndex) {
|
|
14680
|
+
const stored = column.sqlDomain === undefined
|
|
14681
|
+
? storedSqlValueFromExecution(column, value)
|
|
14682
|
+
: normalizeSqlDomainValue(column.sqlDomain, value);
|
|
14683
|
+
validateValue(column, stored, rowIndex);
|
|
14684
|
+
return stored;
|
|
14685
|
+
}
|
|
14686
|
+
/** Computes every stored generated column after defaults/domains have produced the base row. */
|
|
14687
|
+
function fillStoredGeneratedColumns(table, batch, rowCount, generated) {
|
|
14688
|
+
for (const { column, expression } of tableGeneratedExpressions(table)) {
|
|
14689
|
+
const values = new Array(rowCount);
|
|
14690
|
+
for (let rowIndex = 0; rowIndex < rowCount; rowIndex += 1) {
|
|
14691
|
+
const row = batchRowAt(table, batch, rowIndex);
|
|
14692
|
+
const value = evaluateRowExpression(expression, table.name, row);
|
|
14693
|
+
values[rowIndex] = normalizeGeneratedColumnValue(column, value, rowIndex);
|
|
14694
|
+
}
|
|
14695
|
+
batch.columns[column.name] = values;
|
|
14696
|
+
generated?.set(column.name, values.map((value) => externalSqlDomainValue(value)));
|
|
14697
|
+
}
|
|
14698
|
+
}
|
|
14699
|
+
function rejectGeneratedUpdateAssignments(table, input) {
|
|
14700
|
+
for (const column of table.columns) {
|
|
14701
|
+
if (column.generatedValue !== undefined && input.changes[column.name] !== undefined) {
|
|
14702
|
+
throw new TypeError(`Generated column cannot be assigned: ${column.name}`);
|
|
14703
|
+
}
|
|
14704
|
+
}
|
|
14705
|
+
}
|
|
14706
|
+
function generatedEvaluationRow(table, source) {
|
|
14707
|
+
const row = {};
|
|
14708
|
+
for (const column of table.columns) {
|
|
14709
|
+
const value = source[column.name] ?? null;
|
|
14710
|
+
row[column.name] =
|
|
14711
|
+
column.sqlDomain === undefined
|
|
14712
|
+
? executionSqlValueFromStorage(column, value)
|
|
14713
|
+
: normalizeSqlDomainValue(column.sqlDomain, value);
|
|
14714
|
+
}
|
|
14715
|
+
return row;
|
|
14716
|
+
}
|
|
14717
|
+
function applyStoredGeneratedUpdateChanges(table, input, preImages) {
|
|
14718
|
+
const generated = tableGeneratedExpressions(table);
|
|
14719
|
+
if (generated.length === 0)
|
|
14720
|
+
return input;
|
|
14721
|
+
const changes = { ...input.changes };
|
|
14722
|
+
for (const { column, expression } of generated) {
|
|
14723
|
+
changes[column.name] = input.keys.map((_, rowIndex) => {
|
|
14724
|
+
const old = preImages[rowIndex];
|
|
14725
|
+
if (old === undefined)
|
|
14726
|
+
throw new Error(`Generated-column pre-image is missing: ${table.name}`);
|
|
14727
|
+
const row = generatedEvaluationRow(table, old);
|
|
14728
|
+
for (const [name, values] of Object.entries(input.changes)) {
|
|
14729
|
+
row[name] = values[rowIndex] ?? null;
|
|
14730
|
+
}
|
|
14731
|
+
return normalizeGeneratedColumnValue(column, evaluateRowExpression(expression, table.name, row), rowIndex);
|
|
14732
|
+
});
|
|
14733
|
+
}
|
|
14734
|
+
return { keys: input.keys, changes };
|
|
14735
|
+
}
|
|
14298
14736
|
/**
|
|
14299
14737
|
* Applies a table's CHECK constraints to one row (E141-06). A constraint fails only when it
|
|
14300
14738
|
* evaluates to false: SQL's three-valued logic lets an unknown pass, which is why a NULL column
|
|
@@ -15414,7 +15852,7 @@ function tryKeyToken(type, value) {
|
|
|
15414
15852
|
* (level one and above) are the folded state, not fragmentation, and do not count — a large
|
|
15415
15853
|
* keyed table is many partitions by design.
|
|
15416
15854
|
*/
|
|
15417
|
-
function
|
|
15855
|
+
function autoCompactionHint(version, segments) {
|
|
15418
15856
|
let levelZero = 0;
|
|
15419
15857
|
let deltas = 0;
|
|
15420
15858
|
for (const segment of segments) {
|
|
@@ -15424,7 +15862,18 @@ function autoCompactionDue(segments) {
|
|
|
15424
15862
|
if (kind !== "insert" && kind !== "base")
|
|
15425
15863
|
deltas += 1;
|
|
15426
15864
|
}
|
|
15427
|
-
return
|
|
15865
|
+
return {
|
|
15866
|
+
version,
|
|
15867
|
+
visible: segments.length,
|
|
15868
|
+
levelZero,
|
|
15869
|
+
deltas,
|
|
15870
|
+
};
|
|
15871
|
+
}
|
|
15872
|
+
function autoCompactionDueHint(hint) {
|
|
15873
|
+
return hint.levelZero >= AUTO_COMPACT_SCAN_SEGMENTS || hint.deltas >= AUTO_COMPACT_DELTA_SEGMENTS;
|
|
15874
|
+
}
|
|
15875
|
+
function autoCompactionDue(segments) {
|
|
15876
|
+
return autoCompactionDueHint(autoCompactionHint(null, segments));
|
|
15428
15877
|
}
|
|
15429
15878
|
/** A macrotask boundary, so background work lets queued queries and writes run between steps. */
|
|
15430
15879
|
function yieldToEventLoop() {
|