@minnowdb/core 0.7.2 → 0.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine/database.d.ts +6 -0
- package/dist/engine/database.js +385 -88
- package/dist/engine/optimizer.js +48 -26
- package/dist/engine/query.d.ts +98 -0
- package/dist/engine/query.js +1452 -165
- package/dist/engine/sql-domains.d.ts +11 -0
- package/dist/engine/sql-domains.js +56 -0
- package/dist/engine/sql-functions.js +129 -3
- package/dist/engine/sql-json.d.ts +2 -0
- package/dist/engine/sql-json.js +4 -0
- package/dist/engine/sql-semantics.d.ts +9 -1
- package/dist/engine/sql-semantics.js +19 -3
- package/dist/engine/vector.js +29 -13
- package/dist/plan/model.d.ts +14 -2
- package/dist/storage/memory.d.ts +2 -1
- package/dist/storage/memory.js +29 -4
- package/dist/testing/sqllogictest.js +3 -1
- package/package.json +1 -1
- package/postgres-feature-profile.json +9 -4
- package/sql-feature-matrix.json +541 -4
package/dist/engine/database.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { crossJoinPlan } from "../plan/model.js";
|
|
1
2
|
import { toColumnarBatch } from "./batch.js";
|
|
2
3
|
import { ArtifactCache } from "./artifact-cache.js";
|
|
3
4
|
import { estimateBatchBytes, estimateRowBytes, estimateValuesBytes } from "./byte-estimates.js";
|
|
@@ -15,13 +16,13 @@ import { cachedQueryTerms, FTS_TOKENIZER_VERSION, renderDocumentValue, tokenize
|
|
|
15
16
|
import { boundedMaintenanceBatchItems, simpleDataTypes, floorWholeNumberProduct, BlockReadBatchTooLargeError, validateColumnDefault, validateEnumValues, secondaryIndexColumnIds, secondaryIndexDirections, secondaryUniqueKeyNamespace, CompactionJobConflictError, CompactionBacklogError, GarbageCollectionJobConflictError, MAX_LEVEL_ZERO_SEGMENTS, MAX_FTS_CANDIDATE_ROW_IDS, MAX_FTS_POSTING_TERM_CHARACTERS, MAX_FTS_POSTINGS_PER_CHUNK, MAX_FTS_POSTING_ROW_IDS_PER_CHUNK, MAX_INDEXED_STRING_CHARACTERS, MAX_CATALOG_NAME_CHARACTERS, MAX_MANIFEST_BLOCK_PRESENCE_IDS, MAX_MAINTENANCE_BATCH_ITEMS, MAX_STORAGE_BULK_READ_ITEMS, MAX_TRANSACTION_STAGE_BLOCKS, MAX_TRANSACTION_STAGE_BYTES, MAX_SNAPSHOT_SESSION_TTL_MS, MAX_POSTING_BUILD_TTL_MS, MAX_TEMP_OWNER_TTL_MS, MAX_SNAPSHOT_FRAME_BATCH_BYTES, MAX_SNAPSHOT_FRAME_BATCH_ITEMS, MAX_SNAPSHOT_METADATA_BATCH_BYTES, SNAPSHOT_FRAME_KINDS, validateStorageId, SnapshotManifestMissingError, SchemaConflictError, TableInUseError, TableRecordConflictError, TransactionRecordConflictError, UniqueKeyConflictError, UniqueIndexCoverageError, WriteConflictError } from "../storage/types.js";
|
|
16
17
|
import { decodeSnapshotFrameStream, encodeSnapshotFrameStreamFooter, encodeSnapshotFrameStreamHeader, extendSnapshotFrameStreamChecksum, snapshotFrameEnvelopeParts, snapshotFrameStreamHeaderIdentity } from "../storage/snapshot.js";
|
|
17
18
|
import { Snapshot, TransactionManager } from "../transactions/index.js";
|
|
18
|
-
import { applyWindowFunctions, bindPendingSelectShapes, bindPlanParameters, bindStatementParameters, DUAL_TABLE, dualTableRows, blockHasSubqueries, combineUnionResults, compileCheckExpression, compileQuery, hasAggregate, createRecursiveCteState, annotateAvgArgumentScales, compileStatement, comparisonHolds, createPreparedColumnarQuery, evaluateJoinedRowExpression, evaluateRowExpression, externalizeQueryResult, expressionColumnNames, inferBlockSchema, inferResultColumnDomains, isDeferredInsertExpression, isDefaultInsertValue, referencedColumns, childExpressions, expandFtsColumns, expandNaturalJoins, expandSourceColumnAliases, expandViewSources, forEachBlockExpression, forEachNestedBlock, planContainsFts, planHasNaturalJoins, planHasSourceColumnAliases, planReadsTable, planReadsViews, planReadsBeyondSingleScan, projectResultColumns, queryResultNeedsExternalization, resolveStatementDatetimes, subqueryResolutionSteps, topLevelFtsMatchConjuncts, transparentProjectionSource, unknownColumnDomains, validateDefaultExpression, windowOutputDomain, windowOutputType, volatileScalarFunctionNames } from "./query.js";
|
|
19
|
+
import { applyWindowFunctions, bindPendingSelectShapes, bindPlanParameters, bindStatementParameters, DUAL_TABLE, dualTableRows, blockHasSubqueries, combineUnionResults, compileCheckExpression, compileQuery, hasAggregate, createRecursiveCteState, annotateAvgArgumentScales, compileStatement, comparisonHolds, createPreparedColumnarQuery, evaluateJoinedRowExpression, evaluateRowExpression, externalizeQueryResult, expressionColumnNames, inferBlockSchema, inferResultColumnDomains, isDeferredInsertExpression, isDefaultInsertValue, referencedColumns, childExpressions, expandFtsColumns, expandNaturalJoins, expandSourceColumnAliases, expandViewSources, forEachBlockExpression, forEachNestedBlock, planContainsFts, planHasNaturalJoins, planHasSourceColumnAliases, planReadsTable, planReadsViews, planReadsBeyondSingleScan, projectResultColumns, queryResultNeedsExternalization, resolveStatementDatetimes, subqueryResolutionSteps, topLevelFtsMatchConjuncts, transparentProjectionSource, unknownColumnDomains, validateDefaultExpression, windowOutputDomain, windowOutputType, volatileScalarFunctionNames, annotatePlanIntegerDivision, clonePlanTree, planMayHaveIntegerDivision, foldIdentifierCase, extendGroupByWithKeyDependents, expandRowReferences } from "./query.js";
|
|
19
20
|
import { copyQueryResult, planMemoKey, queryResultMemoKey, queryResultRetainedBytes, RESULT_MEMO_MAX_BYTES } from "./query-cache.js";
|
|
20
21
|
import { QueryMemoryBudgetError, QueryMemoryContext, DEFAULT_QUERY_MEMORY_BUDGET_BYTES } from "./memory.js";
|
|
21
22
|
import { LiveQueryLimitError, LiveQuerySet, MAX_LIVE_QUERY_SETS_PER_DATABASE } from "./live.js";
|
|
22
23
|
import { chooseJoinOrder, optimizePlan, qualifyCorrelatedReferences, renderPlan } from "./optimizer.js";
|
|
23
|
-
import { encodeSqlEqualityValue,
|
|
24
|
-
import { exactNumericAsNumber, externalSqlDomainValue, externalSqlTextValue, isDateDomainValue, isExactNumeric, isSqlDomainValue, normalizeSqlDomainValue, protectedSqlTextValue } from "./sql-domains.js";
|
|
24
|
+
import { encodeSqlEqualityValue, readUntypedText } from "./sql-semantics.js";
|
|
25
|
+
import { exactNumericAsNumber, exactNumericValue, externalSqlDomainValue, externalSqlTextValue, isDateDomainValue, isExactNumeric, isSqlDomainValue, normalizeSqlDomainValue, protectedSqlTextValue } from "./sql-domains.js";
|
|
25
26
|
import { toCatalog } from "./catalog.js";
|
|
26
27
|
import { applyColumnSteps, assertColumnDroppable, compileGeneratedColumnExpression, declaredForeignKeys, isDestructiveStep, planMigration } from "./schema.js";
|
|
27
28
|
import { columnarTableFromRows, createColumnarTable, vectorValue } from "./vector.js";
|
|
@@ -77,6 +78,9 @@ const AUTO_COLLECT_RETRY_MIN_MS = 1e3;
|
|
|
77
78
|
const AUTO_COLLECT_RETRY_MAX_MS = 6e4;
|
|
78
79
|
const AUTO_COMPACT_STEP_BLOCKS = 4;
|
|
79
80
|
const AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS = 256;
|
|
81
|
+
const AUTO_COMPACT_RETRY_MIN_MS = 250;
|
|
82
|
+
const AUTO_COMPACT_RETRY_MAX_MS = 6e4;
|
|
83
|
+
const MAX_COMPACTION_PUBLICATION_CONFLICTS = 64;
|
|
80
84
|
const ZONE_DESCRIPTION_CACHE_BYTES = 160;
|
|
81
85
|
const STAGED_OVERLAY_ORDER_BASE = 2 ** 52;
|
|
82
86
|
class StaleTriggerDerivationsError extends Error {
|
|
@@ -202,6 +206,7 @@ function projectReturningItems(table, alias, items, rows, facts) {
|
|
|
202
206
|
having: [],
|
|
203
207
|
orderBy: []
|
|
204
208
|
}, facts.domains, facts.types);
|
|
209
|
+
annotatePlanIntegerDivision(plan, (name) => facts.integers.get(name));
|
|
205
210
|
const visible = visibleTableColumns(table);
|
|
206
211
|
const columns = new Map(visible.map((column) => [
|
|
207
212
|
column.name,
|
|
@@ -338,6 +343,18 @@ function resolveMutationStatementDatetimes(statement, now) {
|
|
|
338
343
|
}
|
|
339
344
|
};
|
|
340
345
|
}
|
|
346
|
+
const sources = (from) => from === void 0 ? {} : {
|
|
347
|
+
from: {
|
|
348
|
+
...from,
|
|
349
|
+
predicates: predicates(from.predicates),
|
|
350
|
+
joins: from.joins.map((join) => ({
|
|
351
|
+
...join,
|
|
352
|
+
left: expression(join.left),
|
|
353
|
+
right: expression(join.right),
|
|
354
|
+
...join.on === void 0 ? {} : { on: expression(join.on) }
|
|
355
|
+
}))
|
|
356
|
+
}
|
|
357
|
+
};
|
|
341
358
|
if (statement.kind === "update") {
|
|
342
359
|
return {
|
|
343
360
|
...statement,
|
|
@@ -345,11 +362,16 @@ function resolveMutationStatementDatetimes(statement, now) {
|
|
|
345
362
|
...assignment,
|
|
346
363
|
expression: expression(assignment.expression)
|
|
347
364
|
})),
|
|
348
|
-
predicates: predicates(statement.predicates)
|
|
365
|
+
predicates: predicates(statement.predicates),
|
|
366
|
+
...sources(statement.from)
|
|
349
367
|
};
|
|
350
368
|
}
|
|
351
369
|
if (statement.kind === "delete") {
|
|
352
|
-
return {
|
|
370
|
+
return {
|
|
371
|
+
...statement,
|
|
372
|
+
predicates: predicates(statement.predicates),
|
|
373
|
+
...sources(statement.from)
|
|
374
|
+
};
|
|
353
375
|
}
|
|
354
376
|
if (statement.kind === "merge") {
|
|
355
377
|
return {
|
|
@@ -580,6 +602,9 @@ class MinnowDatabase {
|
|
|
580
602
|
#commitsSinceCompactionCheck = /* @__PURE__ */ new Map();
|
|
581
603
|
#compactionSteps = /* @__PURE__ */ new Map();
|
|
582
604
|
#writeChain = Promise.resolve();
|
|
605
|
+
#pendingCompactionPublications = /* @__PURE__ */ new Set();
|
|
606
|
+
#commitSignal;
|
|
607
|
+
#pendingEventLoopTurn;
|
|
583
608
|
#pendingWriteReservations = 0;
|
|
584
609
|
#activeReadReservations = 0;
|
|
585
610
|
#autoCollectionTask;
|
|
@@ -695,6 +720,10 @@ class MinnowDatabase {
|
|
|
695
720
|
this.#gzipVerdicts.clear();
|
|
696
721
|
this.#autoCompactionsRequested.clear();
|
|
697
722
|
this.#idleCompactionTableIds.clear();
|
|
723
|
+
for (const backoff of this.#autoCompactionBackoff.values()) {
|
|
724
|
+
if (backoff.retryTimer !== void 0)
|
|
725
|
+
clearTimeout(backoff.retryTimer);
|
|
726
|
+
}
|
|
698
727
|
this.#autoCompactionBackoff.clear();
|
|
699
728
|
this.#autoCompactionHints.clear();
|
|
700
729
|
this.#commitsSinceCompactionCheck.clear();
|
|
@@ -980,6 +1009,7 @@ class MinnowDatabase {
|
|
|
980
1009
|
}
|
|
981
1010
|
async #runWrite(run) {
|
|
982
1011
|
const restarting = async () => {
|
|
1012
|
+
await this.#publishPendingCompactions();
|
|
983
1013
|
await this.#assistAutomaticCollection();
|
|
984
1014
|
for (let attempt = 0; ; attempt += 1) {
|
|
985
1015
|
try {
|
|
@@ -1012,6 +1042,57 @@ class MinnowDatabase {
|
|
|
1012
1042
|
this.#writeChain = Promise.resolve();
|
|
1013
1043
|
}
|
|
1014
1044
|
}
|
|
1045
|
+
async #withCompactionPublicationSlot(attempt) {
|
|
1046
|
+
return new Promise((resolve, reject) => {
|
|
1047
|
+
let settled;
|
|
1048
|
+
const claim = () => {
|
|
1049
|
+
settled ??= (async () => {
|
|
1050
|
+
try {
|
|
1051
|
+
resolve(await attempt());
|
|
1052
|
+
} catch (error) {
|
|
1053
|
+
reject(error instanceof Error ? error : new Error(String(error)));
|
|
1054
|
+
} finally {
|
|
1055
|
+
this.#pendingCompactionPublications.delete(claim);
|
|
1056
|
+
}
|
|
1057
|
+
})();
|
|
1058
|
+
return settled;
|
|
1059
|
+
};
|
|
1060
|
+
this.#pendingCompactionPublications.add(claim);
|
|
1061
|
+
void this.#eventLoopTurn().then(claim);
|
|
1062
|
+
});
|
|
1063
|
+
}
|
|
1064
|
+
async #publishPendingCompactions() {
|
|
1065
|
+
while (this.#pendingCompactionPublications.size > 0) {
|
|
1066
|
+
const [claim] = this.#pendingCompactionPublications;
|
|
1067
|
+
if (claim === void 0)
|
|
1068
|
+
return;
|
|
1069
|
+
await claim();
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
#eventLoopTurn() {
|
|
1073
|
+
const pending = this.#pendingEventLoopTurn;
|
|
1074
|
+
if (pending !== void 0)
|
|
1075
|
+
return pending;
|
|
1076
|
+
const turn = yieldToEventLoop().then(() => {
|
|
1077
|
+
if (this.#pendingEventLoopTurn === turn)
|
|
1078
|
+
this.#pendingEventLoopTurn = void 0;
|
|
1079
|
+
});
|
|
1080
|
+
this.#pendingEventLoopTurn = turn;
|
|
1081
|
+
return turn;
|
|
1082
|
+
}
|
|
1083
|
+
#nextLocalCommit() {
|
|
1084
|
+
if (this.#commitSignal === void 0) {
|
|
1085
|
+
let resolve = () => void 0;
|
|
1086
|
+
const promise = new Promise((settle) => {
|
|
1087
|
+
resolve = settle;
|
|
1088
|
+
});
|
|
1089
|
+
this.#commitSignal = { promise, resolve };
|
|
1090
|
+
}
|
|
1091
|
+
return this.#commitSignal.promise;
|
|
1092
|
+
}
|
|
1093
|
+
async #yieldMaintenance() {
|
|
1094
|
+
await Promise.race([this.#eventLoopTurn(), this.#nextLocalCommit()]);
|
|
1095
|
+
}
|
|
1015
1096
|
async #withWriteReservation(run) {
|
|
1016
1097
|
if (this.#pendingWriteReservations >= MAX_DATABASE_PENDING_WRITES) {
|
|
1017
1098
|
throw new RangeError(`A database cannot queue more than ${String(MAX_DATABASE_PENDING_WRITES)} writes; await a write for backpressure`);
|
|
@@ -2492,7 +2573,7 @@ class MinnowDatabase {
|
|
|
2492
2573
|
catalogEpoch: after.catalogEpoch
|
|
2493
2574
|
};
|
|
2494
2575
|
}
|
|
2495
|
-
await
|
|
2576
|
+
await this.#yieldMaintenance();
|
|
2496
2577
|
}
|
|
2497
2578
|
}
|
|
2498
2579
|
async #acquireSharedLease(version) {
|
|
@@ -2667,7 +2748,15 @@ class MinnowDatabase {
|
|
|
2667
2748
|
}
|
|
2668
2749
|
memberSchema.forEach((column, index) => {
|
|
2669
2750
|
const expected = schema2?.[index];
|
|
2670
|
-
if (expected
|
|
2751
|
+
if (expected === void 0)
|
|
2752
|
+
return;
|
|
2753
|
+
if (column.type === expected.type && column.sqlDomain?.kind === "numeric" && expected.sqlDomain?.kind === "numeric") {
|
|
2754
|
+
if (JSON.stringify(column.sqlDomain) !== JSON.stringify(expected.sqlDomain)) {
|
|
2755
|
+
schema2 = schema2?.map((entry, position) => position === index ? { ...entry, sqlDomain: { kind: "numeric" } } : entry);
|
|
2756
|
+
}
|
|
2757
|
+
return;
|
|
2758
|
+
}
|
|
2759
|
+
if (column.type !== expected.type || JSON.stringify(column.sqlDomain ?? null) !== JSON.stringify(expected.sqlDomain ?? null)) {
|
|
2671
2760
|
throw new TypeError(`UNION member column types must match: ${expected.name}`);
|
|
2672
2761
|
}
|
|
2673
2762
|
});
|
|
@@ -2782,7 +2871,13 @@ class MinnowDatabase {
|
|
|
2782
2871
|
if (template !== null) {
|
|
2783
2872
|
const shape = resolvePointReadShape(template, options.params ?? []);
|
|
2784
2873
|
if (shape !== void 0) {
|
|
2785
|
-
|
|
2874
|
+
let point;
|
|
2875
|
+
try {
|
|
2876
|
+
point = await this.#pointReadResult(shape, options);
|
|
2877
|
+
} catch (error) {
|
|
2878
|
+
if (!(error instanceof UnknownTableError))
|
|
2879
|
+
throw error;
|
|
2880
|
+
}
|
|
2786
2881
|
throwIfAborted(options.signal);
|
|
2787
2882
|
if (point !== void 0)
|
|
2788
2883
|
return externalizeQueryResult(point);
|
|
@@ -2905,9 +3000,11 @@ class MinnowDatabase {
|
|
|
2905
3000
|
async #applyCatalogRewrites(plan, probe) {
|
|
2906
3001
|
const aliased = planHasSourceColumnAliases(plan);
|
|
2907
3002
|
const natural = planHasNaturalJoins(plan);
|
|
2908
|
-
const { views, domains, types, columns: catalogColumns } = await this.#catalogFacts(probe);
|
|
3003
|
+
const { views, domains, types, integers, keys, columns: catalogColumns } = await this.#catalogFacts(probe);
|
|
2909
3004
|
let rewritten = plan.usesSequenceCalls === true ? await this.#resolveSequenceCalls(plan) : plan;
|
|
2910
|
-
|
|
3005
|
+
rewritten = foldIdentifierCase(rewritten, catalogColumns);
|
|
3006
|
+
rewritten = expandRowReferences(rewritten, (name) => catalogColumns.get(name));
|
|
3007
|
+
if (views.size > 0 && planReadsViews(rewritten, (name) => views.has(name))) {
|
|
2911
3008
|
const bodies = /* @__PURE__ */ new Map();
|
|
2912
3009
|
rewritten = expandViewSources(rewritten, (name) => {
|
|
2913
3010
|
const sql = views.get(name);
|
|
@@ -2924,6 +3021,13 @@ class MinnowDatabase {
|
|
|
2924
3021
|
if (planReadsTable(rewritten, (name) => domains.has(name) || types.has(name))) {
|
|
2925
3022
|
rewritten = normalizePlanDomainLiterals(rewritten, domains, types);
|
|
2926
3023
|
}
|
|
3024
|
+
if (integers.size > 0 && planMayHaveIntegerDivision(rewritten)) {
|
|
3025
|
+
rewritten = clonePlanTree(rewritten);
|
|
3026
|
+
annotatePlanIntegerDivision(rewritten, (name) => integers.get(name));
|
|
3027
|
+
}
|
|
3028
|
+
if (keys.size > 0) {
|
|
3029
|
+
rewritten = extendGroupByWithKeyDependents(rewritten, (name) => keys.get(name), (name) => catalogColumns.get(name));
|
|
3030
|
+
}
|
|
2927
3031
|
const columnsOf = (name) => {
|
|
2928
3032
|
const columns = catalogColumns.get(name);
|
|
2929
3033
|
if (columns === void 0)
|
|
@@ -3034,6 +3138,14 @@ class MinnowDatabase {
|
|
|
3034
3138
|
item.expression = await rewrite(item.expression);
|
|
3035
3139
|
return resolved;
|
|
3036
3140
|
}
|
|
3141
|
+
async #foldTableName(name) {
|
|
3142
|
+
const { columns } = await this.#catalogFacts();
|
|
3143
|
+
if (columns.has(name))
|
|
3144
|
+
return name;
|
|
3145
|
+
const lowered = name.toLowerCase();
|
|
3146
|
+
const matches = [...columns.keys()].filter((candidate) => candidate.toLowerCase() === lowered);
|
|
3147
|
+
return matches.length === 1 ? matches[0] ?? name : name;
|
|
3148
|
+
}
|
|
3037
3149
|
async #catalogFacts(probe) {
|
|
3038
3150
|
probe ??= await this.store.getCatalogProbe();
|
|
3039
3151
|
const epoch = probe.catalogEpoch;
|
|
@@ -3045,10 +3157,23 @@ class MinnowDatabase {
|
|
|
3045
3157
|
const domains = /* @__PURE__ */ new Map();
|
|
3046
3158
|
const columns = /* @__PURE__ */ new Map();
|
|
3047
3159
|
const types = /* @__PURE__ */ new Map();
|
|
3160
|
+
const integers = /* @__PURE__ */ new Map();
|
|
3161
|
+
const keys = /* @__PURE__ */ new Map();
|
|
3048
3162
|
for (const table of await this.store.listTables()) {
|
|
3049
3163
|
const visible = table.columns.filter(({ hidden }) => hidden !== true);
|
|
3164
|
+
const keyColumn = getUniqueKeyColumn(table);
|
|
3165
|
+
const keyNames = keyColumn !== void 0 && !keyColumn.hidden ? [keyColumn.name] : (table.primaryKeyColumnIds ?? []).flatMap((columnId) => {
|
|
3166
|
+
const column = table.columns.find((candidate) => candidate.id === columnId);
|
|
3167
|
+
return column === void 0 ? [] : [column.name];
|
|
3168
|
+
});
|
|
3169
|
+
if (keyNames.length > 0)
|
|
3170
|
+
keys.set(table.name, keyNames);
|
|
3050
3171
|
columns.set(table.name, visible.map(({ name }) => name));
|
|
3051
3172
|
types.set(table.name, new Map(visible.map((column) => [column.name, column.type])));
|
|
3173
|
+
const integerColumns = visible.filter((column) => column.integer === true);
|
|
3174
|
+
if (integerColumns.length > 0) {
|
|
3175
|
+
integers.set(table.name, new Set(integerColumns.map(({ name }) => name)));
|
|
3176
|
+
}
|
|
3052
3177
|
const tableDomains = new Map(table.columns.flatMap((column) => column.sqlDomain === void 0 ? [] : [[column.name, column.sqlDomain]]));
|
|
3053
3178
|
if (tableDomains.size > 0)
|
|
3054
3179
|
domains.set(table.name, tableDomains);
|
|
@@ -3062,7 +3187,7 @@ class MinnowDatabase {
|
|
|
3062
3187
|
existing.push({ table, key });
|
|
3063
3188
|
}
|
|
3064
3189
|
}
|
|
3065
|
-
const facts = { views, childKeys, domains, columns, types };
|
|
3190
|
+
const facts = { views, childKeys, domains, columns, types, integers, keys };
|
|
3066
3191
|
this.#catalogCache = { epoch, facts };
|
|
3067
3192
|
return facts;
|
|
3068
3193
|
}
|
|
@@ -3558,7 +3683,7 @@ class MinnowDatabase {
|
|
|
3558
3683
|
if (page.nextCursor === null)
|
|
3559
3684
|
break;
|
|
3560
3685
|
cursor = page.nextCursor;
|
|
3561
|
-
await
|
|
3686
|
+
await this.#yieldMaintenance();
|
|
3562
3687
|
}
|
|
3563
3688
|
return result;
|
|
3564
3689
|
}
|
|
@@ -3583,7 +3708,7 @@ class MinnowDatabase {
|
|
|
3583
3708
|
if (page.nextCursor === null)
|
|
3584
3709
|
break;
|
|
3585
3710
|
cursor = page.nextCursor;
|
|
3586
|
-
await
|
|
3711
|
+
await this.#yieldMaintenance();
|
|
3587
3712
|
}
|
|
3588
3713
|
return result;
|
|
3589
3714
|
}
|
|
@@ -3793,8 +3918,11 @@ class MinnowDatabase {
|
|
|
3793
3918
|
return;
|
|
3794
3919
|
if (!autoCompactionDueHint(hint))
|
|
3795
3920
|
return;
|
|
3796
|
-
|
|
3797
|
-
|
|
3921
|
+
const backoff = this.#autoCompactionBackoff.get(table.id);
|
|
3922
|
+
if (backoff !== void 0) {
|
|
3923
|
+
if (backoff.retryTimer !== void 0 || hint.visible < backoff.minimumSegments)
|
|
3924
|
+
return;
|
|
3925
|
+
}
|
|
3798
3926
|
if (this.#autoCompactionsInFlight.has(table.id)) {
|
|
3799
3927
|
this.#autoCompactionsRequested.add(table.id);
|
|
3800
3928
|
return;
|
|
@@ -3802,22 +3930,43 @@ class MinnowDatabase {
|
|
|
3802
3930
|
const run = this.#runAutoCompaction(table).then((folded) => {
|
|
3803
3931
|
if (folded)
|
|
3804
3932
|
this.#autoCompactionBackoff.delete(table.id);
|
|
3805
|
-
else
|
|
3806
|
-
this.#
|
|
3807
|
-
}
|
|
3933
|
+
else
|
|
3934
|
+
this.#backOffAutoCompaction(table.id, hint.visible);
|
|
3808
3935
|
}).catch(() => {
|
|
3809
|
-
this.#
|
|
3936
|
+
this.#backOffAutoCompaction(table.id, hint.visible);
|
|
3810
3937
|
}).finally(() => {
|
|
3811
3938
|
if (this.#autoCompactionsInFlight.get(table.id) === run) {
|
|
3812
3939
|
this.#autoCompactionsInFlight.delete(table.id);
|
|
3813
3940
|
}
|
|
3814
3941
|
if (this.#autoCompactionsRequested.delete(table.id)) {
|
|
3815
|
-
void
|
|
3942
|
+
void this.#yieldMaintenance().then(() => this.#checkAutoCompaction(table.id));
|
|
3816
3943
|
}
|
|
3817
3944
|
});
|
|
3818
3945
|
this.#autoCompactionsInFlight.set(table.id, run);
|
|
3819
3946
|
void run;
|
|
3820
3947
|
}
|
|
3948
|
+
#backOffAutoCompaction(tableId, visible) {
|
|
3949
|
+
if (this.#closed)
|
|
3950
|
+
return;
|
|
3951
|
+
const previous = this.#autoCompactionBackoff.get(tableId);
|
|
3952
|
+
if (previous?.retryTimer !== void 0)
|
|
3953
|
+
clearTimeout(previous.retryTimer);
|
|
3954
|
+
const failures = (previous?.failures ?? 0) + 1;
|
|
3955
|
+
const delay = Math.min(AUTO_COMPACT_RETRY_MAX_MS, AUTO_COMPACT_RETRY_MIN_MS * 2 ** Math.min(16, failures - 1));
|
|
3956
|
+
const timer = setTimeout(() => {
|
|
3957
|
+
const current = this.#autoCompactionBackoff.get(tableId);
|
|
3958
|
+
if (current?.retryTimer === timer) {
|
|
3959
|
+
this.#autoCompactionBackoff.set(tableId, { ...current, retryTimer: void 0 });
|
|
3960
|
+
}
|
|
3961
|
+
void this.#checkAutoCompaction(tableId);
|
|
3962
|
+
}, delay);
|
|
3963
|
+
timer.unref?.();
|
|
3964
|
+
this.#autoCompactionBackoff.set(tableId, {
|
|
3965
|
+
minimumSegments: Math.min(AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS, Math.max(2, visible * 2)),
|
|
3966
|
+
failures,
|
|
3967
|
+
retryTimer: timer
|
|
3968
|
+
});
|
|
3969
|
+
}
|
|
3821
3970
|
async #assertCompactionCapacity(table, transaction) {
|
|
3822
3971
|
transaction?.limitLevelZeroSegments(table.id, MAX_LEVEL_ZERO_SEGMENTS);
|
|
3823
3972
|
if (!this.#autoCompact)
|
|
@@ -3834,6 +3983,7 @@ class MinnowDatabase {
|
|
|
3834
3983
|
}
|
|
3835
3984
|
if (levelZero < MAX_LEVEL_ZERO_SEGMENTS)
|
|
3836
3985
|
return;
|
|
3986
|
+
await this.#publishPendingCompactions();
|
|
3837
3987
|
await this.compactTableStep(table.name, {
|
|
3838
3988
|
maxBlocks: AUTO_COMPACT_STEP_BLOCKS,
|
|
3839
3989
|
maxLevel0Segments: AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS
|
|
@@ -3859,14 +4009,14 @@ class MinnowDatabase {
|
|
|
3859
4009
|
while (progress.result === null) {
|
|
3860
4010
|
if (progress.jobId === null)
|
|
3861
4011
|
throw new Error("Compaction progress lost its job ID");
|
|
3862
|
-
await
|
|
4012
|
+
await this.#yieldMaintenance();
|
|
3863
4013
|
progress = await this.resumeCompactionJob(progress.jobId, options);
|
|
3864
4014
|
}
|
|
3865
4015
|
if (!progress.result.compacted)
|
|
3866
4016
|
return folded;
|
|
3867
4017
|
folded = true;
|
|
3868
4018
|
this.#maybeScheduleAutoCollection();
|
|
3869
|
-
await
|
|
4019
|
+
await this.#yieldMaintenance();
|
|
3870
4020
|
const current = await this.store.getTable(table.id);
|
|
3871
4021
|
if (current === void 0 || !autoCompactionDue(await this.#currentVisibleSegments(current))) {
|
|
3872
4022
|
return folded;
|
|
@@ -3887,6 +4037,9 @@ class MinnowDatabase {
|
|
|
3887
4037
|
return segments;
|
|
3888
4038
|
}
|
|
3889
4039
|
#afterCommit(manifest, transaction) {
|
|
4040
|
+
const signal = this.#commitSignal;
|
|
4041
|
+
this.#commitSignal = void 0;
|
|
4042
|
+
signal?.resolve();
|
|
3890
4043
|
if (this.#closed)
|
|
3891
4044
|
return;
|
|
3892
4045
|
const contribution = transaction?.committedCatalogContribution;
|
|
@@ -4265,6 +4418,7 @@ class MinnowDatabase {
|
|
|
4265
4418
|
const statementNow = this.#now();
|
|
4266
4419
|
const materialized = await this.#materializeInsertExpressions({ ...compiled, rows: derivedRows }, statementNow);
|
|
4267
4420
|
const input = insertStatementBatch(materialized);
|
|
4421
|
+
coerceSqlWriteBatch(target, input);
|
|
4268
4422
|
const filled = await fillColumnDefaults(target, input, (sql) => this.#evaluateDefaultExpression(sql, statementNow), derivedRows.length);
|
|
4269
4423
|
normalizeDomainBatch(target, filled.batch);
|
|
4270
4424
|
fillStoredGeneratedColumns(target, filled.batch, derivedRows.length, filled.generated);
|
|
@@ -4293,6 +4447,9 @@ class MinnowDatabase {
|
|
|
4293
4447
|
const bound = bindStatementParameters(compiled, params);
|
|
4294
4448
|
if (bound.kind !== "update" && bound.kind !== "delete")
|
|
4295
4449
|
continue;
|
|
4450
|
+
if (bound.from !== void 0) {
|
|
4451
|
+
throw new TypeError(`Trigger body ${bound.kind.toUpperCase()}s cannot join other sources`);
|
|
4452
|
+
}
|
|
4296
4453
|
const referenced = /* @__PURE__ */ new Set([keyColumn.name]);
|
|
4297
4454
|
if (bound.kind === "update") {
|
|
4298
4455
|
for (const assignment of bound.assignments) {
|
|
@@ -4417,6 +4574,7 @@ class MinnowDatabase {
|
|
|
4417
4574
|
return this.#openWriteScope((session) => action(session));
|
|
4418
4575
|
}
|
|
4419
4576
|
async #openWriteScope(action, options = {}) {
|
|
4577
|
+
await this.#publishPendingCompactions();
|
|
4420
4578
|
await this.#assistAutomaticCollection();
|
|
4421
4579
|
const transaction = await this.#transactions.beginDeferred({
|
|
4422
4580
|
durableSnapshot: options.durableSnapshot ?? true
|
|
@@ -4458,6 +4616,9 @@ class MinnowDatabase {
|
|
|
4458
4616
|
}));
|
|
4459
4617
|
}
|
|
4460
4618
|
const statement = bindStatementParameters(compiled, params);
|
|
4619
|
+
if (statement.kind === "set" || statement.kind === "show") {
|
|
4620
|
+
return externalizeExecuteResult(await this.runStatement(statement));
|
|
4621
|
+
}
|
|
4461
4622
|
if (!isTransactionalStatement(statement)) {
|
|
4462
4623
|
throw new TypeError(`${statement.kind.toUpperCase().replace("-", " ")} is not allowed inside a write scope`);
|
|
4463
4624
|
}
|
|
@@ -5371,6 +5532,9 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
5371
5532
|
})
|
|
5372
5533
|
};
|
|
5373
5534
|
}
|
|
5535
|
+
if (statement.kind === "set" || statement.kind === "show") {
|
|
5536
|
+
return externalizeExecuteResult(await this.runStatement(statement));
|
|
5537
|
+
}
|
|
5374
5538
|
const open = this.#openTransaction;
|
|
5375
5539
|
if (open !== void 0) {
|
|
5376
5540
|
const boundStatement2 = bindStatementParameters(statement, params);
|
|
@@ -5692,12 +5856,14 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
5692
5856
|
if (keyColumn === void 0 || conflictColumns.length !== addressColumns.length || conflictColumns.some((name, index) => name !== addressColumns[index])) {
|
|
5693
5857
|
throw new TypeError(`ON CONFLICT targets the table's primary or unique key columns: ${addressColumns.join(", ") || "(none)"}`);
|
|
5694
5858
|
}
|
|
5859
|
+
statement = foldInsertColumns(table, statement);
|
|
5695
5860
|
for (const name of statement.columns) {
|
|
5696
5861
|
if (!table.columns.some((column) => column.name === name)) {
|
|
5697
5862
|
throw new TypeError(`INSERT column does not exist: ${name}`);
|
|
5698
5863
|
}
|
|
5699
5864
|
}
|
|
5700
5865
|
const proposedBatch = insertStatementBatch(statement);
|
|
5866
|
+
coerceSqlWriteBatch(table, proposedBatch);
|
|
5701
5867
|
const statementNow = this.#now();
|
|
5702
5868
|
const filledProposed = await fillColumnDefaults(table, proposedBatch, (sql) => this.#evaluateDefaultExpression(sql, statementNow), statement.rows.length);
|
|
5703
5869
|
normalizeDomainBatch(table, filledProposed.batch);
|
|
@@ -5893,6 +6059,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
5893
6059
|
throw new TypeError(`ON CONFLICT targets the table's primary or unique key columns: ${addressColumns.join(", ") || "(none)"}`);
|
|
5894
6060
|
}
|
|
5895
6061
|
const proposed = insertStatementBatch(statement);
|
|
6062
|
+
coerceSqlWriteBatch(table, proposed);
|
|
5896
6063
|
const statementNow = this.#now();
|
|
5897
6064
|
const filled = await fillColumnDefaults(table, proposed, (sql) => this.#evaluateDefaultExpression(sql, statementNow), statement.rows.length);
|
|
5898
6065
|
normalizeDomainBatch(table, filled.batch);
|
|
@@ -6065,6 +6232,38 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
6065
6232
|
if (statement.kind === "select") {
|
|
6066
6233
|
return { kind: "rows", result: await this.query(statement.sql) };
|
|
6067
6234
|
}
|
|
6235
|
+
if (statement.kind === "set") {
|
|
6236
|
+
return { kind: "set", action: statement.action, name: statement.name };
|
|
6237
|
+
}
|
|
6238
|
+
if (statement.kind === "show") {
|
|
6239
|
+
const settings = {
|
|
6240
|
+
search_path: "public",
|
|
6241
|
+
server_version: "16.0",
|
|
6242
|
+
server_version_num: "160000",
|
|
6243
|
+
timezone: "UTC",
|
|
6244
|
+
time_zone: "UTC",
|
|
6245
|
+
transaction_isolation: "repeatable read",
|
|
6246
|
+
default_transaction_isolation: "repeatable read",
|
|
6247
|
+
client_encoding: "UTF8",
|
|
6248
|
+
server_encoding: "UTF8",
|
|
6249
|
+
standard_conforming_strings: "on",
|
|
6250
|
+
integer_datetimes: "on",
|
|
6251
|
+
datestyle: "ISO, MDY",
|
|
6252
|
+
intervalstyle: "postgres",
|
|
6253
|
+
application_name: "",
|
|
6254
|
+
is_superuser: "on",
|
|
6255
|
+
max_identifier_length: "63"
|
|
6256
|
+
};
|
|
6257
|
+
const name = statement.name.toLowerCase();
|
|
6258
|
+
const value = settings[name];
|
|
6259
|
+
if (value === void 0) {
|
|
6260
|
+
throw new TypeError(`unrecognized configuration parameter "${statement.name}"`);
|
|
6261
|
+
}
|
|
6262
|
+
return {
|
|
6263
|
+
kind: "rows",
|
|
6264
|
+
result: await this.query(`SELECT $1 AS ${quoteIdentifier(name)}`, { params: [value] })
|
|
6265
|
+
};
|
|
6266
|
+
}
|
|
6068
6267
|
if (statement.kind === "create-enum") {
|
|
6069
6268
|
const values = validateEnumValues(statement.values, statement.name);
|
|
6070
6269
|
const storageName = `${ENUM_TYPE_PREFIX}${statement.name}`;
|
|
@@ -6496,12 +6695,14 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
6496
6695
|
throw new TypeError(`ON CONFLICT targets the table's primary or unique key columns: ${addressColumns.join(", ") || "(none)"}`);
|
|
6497
6696
|
}
|
|
6498
6697
|
}
|
|
6499
|
-
|
|
6698
|
+
const spelledColumns = foldInsertColumns(table2, statement).columns;
|
|
6699
|
+
for (const name of spelledColumns) {
|
|
6500
6700
|
if (!table2.columns.some((column) => column.name === name)) {
|
|
6501
6701
|
throw new TypeError(`INSERT column does not exist: ${name}`);
|
|
6502
6702
|
}
|
|
6503
6703
|
}
|
|
6504
|
-
const input = insertStatementBatch(statement);
|
|
6704
|
+
const input = insertStatementBatch({ ...statement, columns: spelledColumns });
|
|
6705
|
+
coerceSqlWriteBatch(table2, input);
|
|
6505
6706
|
normalizeDomainBatch(table2, input);
|
|
6506
6707
|
let returningColumns2;
|
|
6507
6708
|
if (options.returning !== void 0) {
|
|
@@ -6515,7 +6716,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
6515
6716
|
const writer = options.writer;
|
|
6516
6717
|
const stagedKeyColumn = writer === void 0 ? void 0 : getUniqueKeyColumn(table2);
|
|
6517
6718
|
if (writer !== void 0 && !viaUpsert && stagedKeyColumn !== void 0) {
|
|
6518
|
-
const assignedGenerated = table2.columns.find((column) => column.generatedValue !== void 0 &&
|
|
6719
|
+
const assignedGenerated = table2.columns.find((column) => column.generatedValue !== void 0 && spelledColumns.includes(column.name));
|
|
6519
6720
|
if (assignedGenerated !== void 0) {
|
|
6520
6721
|
throw new TypeError(`Generated column cannot be assigned: ${assignedGenerated.name}`);
|
|
6521
6722
|
}
|
|
@@ -6531,7 +6732,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
6531
6732
|
...returningColumns2 === void 0 ? {} : {
|
|
6532
6733
|
...returningExecuteFields(table2, returningColumns2, statement.rows.map((row, rowIndex) => Object.fromEntries(returningColumns2.map((name) => {
|
|
6533
6734
|
const column = table2.columns.find((candidate) => candidate.name === name);
|
|
6534
|
-
const position =
|
|
6735
|
+
const position = spelledColumns.indexOf(name);
|
|
6535
6736
|
const inserted = position < 0 ? void 0 : row[position];
|
|
6536
6737
|
const value = generated[name]?.[rowIndex] ?? input.columns[name]?.[rowIndex] ?? (inserted === void 0 || isDefaultInsertValue(inserted) ? null : boundInsertValue(inserted));
|
|
6537
6738
|
return [name, executionSqlValueFromInput(column, value)];
|
|
@@ -6546,8 +6747,9 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
6546
6747
|
}
|
|
6547
6748
|
const returningColumns = options.returning === void 0 ? void 0 : options.returning === "*" ? visibleTableColumns(table).map(({ name }) => name) : [...options.returning];
|
|
6548
6749
|
const updateAssignments = statement.kind !== "update" ? [] : await (async () => {
|
|
6750
|
+
const spelled = foldAssignmentColumns(table, statement.assignments);
|
|
6549
6751
|
if (!table.columns.some((column) => column.sqlDomain !== void 0)) {
|
|
6550
|
-
return
|
|
6752
|
+
return spelled;
|
|
6551
6753
|
}
|
|
6552
6754
|
const { domains } = await this.#catalogFacts();
|
|
6553
6755
|
if (!domains.has(table.name))
|
|
@@ -6556,7 +6758,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
6556
6758
|
sql: `UPDATE ${table.name}`,
|
|
6557
6759
|
base: { table: table.name, alias: table.name },
|
|
6558
6760
|
joins: [],
|
|
6559
|
-
select:
|
|
6761
|
+
select: spelled.map((assignment) => ({
|
|
6560
6762
|
expression: assignment.expression,
|
|
6561
6763
|
alias: assignment.column
|
|
6562
6764
|
})),
|
|
@@ -6565,20 +6767,25 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
6565
6767
|
having: [],
|
|
6566
6768
|
orderBy: []
|
|
6567
6769
|
}, domains);
|
|
6568
|
-
return
|
|
6770
|
+
return spelled.map((assignment, index) => ({
|
|
6569
6771
|
...assignment,
|
|
6570
6772
|
expression: normalized.select[index]?.expression ?? assignment.expression
|
|
6571
6773
|
}));
|
|
6572
6774
|
})();
|
|
6573
6775
|
const referenced = /* @__PURE__ */ new Set([keyColumn.name, ...returningColumns ?? []]);
|
|
6574
6776
|
const assignmentAlias = (column) => `\0set:${column}`;
|
|
6777
|
+
const targetAlias = statement.alias ?? table.name;
|
|
6778
|
+
const from = statement.from;
|
|
6575
6779
|
const plan = optimizePlan({
|
|
6576
6780
|
sql: `(${statement.kind})`,
|
|
6577
|
-
base: { table: table.name, alias:
|
|
6578
|
-
joins: [],
|
|
6781
|
+
base: { table: table.name, alias: targetAlias },
|
|
6782
|
+
joins: from === void 0 ? [] : [crossJoinPlan(from.base), ...from.joins],
|
|
6579
6783
|
select: [
|
|
6580
6784
|
...[...referenced].map((name) => ({
|
|
6581
|
-
expression: {
|
|
6785
|
+
expression: {
|
|
6786
|
+
kind: "column",
|
|
6787
|
+
reference: from === void 0 ? name : `${targetAlias}.${name}`
|
|
6788
|
+
},
|
|
6582
6789
|
alias: name
|
|
6583
6790
|
})),
|
|
6584
6791
|
...statement.kind === "update" ? updateAssignments.map((assignment) => ({
|
|
@@ -6586,12 +6793,12 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
6586
6793
|
alias: assignmentAlias(assignment.column)
|
|
6587
6794
|
})) : []
|
|
6588
6795
|
],
|
|
6589
|
-
predicates: statement.predicates,
|
|
6796
|
+
predicates: from === void 0 ? statement.predicates : [...statement.predicates, ...from.predicates],
|
|
6590
6797
|
groupBy: [],
|
|
6591
6798
|
having: [],
|
|
6592
6799
|
orderBy: []
|
|
6593
6800
|
});
|
|
6594
|
-
if (statement.kind === "delete" && returningColumns === void 0) {
|
|
6801
|
+
if (statement.kind === "delete" && returningColumns === void 0 && from === void 0) {
|
|
6595
6802
|
const internalWriter = options.writer;
|
|
6596
6803
|
if (internalWriter.queryFirstColumn !== void 0) {
|
|
6597
6804
|
const selectedKeys = await internalWriter.queryFirstColumn(plan);
|
|
@@ -6623,6 +6830,16 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
6623
6830
|
} else {
|
|
6624
6831
|
rows = (await this.#queryCompiled(plan)).rows;
|
|
6625
6832
|
}
|
|
6833
|
+
if (from !== void 0) {
|
|
6834
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6835
|
+
rows = rows.filter((row) => {
|
|
6836
|
+
const key = JSON.stringify(row[keyColumn.name] ?? null);
|
|
6837
|
+
if (seen.has(key))
|
|
6838
|
+
return false;
|
|
6839
|
+
seen.add(key);
|
|
6840
|
+
return true;
|
|
6841
|
+
});
|
|
6842
|
+
}
|
|
6626
6843
|
const keys = keyColumn.type === "string" && keyColumn.sqlDomain === void 0 ? rows.map((row) => storedSqlValueFromExecution(keyColumn, row[keyColumn.name] ?? null)) : rows.map((row) => row[keyColumn.name] ?? null);
|
|
6627
6844
|
if (keys.some((key) => key === null)) {
|
|
6628
6845
|
throw new TypeError(`Unique key values must not be null: ${keyColumn.name}`);
|
|
@@ -6666,8 +6883,9 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
6666
6883
|
if (typeof value === "number" && !Number.isFinite(value)) {
|
|
6667
6884
|
throw new TypeError(`UPDATE assignment produced a non-finite number: ${assignment.column}`);
|
|
6668
6885
|
}
|
|
6669
|
-
|
|
6670
|
-
|
|
6886
|
+
const stored = storedSqlValueFromExecution(column, value);
|
|
6887
|
+
executionValues?.push(column.type === "string" && column.sqlDomain === void 0 ? value : stored);
|
|
6888
|
+
return stored;
|
|
6671
6889
|
});
|
|
6672
6890
|
if (executionValues !== void 0 && returnedChanges !== void 0) {
|
|
6673
6891
|
returnedChanges[assignment.column] = executionValues;
|
|
@@ -8051,7 +8269,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
8051
8269
|
return active;
|
|
8052
8270
|
cursor = page.nextCursor;
|
|
8053
8271
|
if (cursor !== null)
|
|
8054
|
-
await
|
|
8272
|
+
await this.#yieldMaintenance();
|
|
8055
8273
|
} while (cursor !== null);
|
|
8056
8274
|
return void 0;
|
|
8057
8275
|
}
|
|
@@ -8089,11 +8307,15 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
8089
8307
|
}
|
|
8090
8308
|
cursor = page.nextCursor;
|
|
8091
8309
|
if (cursor !== null)
|
|
8092
|
-
await
|
|
8310
|
+
await this.#yieldMaintenance();
|
|
8093
8311
|
} while (cursor !== null);
|
|
8094
8312
|
}
|
|
8095
8313
|
async collectGarbage(options = {}) {
|
|
8096
8314
|
const maxItems = boundedMaintenanceBatchItems(options.maxItemsPerStep ?? 64, "Garbage collection items per step");
|
|
8315
|
+
let reconciliationCursor = null;
|
|
8316
|
+
do {
|
|
8317
|
+
reconciliationCursor = (await this.#reconcileCompactionPage(reconciliationCursor)).nextCursor;
|
|
8318
|
+
} while (reconciliationCursor !== null);
|
|
8097
8319
|
let progress = await this.collectGarbageStep({
|
|
8098
8320
|
maxItems,
|
|
8099
8321
|
...options.maxPlanningItems === void 0 ? {} : { maxPlanningItems: options.maxPlanningItems },
|
|
@@ -8130,7 +8352,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
8130
8352
|
return active;
|
|
8131
8353
|
cursor = page.nextCursor;
|
|
8132
8354
|
if (cursor !== null)
|
|
8133
|
-
await
|
|
8355
|
+
await this.#yieldMaintenance();
|
|
8134
8356
|
} while (cursor !== null);
|
|
8135
8357
|
return void 0;
|
|
8136
8358
|
}
|
|
@@ -8171,14 +8393,14 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
8171
8393
|
this.#autoCollectionRequested = false;
|
|
8172
8394
|
this.#commitsSinceCollection = 0;
|
|
8173
8395
|
this.#lastCollectionAt = dateMilliseconds(this.#now());
|
|
8174
|
-
const run = this.#runAutoCollection().then((moreWork) => {
|
|
8396
|
+
const run = this.#runAutoCollection().then(({ moreWork, reclaimed }) => {
|
|
8175
8397
|
this.#autoCollectionConsecutiveFailures = 0;
|
|
8176
8398
|
this.#autoCollectionLastError = void 0;
|
|
8177
8399
|
this.#lastCollectionCompletedAt = dateMilliseconds(this.#now());
|
|
8178
8400
|
this.#autoCollectionRetryAt = void 0;
|
|
8179
8401
|
if (moreWork)
|
|
8180
8402
|
this.#autoCollectionRequested = true;
|
|
8181
|
-
|
|
8403
|
+
if (reclaimed || !moreWork)
|
|
8182
8404
|
this.#autoCollectionDebtCommits = 0;
|
|
8183
8405
|
}).catch((error) => {
|
|
8184
8406
|
this.#autoCollectionConsecutiveFailures += 1;
|
|
@@ -8194,7 +8416,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
8194
8416
|
this.#autoCollectionTask = void 0;
|
|
8195
8417
|
this.#autoCollectionInFlight = false;
|
|
8196
8418
|
if (this.#autoCollectionRequested) {
|
|
8197
|
-
void
|
|
8419
|
+
void this.#yieldMaintenance().then(() => {
|
|
8198
8420
|
if (this.#closed) {
|
|
8199
8421
|
this.#autoCollectionRequested = false;
|
|
8200
8422
|
return;
|
|
@@ -8239,6 +8461,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
8239
8461
|
async #runAutoCollection() {
|
|
8240
8462
|
const reconciliationMoreWork = await this.#reconcileCompactionMaintenance();
|
|
8241
8463
|
let moreWork = false;
|
|
8464
|
+
let reclaimed = false;
|
|
8242
8465
|
for (let pass = 0; pass < AUTO_COLLECT_MAX_PASSES; pass += 1) {
|
|
8243
8466
|
if (this.#closed)
|
|
8244
8467
|
break;
|
|
@@ -8249,7 +8472,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
8249
8472
|
retainRecentVersions: AUTO_COLLECT_RETAINED_VERSIONS
|
|
8250
8473
|
}, AUTO_COLLECT_RETAINED_VERSION_MS);
|
|
8251
8474
|
while (progress.result === null) {
|
|
8252
|
-
await
|
|
8475
|
+
await this.#yieldMaintenance();
|
|
8253
8476
|
progress = await this.resumeGarbageCollectionJob(progress.jobId, {
|
|
8254
8477
|
maxItems: AUTO_COLLECT_STEP_ITEMS
|
|
8255
8478
|
});
|
|
@@ -8259,14 +8482,19 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
8259
8482
|
if (!moreWork) {
|
|
8260
8483
|
break;
|
|
8261
8484
|
}
|
|
8262
|
-
|
|
8485
|
+
reclaimed = true;
|
|
8486
|
+
await this.#yieldMaintenance();
|
|
8263
8487
|
}
|
|
8264
8488
|
await this.#pruneFinishedJobRecords();
|
|
8265
|
-
return moreWork || reconciliationMoreWork;
|
|
8489
|
+
return { moreWork: moreWork || reconciliationMoreWork, reclaimed };
|
|
8266
8490
|
}
|
|
8267
8491
|
async #reconcileCompactionMaintenance() {
|
|
8268
|
-
const
|
|
8269
|
-
this.#compactionReconciliationCursor =
|
|
8492
|
+
const { nextCursor, moreWork } = await this.#reconcileCompactionPage(this.#compactionReconciliationCursor);
|
|
8493
|
+
this.#compactionReconciliationCursor = nextCursor;
|
|
8494
|
+
return moreWork;
|
|
8495
|
+
}
|
|
8496
|
+
async #reconcileCompactionPage(cursor) {
|
|
8497
|
+
const page = await this.store.listCompactionJobPage(cursor, AUTO_COMPACTION_RECONCILE_RECORDS);
|
|
8270
8498
|
let moreWork = page.nextCursor !== null;
|
|
8271
8499
|
const cutoff = dateIsoString(this.#now());
|
|
8272
8500
|
for (const listed of page.records) {
|
|
@@ -8310,14 +8538,22 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
8310
8538
|
this.#maybeScheduleAutoCompaction(table, await this.#currentVisibleSegments(table));
|
|
8311
8539
|
continue;
|
|
8312
8540
|
}
|
|
8313
|
-
|
|
8314
|
-
|
|
8315
|
-
|
|
8316
|
-
|
|
8541
|
+
try {
|
|
8542
|
+
const progress = await this.resumeCompactionJob(job.id, { maxBlocks: 1 });
|
|
8543
|
+
if (progress.result !== null) {
|
|
8544
|
+
job = await this.store.getCompactionJob(job.id) ?? job;
|
|
8545
|
+
if (!isActiveCompactionState(job.state))
|
|
8546
|
+
continue;
|
|
8547
|
+
}
|
|
8548
|
+
} catch (error) {
|
|
8549
|
+
if (error instanceof CompactionJobCancelledError)
|
|
8317
8550
|
continue;
|
|
8551
|
+
if (this.#closed)
|
|
8552
|
+
throw error;
|
|
8553
|
+
this.#maybeScheduleAutoCompaction(table, await this.#currentVisibleSegments(table));
|
|
8318
8554
|
}
|
|
8319
8555
|
}
|
|
8320
|
-
return moreWork;
|
|
8556
|
+
return { nextCursor: page.nextCursor, moreWork };
|
|
8321
8557
|
}
|
|
8322
8558
|
async #pruneFinishedJobRecords() {
|
|
8323
8559
|
const newestFirst = (left, right) => right.updatedAt.localeCompare(left.updatedAt) || right.id.localeCompare(left.id);
|
|
@@ -8338,7 +8574,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
8338
8574
|
}
|
|
8339
8575
|
cursor = page.nextCursor;
|
|
8340
8576
|
if (cursor !== null)
|
|
8341
|
-
await
|
|
8577
|
+
await this.#yieldMaintenance();
|
|
8342
8578
|
} while (cursor !== null);
|
|
8343
8579
|
const retainedCompactionIds = new Set(newestCompactions.map((job) => job.id));
|
|
8344
8580
|
cursor = null;
|
|
@@ -8388,7 +8624,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
8388
8624
|
}
|
|
8389
8625
|
cursor = page.nextCursor;
|
|
8390
8626
|
if (cursor !== null)
|
|
8391
|
-
await
|
|
8627
|
+
await this.#yieldMaintenance();
|
|
8392
8628
|
} while (cursor !== null);
|
|
8393
8629
|
const newestCollections = [];
|
|
8394
8630
|
cursor = null;
|
|
@@ -8400,7 +8636,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
8400
8636
|
}
|
|
8401
8637
|
cursor = page.nextCursor;
|
|
8402
8638
|
if (cursor !== null)
|
|
8403
|
-
await
|
|
8639
|
+
await this.#yieldMaintenance();
|
|
8404
8640
|
} while (cursor !== null);
|
|
8405
8641
|
const retainedCollectionIds = new Set(newestCollections.map((job) => job.id));
|
|
8406
8642
|
cursor = null;
|
|
@@ -8413,7 +8649,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
8413
8649
|
}
|
|
8414
8650
|
cursor = page.nextCursor;
|
|
8415
8651
|
if (cursor !== null)
|
|
8416
|
-
await
|
|
8652
|
+
await this.#yieldMaintenance();
|
|
8417
8653
|
} while (cursor !== null);
|
|
8418
8654
|
}
|
|
8419
8655
|
async listGarbageCollectionJobs() {
|
|
@@ -8480,7 +8716,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
8480
8716
|
}
|
|
8481
8717
|
cursor = page.nextCursor;
|
|
8482
8718
|
if (cursor !== null)
|
|
8483
|
-
await
|
|
8719
|
+
await this.#yieldMaintenance();
|
|
8484
8720
|
} while (cursor !== null);
|
|
8485
8721
|
return newest?.discovery === void 0 ? void 0 : { jobId: newest.id, discovery: newest.discovery };
|
|
8486
8722
|
}
|
|
@@ -9138,7 +9374,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
9138
9374
|
}
|
|
9139
9375
|
cursor = page.nextCursor;
|
|
9140
9376
|
if (cursor !== null)
|
|
9141
|
-
await
|
|
9377
|
+
await this.#yieldMaintenance();
|
|
9142
9378
|
} while (cursor !== null);
|
|
9143
9379
|
return largest;
|
|
9144
9380
|
}
|
|
@@ -9717,7 +9953,17 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
9717
9953
|
if (job.state === "published") {
|
|
9718
9954
|
return this.#publishedCompactionProgress(table, job);
|
|
9719
9955
|
}
|
|
9720
|
-
|
|
9956
|
+
let linkedTransaction = job.transactionId === null ? void 0 : await this.store.getTransaction(job.transactionId);
|
|
9957
|
+
if (job.transactionId !== null && linkedTransaction?.status === "active" && Date.parse(linkedTransaction.expiresAt) <= dateMilliseconds(this.#now())) {
|
|
9958
|
+
const cutoff = dateIsoString(this.#now());
|
|
9959
|
+
await this.store.abortTransactionIfExpired({
|
|
9960
|
+
transactionId: linkedTransaction.id,
|
|
9961
|
+
expectedOwnerId: linkedTransaction.ownerId,
|
|
9962
|
+
expiresAtCutoff: cutoff,
|
|
9963
|
+
updatedAt: cutoff
|
|
9964
|
+
});
|
|
9965
|
+
linkedTransaction = await this.store.getTransaction(job.transactionId);
|
|
9966
|
+
}
|
|
9721
9967
|
if (linkedTransaction?.status === "committed" && linkedTransaction.committedVersion !== null) {
|
|
9722
9968
|
job = await this.#markCompactionPublished(job, linkedTransaction.committedVersion);
|
|
9723
9969
|
const recoveredManifest = await this.store.getManifest(linkedTransaction.committedVersion);
|
|
@@ -9882,18 +10128,20 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
9882
10128
|
}
|
|
9883
10129
|
transaction.setCompactionIntent(job.id, job.sourceBlockIds);
|
|
9884
10130
|
transaction.markLogicallyUnchanged();
|
|
9885
|
-
let
|
|
9886
|
-
|
|
9887
|
-
let publicationConflict;
|
|
10131
|
+
let publicationConflict;
|
|
10132
|
+
const publish = async () => {
|
|
9888
10133
|
try {
|
|
9889
|
-
|
|
9890
|
-
this.#afterCompactionCommit(
|
|
9891
|
-
|
|
10134
|
+
const manifest2 = await transaction.commit();
|
|
10135
|
+
this.#afterCompactionCommit(manifest2, table.id);
|
|
10136
|
+
return manifest2;
|
|
9892
10137
|
} catch (error) {
|
|
9893
10138
|
if (!(error instanceof WriteConflictError))
|
|
9894
10139
|
throw error;
|
|
9895
10140
|
publicationConflict = error;
|
|
10141
|
+
return void 0;
|
|
9896
10142
|
}
|
|
10143
|
+
};
|
|
10144
|
+
const rebaseOverConcurrentCommits = async () => {
|
|
9897
10145
|
const currentVersion = await this.store.getCurrentManifestVersion();
|
|
9898
10146
|
const sourcePresence = await this.#manifestBlockPresence(currentVersion, job.sourceBlockIds);
|
|
9899
10147
|
if (sourcePresence.some((present) => !present)) {
|
|
@@ -9920,7 +10168,24 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
9920
10168
|
}
|
|
9921
10169
|
transaction.setCompactionIntent(job.id, job.sourceBlockIds);
|
|
9922
10170
|
transaction.markLogicallyUnchanged();
|
|
9923
|
-
|
|
10171
|
+
};
|
|
10172
|
+
let manifest = await publish();
|
|
10173
|
+
if (manifest === void 0) {
|
|
10174
|
+
manifest = await this.#withCompactionPublicationSlot(async () => {
|
|
10175
|
+
for (let conflicts = 1; conflicts <= MAX_COMPACTION_PUBLICATION_CONFLICTS; conflicts += 1) {
|
|
10176
|
+
await rebaseOverConcurrentCommits();
|
|
10177
|
+
const published = await publish();
|
|
10178
|
+
if (published !== void 0)
|
|
10179
|
+
return published;
|
|
10180
|
+
}
|
|
10181
|
+
return void 0;
|
|
10182
|
+
});
|
|
10183
|
+
if (manifest === void 0) {
|
|
10184
|
+
if (transaction.status === "active")
|
|
10185
|
+
await transaction.abort();
|
|
10186
|
+
job = await this.#abortCompactionJob(job, `Compaction publication lost ${String(MAX_COMPACTION_PUBLICATION_CONFLICTS + 1)} manifest races`);
|
|
10187
|
+
throw new Error(job.error, { cause: publicationConflict });
|
|
10188
|
+
}
|
|
9924
10189
|
}
|
|
9925
10190
|
job = await this.#markCompactionPublished(job, manifest.version);
|
|
9926
10191
|
return compactionProgress(table.name, job, this.#compactionResult(table, job, manifest.version));
|
|
@@ -10080,7 +10345,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
10080
10345
|
}
|
|
10081
10346
|
segmentCursor = page.nextCursor;
|
|
10082
10347
|
if (segmentCursor !== null)
|
|
10083
|
-
await
|
|
10348
|
+
await this.#yieldMaintenance();
|
|
10084
10349
|
} while (segmentCursor !== null);
|
|
10085
10350
|
if (job.outputPartitionOrdinal === void 0 && job.targetLevel === 1 && plan.kind !== "copy-v1" && plan.partitions !== void 0) {
|
|
10086
10351
|
const table = await this.store.getTable(job.tableId);
|
|
@@ -12634,7 +12899,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
|
|
|
12634
12899
|
}
|
|
12635
12900
|
async #findTable(name) {
|
|
12636
12901
|
validateName(name, "Table");
|
|
12637
|
-
const table = await this.store.getTableByName(name);
|
|
12902
|
+
const table = await this.store.getTableByName(name) ?? await this.store.getTableByName(await this.#foldTableName(name));
|
|
12638
12903
|
if (table === void 0)
|
|
12639
12904
|
throw new UnknownTableError(name);
|
|
12640
12905
|
if (table.view !== void 0)
|
|
@@ -13329,7 +13594,26 @@ function postingFrequencyTotal(postings) {
|
|
|
13329
13594
|
}
|
|
13330
13595
|
const secondaryNumberBits = new DataView(new ArrayBuffer(8));
|
|
13331
13596
|
function storedSqlValueFromExecution(column, value) {
|
|
13332
|
-
|
|
13597
|
+
if (column?.type === "string" && column.sqlDomain === void 0 && typeof value === "string") {
|
|
13598
|
+
return externalSqlTextValue(value);
|
|
13599
|
+
}
|
|
13600
|
+
return coerceSqlWriteValue(column, value);
|
|
13601
|
+
}
|
|
13602
|
+
function coerceSqlWriteValue(column, value) {
|
|
13603
|
+
if (column === void 0 || column.sqlDomain !== void 0 || column.type === "string" || typeof value !== "string" || isSqlDomainValue(value)) {
|
|
13604
|
+
return value;
|
|
13605
|
+
}
|
|
13606
|
+
return readUntypedText(column.type, value);
|
|
13607
|
+
}
|
|
13608
|
+
function coerceSqlWriteBatch(table, input) {
|
|
13609
|
+
for (const column of table.columns) {
|
|
13610
|
+
if (column.type === "string" || column.sqlDomain !== void 0)
|
|
13611
|
+
continue;
|
|
13612
|
+
const values = input.columns[column.name];
|
|
13613
|
+
if (!values?.some((value) => typeof value === "string"))
|
|
13614
|
+
continue;
|
|
13615
|
+
input.columns[column.name] = values.map((value) => coerceSqlWriteValue(column, value));
|
|
13616
|
+
}
|
|
13333
13617
|
}
|
|
13334
13618
|
function executionSqlValueFromStorage(column, value) {
|
|
13335
13619
|
return column?.type === "string" && column.sqlDomain === void 0 && typeof value === "string" ? protectedSqlTextValue(value) : value;
|
|
@@ -13816,6 +14100,28 @@ function compositeKeyValue(columns, values) {
|
|
|
13816
14100
|
}
|
|
13817
14101
|
return columns.map((column, index) => secondaryTupleComponent(column.type, values[index] ?? null)).join("");
|
|
13818
14102
|
}
|
|
14103
|
+
function foldColumnName(table, name) {
|
|
14104
|
+
const columns = visibleTableColumns(table);
|
|
14105
|
+
if (columns.some((column) => column.name === name))
|
|
14106
|
+
return name;
|
|
14107
|
+
const lowered = name.toLowerCase();
|
|
14108
|
+
const matches = columns.filter((column) => column.name.toLowerCase() === lowered);
|
|
14109
|
+
return matches.length === 1 ? matches[0]?.name ?? name : name;
|
|
14110
|
+
}
|
|
14111
|
+
function foldInsertColumns(table, statement) {
|
|
14112
|
+
const columns = statement.columns.map((name) => foldColumnName(table, name));
|
|
14113
|
+
return columns.some((name, index) => name !== statement.columns[index]) ? { ...statement, columns } : statement;
|
|
14114
|
+
}
|
|
14115
|
+
function foldAssignmentColumns(table, assignments) {
|
|
14116
|
+
const folded = assignments.map((assignment) => {
|
|
14117
|
+
const column = foldColumnName(table, assignment.column);
|
|
14118
|
+
return column === assignment.column ? assignment : { ...assignment, column };
|
|
14119
|
+
});
|
|
14120
|
+
return folded.some((assignment, index) => assignment !== assignments[index]) ? folded : assignments;
|
|
14121
|
+
}
|
|
14122
|
+
function quoteIdentifier(name) {
|
|
14123
|
+
return `"${name.replace(/"/g, '""')}"`;
|
|
14124
|
+
}
|
|
13819
14125
|
function normalizeDomainBatch(table, input) {
|
|
13820
14126
|
for (const column of table.columns) {
|
|
13821
14127
|
const values = input.columns[column.name];
|
|
@@ -13900,23 +14206,8 @@ function normalizeDomainUpdate(table, input) {
|
|
|
13900
14206
|
function normalizePlanDomainLiterals(input, domains, types = /* @__PURE__ */ new Map()) {
|
|
13901
14207
|
const pass = { probing: true, needed: false };
|
|
13902
14208
|
const coerceText = (type, text) => {
|
|
13903
|
-
|
|
13904
|
-
|
|
13905
|
-
if (type === "number") {
|
|
13906
|
-
const trimmed = text.trim();
|
|
13907
|
-
if (trimmed === "")
|
|
13908
|
-
return void 0;
|
|
13909
|
-
const parsed = Number(trimmed);
|
|
13910
|
-
return Number.isFinite(parsed) ? parsed : void 0;
|
|
13911
|
-
}
|
|
13912
|
-
if (type === "boolean") {
|
|
13913
|
-
const lowered = text.trim().toLowerCase();
|
|
13914
|
-
if (lowered === "t" || lowered === "true" || lowered === "1")
|
|
13915
|
-
return true;
|
|
13916
|
-
if (lowered === "f" || lowered === "false" || lowered === "0")
|
|
13917
|
-
return false;
|
|
13918
|
-
}
|
|
13919
|
-
return void 0;
|
|
14209
|
+
const read = readUntypedText(type, text);
|
|
14210
|
+
return read === text ? void 0 : read;
|
|
13920
14211
|
};
|
|
13921
14212
|
const rewriteBlock = (block) => {
|
|
13922
14213
|
if (pass.probing && pass.needed)
|
|
@@ -14494,9 +14785,15 @@ function installStreamedWindow(vector, resident, windowStart, windowRows, memory
|
|
|
14494
14785
|
mutable.window = { start: windowStart, length: windowRows };
|
|
14495
14786
|
}
|
|
14496
14787
|
function derivedColumnarTable(name, result, schema) {
|
|
14497
|
-
const columns = new Map(schema.map(({ name: columnName, type }) => [
|
|
14788
|
+
const columns = new Map(schema.map(({ name: columnName, type, sqlDomain }) => [
|
|
14498
14789
|
columnName,
|
|
14499
|
-
{
|
|
14790
|
+
{
|
|
14791
|
+
type,
|
|
14792
|
+
values: sqlDomain?.kind === "numeric" ? result.rows.map((row) => {
|
|
14793
|
+
const value = row[columnName] ?? null;
|
|
14794
|
+
return typeof value === "number" ? exactNumericValue(value) : value;
|
|
14795
|
+
}) : result.rows.map((row) => row[columnName] ?? null)
|
|
14796
|
+
}
|
|
14500
14797
|
]));
|
|
14501
14798
|
return createColumnarTable(name, columns);
|
|
14502
14799
|
}
|