@minnowdb/core 0.7.5 → 0.7.8

package/dist/engine/database.js

@@ -1,5 +1,5 @@
 import { crossJoinPlan } from "../plan/model.js";
-import { toColumnarBatch } from "./batch.js";
+import { definedVectors, toColumnarBatch } from "./batch.js";
 import { ArtifactCache } from "./artifact-cache.js";
 import { estimateBatchBytes, estimateRowBytes, estimateValuesBytes } from "./byte-estimates.js";
 import { throwIfAborted } from "./cancellation.js";
@@ -13,7 +13,7 @@ import { estimateCompactionRowsPerOutput, planAlignedWriteBlockRanges as writeBl
 import { MAX_CACHEABLE_TEXT_CHARACTERS, MAX_SQL_PARAMETERS } from "./cache-limits.js";
 import { fillColumnDefaults, patchAutoIncrementValues } from "./defaults.js";
 import { cachedQueryTerms, FTS_TOKENIZER_VERSION, renderDocumentValue, tokenize as ftsTokenize } from "./fts.js";
-import { boundedMaintenanceBatchItems, simpleDataTypes, floorWholeNumberProduct, BlockReadBatchTooLargeError, validateColumnDefault, validateEnumValues, secondaryIndexColumnIds, secondaryIndexDirections, secondaryUniqueKeyNamespace, CompactionJobConflictError, CompactionBacklogError, GarbageCollectionJobConflictError, MAX_LEVEL_ZERO_SEGMENTS, MAX_FTS_CANDIDATE_ROW_IDS, MAX_FTS_POSTING_TERM_CHARACTERS, MAX_FTS_POSTINGS_PER_CHUNK, MAX_FTS_POSTING_ROW_IDS_PER_CHUNK, MAX_INDEXED_STRING_CHARACTERS, MAX_CATALOG_NAME_CHARACTERS, MAX_MANIFEST_BLOCK_PRESENCE_IDS, MAX_MAINTENANCE_BATCH_ITEMS, MAX_STORAGE_BULK_READ_ITEMS, MAX_TRANSACTION_STAGE_BLOCKS, MAX_TRANSACTION_STAGE_BYTES, MAX_SNAPSHOT_SESSION_TTL_MS, MAX_POSTING_BUILD_TTL_MS, MAX_TEMP_OWNER_TTL_MS, MAX_SNAPSHOT_FRAME_BATCH_BYTES, MAX_SNAPSHOT_FRAME_BATCH_ITEMS, MAX_SNAPSHOT_METADATA_BATCH_BYTES, SNAPSHOT_FRAME_KINDS, validateStorageId, SnapshotManifestMissingError, SchemaConflictError, TableInUseError, TableRecordConflictError, TransactionRecordConflictError, UniqueKeyConflictError, UniqueIndexCoverageError, WriteConflictError } from "../storage/types.js";
+import { boundedMaintenanceBatchItems, simpleDataTypes, floorWholeNumberProduct, BlockReadBatchTooLargeError, validateColumnDefault, validateEnumValues, secondaryIndexColumnIds, secondaryIndexDirections, secondaryUniqueKeyNamespace, CompactionJobConflictError, CompactionBacklogError, GarbageCollectionJobConflictError, MAX_LEVEL_ZERO_SEGMENTS, MAX_FTS_CANDIDATE_ROW_IDS, MAX_FTS_POSTING_TERM_CHARACTERS, MAX_FTS_POSTINGS_PER_CHUNK, MAX_FTS_POSTING_ROW_IDS_PER_CHUNK, MAX_INDEXED_STRING_CHARACTERS, MAX_CATALOG_NAME_CHARACTERS, MAX_MANIFEST_BLOCK_PRESENCE_IDS, MAX_MAINTENANCE_BATCH_ITEMS, MAX_STORAGE_BULK_READ_ITEMS, MAX_BLOCK_READ_BATCH_BYTES, MAX_TRANSACTION_STAGE_BLOCKS, MAX_TRANSACTION_STAGE_BYTES, MAX_SNAPSHOT_SESSION_TTL_MS, MAX_POSTING_BUILD_TTL_MS, MAX_TEMP_OWNER_TTL_MS, MAX_SNAPSHOT_FRAME_BATCH_BYTES, MAX_SNAPSHOT_FRAME_BATCH_ITEMS, MAX_SNAPSHOT_METADATA_BATCH_BYTES, SNAPSHOT_FRAME_KINDS, validateStorageId, SnapshotManifestMissingError, SchemaConflictError, TableInUseError, TableRecordConflictError, TransactionRecordConflictError, UniqueKeyConflictError, UniqueIndexCoverageError, WriteConflictError } from "../storage/types.js";
 import { decodeSnapshotFrameStream, encodeSnapshotFrameStreamFooter, encodeSnapshotFrameStreamHeader, extendSnapshotFrameStreamChecksum, snapshotFrameEnvelopeParts, snapshotFrameStreamHeaderIdentity } from "../storage/snapshot.js";
 import { Snapshot, TransactionManager } from "../transactions/index.js";
 import { applyWindowFunctions, bindPendingSelectShapes, bindPlanParameters, bindStatementParameters, DUAL_TABLE, dualTableRows, blockHasSubqueries, combineUnionResults, compileCheckExpression, compileQuery, hasAggregate, createRecursiveCteState, annotateAvgArgumentScales, compileStatement, comparisonHolds, createPreparedColumnarQuery, evaluateJoinedRowExpression, evaluateRowExpression, externalizeQueryResult, expressionColumnNames, inferBlockSchema, inferResultColumnDomains, isDeferredInsertExpression, isDefaultInsertValue, referencedColumns, childExpressions, expandFtsColumns, expandNaturalJoins, expandSourceColumnAliases, expandViewSources, forEachBlockExpression, forEachNestedBlock, planContainsFts, planHasNaturalJoins, planHasSourceColumnAliases, planReadsTable, planReadsViews, planReadsBeyondSingleScan, projectResultColumns, queryResultNeedsExternalization, resolveStatementDatetimes, subqueryResolutionSteps, topLevelFtsMatchConjuncts, transparentProjectionSource, unknownColumnDomains, validateDefaultExpression, windowOutputDomain, windowOutputType, volatileScalarFunctionNames, annotatePlanIntegerDivision, clonePlanTree, planMayHaveIntegerDivision, foldIdentifierCase, extendGroupByWithKeyDependents, expandRowReferences } from "./query.js";
@@ -78,6 +78,7 @@ const AUTO_COLLECT_RETRY_MIN_MS = 1e3;
 const AUTO_COLLECT_RETRY_MAX_MS = 6e4;
 const AUTO_COMPACT_STEP_BLOCKS = 4;
 const AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS = 256;
+const AUTO_COMPACT_BACKPRESSURE_LEVEL_ZERO_SEGMENTS = 2 * AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS;
 const AUTO_COMPACT_RETRY_MIN_MS = 250;
 const AUTO_COMPACT_RETRY_MAX_MS = 6e4;
 const MAX_COMPACTION_PUBLICATION_CONFLICTS = 64;
@@ -181,6 +182,9 @@ function migrationResult(steps) {
     steps: [...steps]
   };
 }
+function compactUpdateBatchInput(input) {
+  return Object.values(input.changes).includes(void 0) ? { keys: input.keys, changes: definedVectors(input.changes) } : input;
+}
 const MAX_GARBAGE_COLLECTION_RETAIN_RECENT_VERSIONS = 1024;
 const REFERENTIAL_CASCADES = 8;
 class TransactionRollback extends Error {
@@ -547,6 +551,10 @@ async function* singleSnapshotChunk(bytes) {
 }
 class MinnowDatabase {
   store;
+  #schema;
+  get #erased() {
+    return this;
+  }
   #closed = false;
   #closePromise;
   #transactions;
@@ -624,6 +632,7 @@ class MinnowDatabase {
   #preparedCatalogStates = /* @__PURE__ */ new WeakMap();
   constructor(store, options = {}) {
     this.store = store;
+    this.#schema = options.schema;
     this.#compression = options.compression ?? "gzip";
     this.#rowsPerBlock = options.rowsPerBlock ?? 65536;
     if (!Number.isSafeInteger(this.#rowsPerBlock) || this.#rowsPerBlock <= 0 || this.#rowsPerBlock > MAX_BLOCK_ROW_COUNT) {
@@ -1588,7 +1597,7 @@ class MinnowDatabase {
     return { ...statement, rows };
   }
   async insert(tableName, row) {
-    return this.insertBatch(tableName, [row]);
+    return this.#erased.insertBatch(tableName, [row]);
   }
   upsertBatch(tableName, input, options = {}) {
     return this.#withWriteReservation(() => this.#upsertBatchReserved(tableName, input, options));
@@ -1612,9 +1621,7 @@ class MinnowDatabase {
     const { result, batch, generated, autoIncrement } = completed;
     collectAutoIncrementGenerated(batch, generated, autoIncrement);
     if (result.acceptedRowIndexes.length !== result.requestedRowCount) {
-      for (const [name, values] of generated) {
-        generated.set(name, result.acceptedRowIndexes.map((index) => values[index] ?? null));
-      }
+      remapGeneratedColumns(generated, result.acceptedRowIndexes);
     }
     return {
       tableName: result.tableName,
@@ -1632,10 +1639,10 @@ class MinnowDatabase {
     };
   }
   async upsert(tableName, row, options = {}) {
-    return this.upsertBatch(tableName, [row], options);
+    return this.#erased.upsertBatch(tableName, [row], options);
   }
   updateBatch(tableName, input) {
-    return this.#withWriteReservation(() => this.#updateBatchReserved(tableName, input));
+    return this.#withWriteReservation(() => this.#updateBatchReserved(tableName, compactUpdateBatchInput(input)));
   }
   async #updateBatchReserved(tableName, input) {
     return this.#runWrite(async () => {
@@ -1658,13 +1665,13 @@ class MinnowDatabase {
     });
   }
   async update(tableName, key, changes) {
-    return this.updateBatch(tableName, {
+    return this.#erased.updateBatch(tableName, {
       keys: [key],
-      changes: Object.fromEntries(Object.entries(changes).map(([name, value]) => [name, [value]]))
+      changes: Object.fromEntries(Object.entries(changes).flatMap(([name, value]) => value === void 0 ? [] : [[name, [value]]]))
     });
   }
   async delete(tableName, key) {
-    return this.deleteBatch(tableName, { keys: [key] });
+    return this.#erased.deleteBatch(tableName, { keys: [key] });
   }
   deleteBatch(tableName, input) {
     return this.#withWriteReservation(() => this.#deleteBatchReserved(tableName, input));
@@ -1679,7 +1686,7 @@ class MinnowDatabase {
       if (dependents.length === 0)
         return this.#deleteBatchOnce(tableName, normalizedInput);
       const started = performance.now();
-      const { result, version } = await this.write(async (session) => {
+      const { result, version } = await this.#erased.write(async (session) => {
         await this.#applyReferentialActions(table, [...normalizedInput.keys], session, REFERENTIAL_CASCADES);
         return session.deleteBatch(tableName, normalizedInput);
       });
@@ -1883,7 +1890,7 @@ class MinnowDatabase {
     }
   }
   bufferedWriter(tableName, options = {}) {
-    return new BufferedTableWriter(this, tableName, options);
+    return new BufferedTableWriter(this.#erased, tableName, options);
   }
   async #writeUpdateBatch(table, keyColumn, input, keys) {
     await this.#assertCompactionCapacity(table);
@@ -2092,30 +2099,21 @@ class MinnowDatabase {
       const normalizedConflictWhere = kind === "upsert" && conflictWhere !== void 0 ? normalizeUpsertConflictWhere(table, conflictWhere) : void 0;
       const upsertKeyColumn = kind === "upsert" ? getUniqueKeyColumn(table) : void 0;
       if (upsertKeyColumn !== void 0) {
-        upsertFirings = await this.#upsertTriggerFirings(table, upsertKeyColumn, input, rowCount, (sql, params) => this.#sessionQuery(transaction, sql, { params }), normalizedConflictWhere, normalizedConflictWhere !== void 0);
+        upsertFirings = await this.#upsertTriggerFirings(table, upsertKeyColumn, input, rowCount, (sql, params) => this.#sessionQuery(transaction, sql, { params }), normalizedConflictWhere);
       }
       if (normalizedConflictWhere !== void 0) {
         if (upsertFirings === void 0) {
           throw new Error("Upsert conflict classification is missing");
         }
-        const classifiedFirings = upsertFirings;
-        acceptedRowIndexes = [...classifiedFirings.inserts, ...classifiedFirings.updates].sort((left, right) => left - right);
-        skippedRowCount = classifiedFirings.skipped.length;
-        input = selectBatchRows(input, acceptedRowIndexes);
-        rowCount = acceptedRowIndexes.length;
+        counts = { inserted: upsertFirings.inserts.length, updated: upsertFirings.updates.length };
+        const filtered = applyUpsertConflictFilter(input, rowCount, upsertFirings);
+        acceptedRowIndexes = filtered.acceptedRowIndexes;
+        skippedRowCount = filtered.skippedRowCount;
+        input = filtered.batch;
+        rowCount = filtered.rowCount;
+        upsertFirings = filtered.firings;
         logicalBytes = estimateBatchBytes(input);
         resolvedKeys = batchKeys(table, input);
-        counts = {
-          inserted: classifiedFirings.inserts.length,
-          updated: classifiedFirings.updates.length
-        };
-        const acceptedPosition = new Map(acceptedRowIndexes.map((original, position) => [original, position]));
-        upsertFirings = {
-          inserts: classifiedFirings.inserts.map((index) => acceptedPosition.get(index) ?? -1),
-          updates: classifiedFirings.updates.map((index) => acceptedPosition.get(index) ?? -1),
-          skipped: [],
-          oldImages: acceptedRowIndexes.map((index) => classifiedFirings.oldImages[index])
-        };
         if (rowCount === 0) {
           await transaction.abort();
           const version = transaction.snapshotVersion;
@@ -3973,23 +3971,38 @@ class MinnowDatabase {
       return;
     const hintVersion = transaction?.snapshotVersion ?? await this.store.getCurrentManifestVersion();
     const hint = this.#autoCompactionHints.get(table.id);
-    if (hint?.version === hintVersion && hint.levelZero < MAX_LEVEL_ZERO_SEGMENTS)
-      return;
-    let segments = await this.#currentVisibleSegments(table);
-    let levelZero = 0;
-    for (const segment of segments) {
-      if (segment.level === 0)
-        levelZero += 1;
+    let visible;
+    let levelZero;
+    if (hint?.version === hintVersion) {
+      ({ visible, levelZero } = hint);
+    } else {
+      const segments = await this.#currentVisibleSegments(table);
+      visible = segments.length;
+      levelZero = countLevelZeroSegments(segments);
     }
-    if (levelZero < MAX_LEVEL_ZERO_SEGMENTS)
+    if (levelZero < AUTO_COMPACT_BACKPRESSURE_LEVEL_ZERO_SEGMENTS)
+      return;
+    const atCeiling = levelZero >= MAX_LEVEL_ZERO_SEGMENTS;
+    if (!atCeiling && this.#autoCompactionBackoff.get(table.id)?.retryTimer !== void 0)
       return;
     await this.#publishPendingCompactions();
-    await this.compactTableStep(table.name, {
-      maxBlocks: AUTO_COMPACT_STEP_BLOCKS,
-      maxLevel0Segments: AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS
-    });
-    segments = await this.#currentVisibleSegments(table);
-    levelZero = segments.reduce((count, segment) => count + (segment.level === 0 ? 1 : 0), 0);
+    try {
+      const progress = await this.compactTableStep(table.name, {
+        maxBlocks: AUTO_COMPACT_STEP_BLOCKS,
+        maxLevel0Segments: AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS
+      });
+      if (!atCeiling && progress.result !== null && !progress.result.compacted) {
+        this.#backOffAutoCompaction(table.id, visible);
+      }
+    } catch (error) {
+      if (atCeiling)
+        throw error;
+      this.#backOffAutoCompaction(table.id, visible);
+      return;
+    }
+    if (!atCeiling)
+      return;
+    levelZero = countLevelZeroSegments(await this.#currentVisibleSegments(table));
     if (levelZero >= MAX_LEVEL_ZERO_SEGMENTS) {
       throw new CompactionBacklogError(table.name, levelZero, MAX_LEVEL_ZERO_SEGMENTS);
     }
@@ -4299,9 +4312,9 @@ class MinnowDatabase {
       }
     }
   }
-  async #upsertTriggerFirings(table, keyColumn, batch, rowCount, readRows, conflictWhere, force = false) {
+  async #upsertTriggerFirings(table, keyColumn, batch, rowCount, readRows, conflictWhere) {
     const fires = (table.triggers ?? []).some((trigger) => trigger.event === "insert" || trigger.event === "update");
-    if (!fires && !force || rowCount === 0)
+    if (!fires && conflictWhere === void 0 || rowCount === 0)
       return void 0;
     const quote = (name) => `"${name.replaceAll('"', '""')}"`;
     const keyValues = batch.columns[keyColumn.name] ?? [];
@@ -4415,8 +4428,15 @@ class MinnowDatabase {
     if (target.uniqueKeyColumnId !== void 0) {
       throw new TypeError(`Trigger bodies insert into keyless tables only: ${target.name}`);
     }
+    let columns = compiled.columns;
+    if (columns.length === 0 && compiled.defaultValues !== true) {
+      columns = visibleTableColumns(target).map((column) => column.name);
+      if (derivedRows.some((row) => row.length !== columns.length)) {
+        throw new TypeError(`Trigger body INSERT must match the ${String(columns.length)} visible columns of ${target.name}`);
+      }
+    }
     const statementNow = this.#now();
-    const materialized = await this.#materializeInsertExpressions({ ...compiled, rows: derivedRows }, statementNow);
+    const materialized = await this.#materializeInsertExpressions({ ...compiled, columns, rows: derivedRows }, statementNow);
     const input = insertStatementBatch(materialized);
     coerceSqlWriteBatch(target, input);
     const filled = await fillColumnDefaults(target, input, (sql) => this.#evaluateDefaultExpression(sql, statementNow), derivedRows.length);
@@ -4629,10 +4649,10 @@ class MinnowDatabase {
         staged += 1;
         return guarded(() => this.#sessionInsert(transaction, tableName, input, "insert"));
       },
-      upsertBatch: async (tableName, input) => {
+      upsertBatch: async (tableName, input, options2) => {
         open();
         staged += 1;
-        return guarded(() => this.#sessionInsert(transaction, tableName, input, "upsert"));
+        return guarded(() => this.#sessionInsert(transaction, tableName, input, "upsert", options2));
       },
       updateBatch: async (tableName, input) => {
         open();
@@ -4866,10 +4886,17 @@ class MinnowDatabase {
     }
     return { added, removed };
   }
-  async #sessionInsert(transaction, tableName, input, kind, cascadeBudget = 1) {
+  async #sessionInsert(transaction, tableName, input, kind, options, cascadeBudget = 1) {
     const table = await this.#findTable(tableName);
-    await this.#assertCompactionCapacity(table, transaction);
-    const { batch, generated, autoIncrement, rowCount } = await this.#fillDefaults(table, input);
+    const sessionUpsertKeyColumn = kind === "upsert" ? getUniqueKeyColumn(table) : void 0;
+    const normalizedConflictWhere = kind === "upsert" && options?.conflictWhere !== void 0 ? normalizeUpsertConflictWhere(table, options.conflictWhere) : void 0;
+    if (normalizedConflictWhere !== void 0 && sessionUpsertKeyColumn === void 0) {
+      throw new TypeError(`Table needs a unique key before it can be upserted: ${table.name}`);
+    }
+    const filled = await this.#fillDefaults(table, input);
+    const { generated, autoIncrement } = filled;
+    let batch = filled.batch;
+    let rowCount = filled.rowCount;
     if (autoIncrement !== void 0 && autoIncrement.missingIndexes.length > 0) {
       const values = await this.store.reserveAutoIncrement(table.id, autoIncrement.column.id, autoIncrement.missingIndexes.length, autoIncrement.atLeast);
       patchAutoIncrementValues(batch, autoIncrement, values);
@@ -4878,6 +4905,32 @@ class MinnowDatabase {
         validateValue(autoIncrement.column, patched[rowIndex] ?? null, rowIndex);
       }
     }
+    let sessionUpsertFirings = sessionUpsertKeyColumn === void 0 ? void 0 : await this.#upsertTriggerFirings(table, sessionUpsertKeyColumn, batch, rowCount, (sql, params) => this.#sessionQuery(transaction, sql, { params }), normalizedConflictWhere);
+    let skippedRowCount = 0;
+    if (normalizedConflictWhere !== void 0) {
+      if (sessionUpsertFirings === void 0) {
+        throw new Error("Upsert conflict classification is missing");
+      }
+      const filtered = applyUpsertConflictFilter(batch, rowCount, sessionUpsertFirings);
+      skippedRowCount = filtered.skippedRowCount;
+      if (filtered.rowCount !== rowCount) {
+        remapGeneratedColumns(generated, filtered.acceptedRowIndexes);
+      }
+      batch = filtered.batch;
+      rowCount = filtered.rowCount;
+      sessionUpsertFirings = filtered.firings;
+      if (rowCount === 0) {
+        collectAutoIncrementGenerated(batch, generated, autoIncrement);
+        return {
+          tableName: table.name,
+          segmentId: null,
+          rowCount: 0,
+          skippedRowCount,
+          ...generated.size === 0 ? {} : { generatedColumns: Object.fromEntries(generated) }
+        };
+      }
+    }
+    await this.#assertCompactionCapacity(table, transaction);
     const keys = batchKeys(table, batch);
     if (keys !== void 0) {
       transaction.setUniqueKeyChanges({
@@ -4889,8 +4942,6 @@ class MinnowDatabase {
     await this.#assertForeignKeysPresent(table, (column) => batch.columns[column] ?? [], (sql, params) => this.#sessionQuery(transaction, sql, { params }), transaction);
     const rowIds = await this.store.reserveRowIds(table.id, rowCount);
     const insertValueAt = (source, column, rowIndex) => source === "new" ? batch.columns[column]?.[rowIndex] ?? null : null;
-    const sessionUpsertKeyColumn = kind === "upsert" ? getUniqueKeyColumn(table) : void 0;
-    const sessionUpsertFirings = sessionUpsertKeyColumn === void 0 ? void 0 : await this.#upsertTriggerFirings(table, sessionUpsertKeyColumn, batch, rowCount, (sql, params) => this.#sessionQuery(transaction, sql, { params }));
     stageSecondaryUniqueInsertChanges(transaction, table, batch, kind === "upsert" ? sessionUpsertFirings?.oldImages : void 0);
     if (kind === "insert") {
       await this.#stageTriggerDerivedInserts(transaction, table, "insert", rowCount, insertValueAt, "before", cascadeBudget);
@@ -4908,6 +4959,7 @@ class MinnowDatabase {
       tableName: table.name,
       segmentId,
       rowCount,
+      skippedRowCount,
       ...generated.size === 0 ? {} : { generatedColumns: Object.fromEntries(generated) }
     };
   }
@@ -5135,7 +5187,10 @@ class MinnowDatabase {
     }
     return (await this.#memoizedQuery(query.plan, `typed ${planMemoKey(query.plan)}`, {}, probe)).rows;
   }
-  async migrate(definition, options = {}) {
+  async migrate(definition = this.#schema, options = {}) {
+    if (definition === void 0) {
+      throw new TypeError("migrate() needs a schema: pass a definition, or construct the database with { schema }");
+    }
     let originalSteps;
     for (let attempt = 0; ; attempt += 1) {
       try {
@@ -6036,7 +6091,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
     let outcome;
     let version;
     if (options.writer === void 0) {
-      const completed = await this.write(apply);
+      const completed = await this.#erased.write(apply);
       outcome = completed.result;
       version = completed.version;
     } else {
@@ -6141,7 +6196,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
     }
     const sourceKey = mergeSourceKeyExpression(statement, keyColumn.name);
     const sourceSql = mergeSourceSql(statement);
-    const { result: applied, version } = await this.write(async (session) => {
+    const { result: applied, version } = await this.#erased.write(async (session) => {
       const sourceRows = (await session.query(sourceSql)).rows;
       const keys = sourceRows.map((row) => storedSqlValueFromExecution(keyColumn, evaluateJoinedRowExpression(sourceKey, { [statement.source.alias]: row })));
       const present = /* @__PURE__ */ new Map();
@@ -6722,7 +6777,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
         }
         await this.#assertStagedInsertKeysFree(table2, stagedKeyColumn, input, writer);
       }
-      const result = await (viaUpsert ? writer?.upsertBatch(statement.table, input) ?? this.upsertBatch(statement.table, input) : writer?.insertBatch(statement.table, input) ?? this.insertBatch(statement.table, input));
+      const result = await (viaUpsert ? writer?.upsertBatch(statement.table, input) ?? this.#erased.upsertBatch(statement.table, input) : writer?.insertBatch(statement.table, input) ?? this.#erased.insertBatch(statement.table, input));
       const generated = "generatedColumns" in result ? result.generatedColumns ?? {} : {};
       return {
         kind: "insert",
@@ -6854,7 +6909,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
           ...returnedRows2 === void 0 || returningColumns === void 0 ? {} : returningExecuteFields(table, returningColumns, returnedRows2)
         };
       }
-      const deleted = options.writer === void 0 ? await this.deleteBatch(table.name, { keys }) : await options.writer.deleteBatch(table.name, { keys });
+      const deleted = options.writer === void 0 ? await this.#erased.deleteBatch(table.name, { keys }) : await options.writer.deleteBatch(table.name, { keys });
       return {
         kind: "delete",
         table: table.name,
@@ -6891,7 +6946,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
         returnedChanges[assignment.column] = executionValues;
       }
     }
-    const updated = options.writer === void 0 ? await this.updateBatch(table.name, { keys, changes }) : await options.writer.updateBatch(table.name, { keys, changes });
+    const updated = options.writer === void 0 ? await this.#erased.updateBatch(table.name, { keys, changes }) : await options.writer.updateBatch(table.name, { keys, changes });
     const returnedRows = returningColumns === void 0 ? void 0 : rows.map((row, index) => Object.fromEntries(returningColumns.map((name) => [
       name,
       returnedChanges !== void 0 && name in returnedChanges ? returnedChanges[name]?.[index] ?? null : updated.generatedColumns?.[name] !== void 0 ? updated.generatedColumns[name][index] ?? null : row[name] ?? null
@@ -9275,7 +9330,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
       }
       const priorAttemptOutputStoredBytes = await this.#priorCompactionAttemptOutputStoredBytes(table.id, baseJobId);
       const maximumOutputStoredBytes = Math.max(0, floorWholeNumberProduct(selection.level0SourceStoredBytes, maxWriteAmplification, "Compaction maximum output stored bytes") - priorAttemptOutputStoredBytes);
-      const plannedOutputStoredBytesUpperBound = await this.#plannedPhysicalOutputStoredBytesUpperBound(rewritePlan, snapshot);
+      const plannedOutputStoredBytesUpperBound = await this.#plannedPhysicalOutputStoredBytesUpperBound(rewritePlan, memoryBudgetBytes, snapshot);
       levelTwoBudget = {
         outputPartitionOrdinal,
         maxWriteAmplification,
@@ -9731,17 +9786,15 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
     const columnIndexById = new Map(table.columns.map((column, index) => [column.id, index]));
     const touched = /* @__PURE__ */ new Set();
     const deltaKeys = /* @__PURE__ */ new Map();
-    for (const segment of segments) {
-      if (!mergeSourceReferencesKeys(segment.kind))
-        continue;
-      const keys = [];
-      await this.#forEachMergeSourceKey(segment, keyColumn, snapshot, (value) => {
-        const key = overlayKeyOf(keyColumn.type, value);
-        keys.push(key);
-        touched.add(key);
-      });
-      deltaKeys.set(segment.segmentId, keys);
-    }
+    const deltas = segments.filter((segment) => mergeSourceReferencesKeys(segment.kind));
+    for (const segment of deltas)
+      deltaKeys.set(segment.segmentId, []);
+    const unusedBudgetBytes = memoryBudgetBytes - plannerMemoryBytes;
+    await this.#forEachMergeSourceKey(deltas, keyColumn, unusedBudgetBytes, snapshot, (segment, value) => {
+      const key = overlayKeyOf(keyColumn.type, value);
+      deltaKeys.get(segment.segmentId)?.push(key);
+      touched.add(key);
+    });
     let slotCount = 0;
     for (const segment of segments) {
       if (mergeSourceBearsRows(segment.kind))
@@ -9813,7 +9866,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
       if (keys !== void 0) {
         keys.forEach(visit);
       } else if (touched.size > 0) {
-        await this.#forEachMergeSourceKey(segment, keyColumn, snapshot, (value, rowIndex) => {
+        await this.#forEachMergeSourceKey([segment], keyColumn, unusedBudgetBytes, snapshot, (_, value, rowIndex) => {
           visit(overlayKeyOf(keyColumn.type, value), rowIndex);
         });
       }
@@ -9850,28 +9903,48 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
     }
     return { ...output.finish(), sourceOutputRowStarts };
   }
-  async #forEachMergeSourceKey(segment, column, snapshot, action) {
-    const planned = segment.columns.find((candidate) => candidate.columnId === column.id);
-    if (planned === void 0) {
-      throw new Error(`Mutation segment has no key column: ${segment.segmentId}`);
+  async #forEachMergeSourceKey(segments, column, unusedBudgetBytes, snapshot, action) {
+    const blocks = [];
+    let largestEncodedBytes = 0;
+    for (const segment of segments) {
+      const planned = segment.columns.find((candidate) => candidate.columnId === column.id);
+      if (planned === void 0) {
+        throw new Error(`Mutation segment has no key column: ${segment.segmentId}`);
+      }
+      for (const source of planned.sourceBlocks) {
+        blocks.push({
+          blockId: source.blockId,
+          storedBytes: source.storedBytes,
+          encodedBytes: source.encodedBytes,
+          segment,
+          source
+        });
+        largestEncodedBytes = Math.max(largestEncodedBytes, source.encodedBytes);
+      }
     }
-    let rowIndex = 0;
-    for (const source of planned.sourceBlocks) {
-      await this.#renewInternalLeaseIfNeeded(snapshot);
-      const bytes = await this.store.getBlock(source.blockId);
-      if (bytes === void 0)
-        throw new Error(`Compaction source block is missing: ${source.blockId}`);
+    const limitBytes = safeWholeNumberSum([
+      Math.max(0, unusedBudgetBytes),
+      safeWholeNumberProduct(largestEncodedBytes, MERGE_PLANNER_DECODED_KEY_BLOCK_FACTOR, "Mutation compaction key decode memory")
+    ], "Mutation compaction key decode memory");
+    const rowIndexes = /* @__PURE__ */ new Map();
+    await this.#decodeCompactionBlocks(blocks, { limitBytes, retainsDecoded: false }, snapshot, async (bytes, { source }) => {
       const description = inspectBlock(bytes);
       if (bytes.byteLength !== source.storedBytes || description.encodedLength !== source.encodedBytes || description.checksum !== source.checksum || description.rowCount !== source.rowCount || description.type !== column.type) {
         throw new Error(`Compaction source block differs from its plan: ${source.blockId}`);
       }
-      for (const value of (await decodeBlock(bytes)).column.values) {
-        action(value, rowIndex);
+      return (await decodeBlock(bytes)).column.values;
+    }, (values, { segment }) => {
+      let rowIndex = rowIndexes.get(segment.segmentId) ?? 0;
+      for (const value of values) {
+        action(segment, value, rowIndex);
         rowIndex += 1;
       }
-    }
-    if (rowIndex !== segment.rowCount) {
-      throw new Error(`Mutation segment key rows differ: ${segment.segmentId}`);
+      rowIndexes.set(segment.segmentId, rowIndex);
+    });
+    for (const segment of segments) {
+      if ((rowIndexes.get(segment.segmentId) ?? 0) !== segment.rowCount) {
+        throw new Error(`Mutation segment key rows differ: ${segment.segmentId}`);
+      }
     }
   }
   async #refinePhysicalOutputWindows(columns, estimatedOutputs, targetBlockBytes, outputCompression, memoryBudgetBytes, snapshot) {
@@ -9930,12 +10003,12 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
     const loaded = await this.#loadPhysicalCompactionRanges(column, output, outputCompression, memoryBudgetBytes, snapshot);
     return measurePhysicalColumnRanges(column.type, loaded.ranges);
   }
-  async #plannedPhysicalOutputStoredBytesUpperBound(plan, snapshot) {
+  async #plannedPhysicalOutputStoredBytesUpperBound(plan, memoryBudgetBytes, snapshot) {
     const layout = physicalRewriteLayout(plan);
     let total = 0;
     for (const output of layout.outputs) {
       for (const column of layout.columns) {
-        const measurement = await this.#measurePhysicalCompactionOutput(column, output, plan.outputCompression, Number.MAX_SAFE_INTEGER, snapshot);
+        const measurement = await this.#measurePhysicalCompactionOutput(column, output, plan.outputCompression, memoryBudgetBytes, snapshot);
         total = safeWholeNumberSum([
           total,
           maximumPhysicalBlockByteLength(measurement.encodedByteLength, measurement.metadata, plan.outputCompression)
@@ -10296,26 +10369,84 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
       throw new CompactionMemoryBudgetError(memoryBudgetBytes, memoryBound);
     }
     const outputEnd = safeWholeNumberSum([output.rowStart, output.rowCount], "Compaction output row range");
-    const ranges = [];
-    for (const sourceBlock of overlappingPhysicalSourceRanges(column, output)) {
-      if (snapshot !== void 0)
-        await this.#renewInternalLeaseIfNeeded(snapshot);
-      const bytes = await this.store.getBlock(sourceBlock.blockId);
-      if (bytes === void 0) {
-        throw new Error(`A compaction source block is missing: ${sourceBlock.blockId}`);
-      }
+    const sourceBlocks = overlappingPhysicalSourceRanges(column, output);
+    const distinct = [];
+    const seen = /* @__PURE__ */ new Set();
+    for (const sourceBlock of sourceBlocks) {
+      if (seen.has(sourceBlock.blockId))
+        continue;
+      seen.add(sourceBlock.blockId);
+      distinct.push(sourceBlock);
+    }
+    const decoded = /* @__PURE__ */ new Map();
+    const peakWorkingBytes = await this.#decodeCompactionBlocks(distinct, { limitBytes: memoryBudgetBytes, retainsDecoded: true }, snapshot, async (bytes, sourceBlock) => {
       const description = inspectBlock(bytes);
       if (bytes.byteLength !== sourceBlock.storedBytes || description.encodedLength !== sourceBlock.encodedBytes || description.checksum !== sourceBlock.checksum || description.rowCount !== sourceBlock.sourceBlockRowCount || description.type !== column.type) {
         throw new Error(`A compaction source block differs from its plan: ${sourceBlock.blockId}`);
       }
-      const decoded = await decodePhysicalBlock(bytes);
+      return decodePhysicalBlock(bytes);
+    }, (block, sourceBlock) => {
+      decoded.set(sourceBlock.blockId, block);
+    });
+    const ranges = [];
+    for (const sourceBlock of sourceBlocks) {
+      const block = decoded.get(sourceBlock.blockId);
+      if (block === void 0) {
+        throw new Error(`A compaction source block is missing: ${sourceBlock.blockId}`);
+      }
       const sourceEnd = safeWholeNumberSum([sourceBlock.outputRowStart, sourceBlock.rowCount], "Compaction source row range");
       const start = sourceBlock.sourceRowStart + Math.max(output.rowStart, sourceBlock.outputRowStart) - sourceBlock.outputRowStart;
       const end = sourceBlock.sourceRowStart + Math.min(outputEnd, sourceEnd) - sourceBlock.outputRowStart;
-      const slice = slicePhysicalColumn(decoded.column, start, end);
+      const slice = slicePhysicalColumn(block.column, start, end);
       ranges.push({ column: slice, start: 0, end: slice.rowCount });
     }
-    return { ranges, peakWorkingBytes: memoryBound };
+    return { ranges, peakWorkingBytes: Math.max(memoryBound, peakWorkingBytes) };
+  }
+  async #decodeCompactionBlocks(blocks, budget, snapshot, decode, consume) {
+    let retainedBytes = 0;
+    let peakBytes = 0;
+    let start = 0;
+    while (start < blocks.length) {
+      let end = start;
+      let transientBytes = 0;
+      let storedBytes = 0;
+      for (; end < blocks.length && end - start < MAX_STORAGE_BULK_READ_ITEMS; end += 1) {
+        const block = blocks[end];
+        if (block === void 0)
+          break;
+        const blockTransientBytes = safeWholeNumberSum([
+          safeWholeNumberProduct(block.storedBytes, 2, "Compaction decode memory"),
+          safeWholeNumberProduct(block.encodedBytes, 2, "Compaction decode memory")
+        ], "Compaction decode memory");
+        if (end > start && (transientBytes + blockTransientBytes > budget.limitBytes - retainedBytes || storedBytes + block.storedBytes > MAX_BLOCK_READ_BATCH_BYTES)) {
+          break;
+        }
+        transientBytes += blockTransientBytes;
+        storedBytes += block.storedBytes;
+      }
+      const group = blocks.slice(start, end);
+      if (snapshot !== void 0)
+        await this.#renewInternalLeaseIfNeeded(snapshot);
+      const bytes = await this.#readBlockWindow(group.map((block) => block.blockId));
+      const decoded = await Promise.all(group.map((block, index) => {
+        const stored = bytes[index];
+        if (stored === void 0) {
+          throw new Error(`A compaction source block is missing: ${block.blockId}`);
+        }
+        return decode(stored, block);
+      }));
+      peakBytes = Math.max(peakBytes, retainedBytes + transientBytes);
+      for (const [index, block] of group.entries()) {
+        const value = decoded[index];
+        if (value === void 0)
+          throw new Error("Compaction decode lost a block");
+        consume(value, block);
+        if (budget.retainsDecoded)
+          retainedBytes += block.encodedBytes;
+      }
+      start = end;
+    }
+    return peakBytes;
   }
   async #loadCompactionSources(job) {
     const segments = await Promise.all(job.sourceSegmentIds.map((id) => this.store.getSegment(id)));
@@ -14152,6 +14283,49 @@ function normalizeColumnLogicalValue(column, value) {
   }
   return value;
 }
+function mergeAscendingIndexes(left, right) {
+  const merged = [];
+  let leftIndex = 0;
+  let rightIndex = 0;
+  while (leftIndex < left.length || rightIndex < right.length) {
+    const a = left[leftIndex];
+    const b = right[rightIndex];
+    if (b === void 0 || a !== void 0 && a <= b) {
+      if (a !== void 0)
+        merged.push(a);
+      leftIndex += 1;
+    } else {
+      merged.push(b);
+      rightIndex += 1;
+    }
+  }
+  return merged;
+}
+function applyUpsertConflictFilter(batch, rowCount, classified) {
+  const acceptedRowIndexes = mergeAscendingIndexes(classified.inserts, classified.updates);
+  const skippedRowCount = classified.skipped.length;
+  if (acceptedRowIndexes.length === rowCount) {
+    return { batch, rowCount, skippedRowCount, acceptedRowIndexes, firings: classified };
+  }
+  const acceptedPosition = new Map(acceptedRowIndexes.map((original, position) => [original, position]));
+  return {
+    batch: selectBatchRows(batch, acceptedRowIndexes),
+    rowCount: acceptedRowIndexes.length,
+    skippedRowCount,
+    acceptedRowIndexes,
+    firings: {
+      inserts: classified.inserts.map((index) => acceptedPosition.get(index) ?? -1),
+      updates: classified.updates.map((index) => acceptedPosition.get(index) ?? -1),
+      skipped: [],
+      oldImages: acceptedRowIndexes.map((index) => classified.oldImages[index])
+    }
+  };
+}
+function remapGeneratedColumns(generated, acceptedRowIndexes) {
+  for (const [name, values] of generated) {
+    generated.set(name, acceptedRowIndexes.map((index) => values[index] ?? null));
+  }
+}
 function selectBatchRows(input, indexes) {
   return {
     columns: Object.fromEntries(Object.entries(input.columns).map(([name, values]) => [
@@ -14431,6 +14605,14 @@ function autoCompactionHint(version, segments) {
     deltas
   };
 }
+function countLevelZeroSegments(segments) {
+  let levelZero = 0;
+  for (const segment of segments) {
+    if (segment.level === 0)
+      levelZero += 1;
+  }
+  return levelZero;
+}
 function autoCompactionDueHint(hint) {
   return hint.levelZero >= AUTO_COMPACT_SCAN_SEGMENTS || hint.deltas >= AUTO_COMPACT_DELTA_SEGMENTS;
 }
@@ -15881,6 +16063,7 @@ function sourceOrderTuple(segment, transactions, label) {
   };
 }
 const MERGE_PLANNER_KEY_BYTES = 96;
+const MERGE_PLANNER_DECODED_KEY_BLOCK_FACTOR = 4;
 function planLinearOutputPartitions(totalRows, partitionRows, firstOrder, nextOrder) {
   const count = Math.max(1, Math.ceil(totalRows / partitionRows));
   const orders = fractionalLogicalOrders(firstOrder, nextOrder, count);
@@ -15996,7 +16179,6 @@ function mergePlannerMemoryBound(table, segments, keyColumnId) {
   const PATCH_ROW_BYTES = 64;
   const PATCH_CELL_BYTES = 48;
   const RANGE_BYTES = 80;
-  const DECODED_KEY_BLOCK_FACTOR = 4;
   let slotRows = 0;
   let deltaKeys = 0;
   let patchRows = 0;
@@ -16027,7 +16209,7 @@ function mergePlannerMemoryBound(table, segments, keyColumnId) {
       safeWholeNumberProduct(columns, PATCH_CELL_BYTES, "Mutation patch cells")
     ], "Mutation compaction patch row"), "Mutation compaction patches"),
     safeWholeNumberProduct(safeWholeNumberSum([sourceBlocks, safeWholeNumberProduct(patchRows, columns, "Mutation patched cells")], "Mutation compaction ranges"), RANGE_BYTES, "Mutation compaction range bytes"),
-    safeWholeNumberProduct(largestKeyBlockBytes, DECODED_KEY_BLOCK_FACTOR, "Mutation compaction decoded key block")
+    safeWholeNumberProduct(largestKeyBlockBytes, MERGE_PLANNER_DECODED_KEY_BLOCK_FACTOR, "Mutation compaction decoded key block")
   ], "Mutation compaction planner memory");
 }
 function mergeSourceBearsRows(kind) {