@minnowdb/core 0.7.5 → 0.7.7

package/dist/engine/client.d.ts

@@ -1,7 +1,7 @@
 import { type CompactionJobRecord, type GarbageCollectionJobRecord, type StorageIntegrityMode, type StorageIntegrityReport, type StorageStats, type InterruptedSnapshotImport, type InterruptedSnapshotImportAbortResult } from "../storage/types.js";
 import { type BatchRow, type InsertBatchInput } from "./batch.js";
 import type { Catalog } from "./catalog.js";
-import type { BatchValue, BufferPoolStats, StagedWriteResult, BufferedFlushResult, BufferedWriterOptions, CancelCompactionJobResult, CollectGarbageOptions, CollectGarbageStepOptions, CompactTableOptions, CompactTableResult, CompactTableStepOptions, CompactionJobProgress, CreateTableInput, DatabaseRow, MigrateOptions, DeleteBatchInput, DeleteBatchResult, ExecuteOptions, ExecuteResult, GarbageCollectionProgress, GarbageCollectionResult, MaintenanceStatus, InsertBatchResult, QueryOptions, QueryExecutionStats, QueryCursorOptions, QuerySpillCleanupOptions, QuerySpillCleanupResult, ReadTableOptions, RunStatementOptions, SnapshotExportOptions, SnapshotImportOptions, TableDefinition, UpdateBatchInput, UpdateBatchResult, UpsertBatchResult, UpsertOptions, VisibleSegmentPage, VisibleSegmentPageOptions } from "./database.js";
+import type { BatchValue, BufferPoolStats, StagedWriteResult, StagedUpsertResult, BufferedFlushResult, BufferedWriterOptions, CancelCompactionJobResult, CollectGarbageOptions, CollectGarbageStepOptions, CompactTableOptions, CompactTableResult, CompactTableStepOptions, CompactionJobProgress, CreateTableInput, DatabaseRow, MigrateOptions, DeleteBatchInput, DeleteBatchResult, ExecuteOptions, ExecuteResult, GarbageCollectionProgress, GarbageCollectionResult, MaintenanceStatus, InsertBatchResult, QueryOptions, QueryExecutionStats, QueryCursorOptions, QuerySpillCleanupOptions, QuerySpillCleanupResult, ReadTableOptions, RunStatementOptions, SnapshotExportOptions, SnapshotImportOptions, TableDefinition, UpdateBatchInput, UpdateBatchResult, UpsertBatchResult, UpsertOptions, VisibleSegmentPage, VisibleSegmentPageOptions } from "./database.js";
 import type { LiveQueryInput, LiveQueryInvalidation, LiveQueryObserveOptions, LiveQueryStats, LiveQuerySubscribeOptions } from "./live.js";
 import type { CompiledQuery, CompiledStatement, QueryResult, QueryValue } from "./query.js";
 import type { AnyTable, SchemaDefinition } from "./schema.js";
@@ -210,7 +210,7 @@ export interface ClientWriteSession {
     query(sql: string, options?: QueryOptions): Promise<QueryResult>;
     execute(sql: string, params?: readonly QueryValue[]): Promise<ExecuteResult>;
     insertBatch(tableName: string, input: InsertBatchInput): Promise<StagedWriteResult>;
-    upsertBatch(tableName: string, input: InsertBatchInput): Promise<StagedWriteResult>;
+    upsertBatch(tableName: string, input: InsertBatchInput, options?: UpsertOptions): Promise<StagedUpsertResult>;
     updateBatch(tableName: string, input: UpdateBatchInput): Promise<StagedWriteResult>;
     deleteBatch(tableName: string, input: DeleteBatchInput): Promise<StagedWriteResult>;
 }

package/dist/engine/client.js

@@ -290,7 +290,12 @@ class MinnowDatabaseClient {
   }
   async write(action) {
     const opened = await this.#call("writeOpen", []);
-    const stage = (op, tableName, input) => this._invoke(opened.handleId, "stage", [op, tableName, input]);
+    const stage = (op, tableName, input, options) => this._invoke(opened.handleId, "stage", [
+      op,
+      tableName,
+      input,
+      options
+    ]);
     const session = {
       query: async (sql, options = {}) => {
         const { signal, onStats, ...wireOptions } = options;
@@ -298,7 +303,7 @@ class MinnowDatabaseClient {
       },
       execute: (sql, params) => this._invoke(opened.handleId, "execute", params === void 0 ? [sql] : [sql, params]),
       insertBatch: (tableName, input) => stage("insertBatch", tableName, input),
-      upsertBatch: (tableName, input) => stage("upsertBatch", tableName, input),
+      upsertBatch: (tableName, input, options) => stage("upsertBatch", tableName, input, options),
       updateBatch: (tableName, input) => stage("updateBatch", tableName, input),
       deleteBatch: (tableName, input) => stage("deleteBatch", tableName, input)
     };

package/dist/engine/database.d.ts

@@ -174,6 +174,10 @@ export interface StagedWriteResult {
     /** Values filled by defaults or auto-increment while the rows were staged. */
     generatedColumns?: Record<string, QueryValue[]>;
 }
+export interface StagedUpsertResult extends StagedWriteResult {
+    /** Input rows rejected by `conflictWhere`; always 0 without it. */
+    skippedRowCount: number;
+}
 /**
  * The scope handed to `write()`: every mutation stages into one transaction and publishes
  * as one commit — all of it or none of it, in every tab. Reads observe the pre-scope snapshot
@@ -194,7 +198,7 @@ export interface WriteSession {
     /** Runs a SELECT, INSERT, UPDATE, or DELETE inside this write scope. */
     execute(sql: string, params?: readonly QueryValue[]): Promise<ExecuteResult>;
     insertBatch(tableName: string, input: InsertBatchInput): Promise<StagedWriteResult>;
-    upsertBatch(tableName: string, input: InsertBatchInput): Promise<StagedWriteResult>;
+    upsertBatch(tableName: string, input: InsertBatchInput, options?: UpsertOptions): Promise<StagedUpsertResult>;
     updateBatch(tableName: string, input: UpdateBatchInput): Promise<StagedWriteResult>;
     deleteBatch(tableName: string, input: DeleteBatchInput): Promise<StagedWriteResult>;
 }

package/dist/engine/database.js

@@ -13,7 +13,7 @@ import { estimateCompactionRowsPerOutput, planAlignedWriteBlockRanges as writeBl
 import { MAX_CACHEABLE_TEXT_CHARACTERS, MAX_SQL_PARAMETERS } from "./cache-limits.js";
 import { fillColumnDefaults, patchAutoIncrementValues } from "./defaults.js";
 import { cachedQueryTerms, FTS_TOKENIZER_VERSION, renderDocumentValue, tokenize as ftsTokenize } from "./fts.js";
-import { boundedMaintenanceBatchItems, simpleDataTypes, floorWholeNumberProduct, BlockReadBatchTooLargeError, validateColumnDefault, validateEnumValues, secondaryIndexColumnIds, secondaryIndexDirections, secondaryUniqueKeyNamespace, CompactionJobConflictError, CompactionBacklogError, GarbageCollectionJobConflictError, MAX_LEVEL_ZERO_SEGMENTS, MAX_FTS_CANDIDATE_ROW_IDS, MAX_FTS_POSTING_TERM_CHARACTERS, MAX_FTS_POSTINGS_PER_CHUNK, MAX_FTS_POSTING_ROW_IDS_PER_CHUNK, MAX_INDEXED_STRING_CHARACTERS, MAX_CATALOG_NAME_CHARACTERS, MAX_MANIFEST_BLOCK_PRESENCE_IDS, MAX_MAINTENANCE_BATCH_ITEMS, MAX_STORAGE_BULK_READ_ITEMS, MAX_TRANSACTION_STAGE_BLOCKS, MAX_TRANSACTION_STAGE_BYTES, MAX_SNAPSHOT_SESSION_TTL_MS, MAX_POSTING_BUILD_TTL_MS, MAX_TEMP_OWNER_TTL_MS, MAX_SNAPSHOT_FRAME_BATCH_BYTES, MAX_SNAPSHOT_FRAME_BATCH_ITEMS, MAX_SNAPSHOT_METADATA_BATCH_BYTES, SNAPSHOT_FRAME_KINDS, validateStorageId, SnapshotManifestMissingError, SchemaConflictError, TableInUseError, TableRecordConflictError, TransactionRecordConflictError, UniqueKeyConflictError, UniqueIndexCoverageError, WriteConflictError } from "../storage/types.js";
+import { boundedMaintenanceBatchItems, simpleDataTypes, floorWholeNumberProduct, BlockReadBatchTooLargeError, validateColumnDefault, validateEnumValues, secondaryIndexColumnIds, secondaryIndexDirections, secondaryUniqueKeyNamespace, CompactionJobConflictError, CompactionBacklogError, GarbageCollectionJobConflictError, MAX_LEVEL_ZERO_SEGMENTS, MAX_FTS_CANDIDATE_ROW_IDS, MAX_FTS_POSTING_TERM_CHARACTERS, MAX_FTS_POSTINGS_PER_CHUNK, MAX_FTS_POSTING_ROW_IDS_PER_CHUNK, MAX_INDEXED_STRING_CHARACTERS, MAX_CATALOG_NAME_CHARACTERS, MAX_MANIFEST_BLOCK_PRESENCE_IDS, MAX_MAINTENANCE_BATCH_ITEMS, MAX_STORAGE_BULK_READ_ITEMS, MAX_BLOCK_READ_BATCH_BYTES, MAX_TRANSACTION_STAGE_BLOCKS, MAX_TRANSACTION_STAGE_BYTES, MAX_SNAPSHOT_SESSION_TTL_MS, MAX_POSTING_BUILD_TTL_MS, MAX_TEMP_OWNER_TTL_MS, MAX_SNAPSHOT_FRAME_BATCH_BYTES, MAX_SNAPSHOT_FRAME_BATCH_ITEMS, MAX_SNAPSHOT_METADATA_BATCH_BYTES, SNAPSHOT_FRAME_KINDS, validateStorageId, SnapshotManifestMissingError, SchemaConflictError, TableInUseError, TableRecordConflictError, TransactionRecordConflictError, UniqueKeyConflictError, UniqueIndexCoverageError, WriteConflictError } from "../storage/types.js";
 import { decodeSnapshotFrameStream, encodeSnapshotFrameStreamFooter, encodeSnapshotFrameStreamHeader, extendSnapshotFrameStreamChecksum, snapshotFrameEnvelopeParts, snapshotFrameStreamHeaderIdentity } from "../storage/snapshot.js";
 import { Snapshot, TransactionManager } from "../transactions/index.js";
 import { applyWindowFunctions, bindPendingSelectShapes, bindPlanParameters, bindStatementParameters, DUAL_TABLE, dualTableRows, blockHasSubqueries, combineUnionResults, compileCheckExpression, compileQuery, hasAggregate, createRecursiveCteState, annotateAvgArgumentScales, compileStatement, comparisonHolds, createPreparedColumnarQuery, evaluateJoinedRowExpression, evaluateRowExpression, externalizeQueryResult, expressionColumnNames, inferBlockSchema, inferResultColumnDomains, isDeferredInsertExpression, isDefaultInsertValue, referencedColumns, childExpressions, expandFtsColumns, expandNaturalJoins, expandSourceColumnAliases, expandViewSources, forEachBlockExpression, forEachNestedBlock, planContainsFts, planHasNaturalJoins, planHasSourceColumnAliases, planReadsTable, planReadsViews, planReadsBeyondSingleScan, projectResultColumns, queryResultNeedsExternalization, resolveStatementDatetimes, subqueryResolutionSteps, topLevelFtsMatchConjuncts, transparentProjectionSource, unknownColumnDomains, validateDefaultExpression, windowOutputDomain, windowOutputType, volatileScalarFunctionNames, annotatePlanIntegerDivision, clonePlanTree, planMayHaveIntegerDivision, foldIdentifierCase, extendGroupByWithKeyDependents, expandRowReferences } from "./query.js";
@@ -78,6 +78,7 @@ const AUTO_COLLECT_RETRY_MIN_MS = 1e3;
 const AUTO_COLLECT_RETRY_MAX_MS = 6e4;
 const AUTO_COMPACT_STEP_BLOCKS = 4;
 const AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS = 256;
+const AUTO_COMPACT_BACKPRESSURE_LEVEL_ZERO_SEGMENTS = 2 * AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS;
 const AUTO_COMPACT_RETRY_MIN_MS = 250;
 const AUTO_COMPACT_RETRY_MAX_MS = 6e4;
 const MAX_COMPACTION_PUBLICATION_CONFLICTS = 64;
@@ -1612,9 +1613,7 @@ class MinnowDatabase {
     const { result, batch, generated, autoIncrement } = completed;
     collectAutoIncrementGenerated(batch, generated, autoIncrement);
     if (result.acceptedRowIndexes.length !== result.requestedRowCount) {
-      for (const [name, values] of generated) {
-        generated.set(name, result.acceptedRowIndexes.map((index) => values[index] ?? null));
-      }
+      remapGeneratedColumns(generated, result.acceptedRowIndexes);
     }
     return {
       tableName: result.tableName,
@@ -2092,30 +2091,21 @@ class MinnowDatabase {
       const normalizedConflictWhere = kind === "upsert" && conflictWhere !== void 0 ? normalizeUpsertConflictWhere(table, conflictWhere) : void 0;
       const upsertKeyColumn = kind === "upsert" ? getUniqueKeyColumn(table) : void 0;
       if (upsertKeyColumn !== void 0) {
-        upsertFirings = await this.#upsertTriggerFirings(table, upsertKeyColumn, input, rowCount, (sql, params) => this.#sessionQuery(transaction, sql, { params }), normalizedConflictWhere, normalizedConflictWhere !== void 0);
+        upsertFirings = await this.#upsertTriggerFirings(table, upsertKeyColumn, input, rowCount, (sql, params) => this.#sessionQuery(transaction, sql, { params }), normalizedConflictWhere);
       }
       if (normalizedConflictWhere !== void 0) {
         if (upsertFirings === void 0) {
           throw new Error("Upsert conflict classification is missing");
         }
-        const classifiedFirings = upsertFirings;
-        acceptedRowIndexes = [...classifiedFirings.inserts, ...classifiedFirings.updates].sort((left, right) => left - right);
-        skippedRowCount = classifiedFirings.skipped.length;
-        input = selectBatchRows(input, acceptedRowIndexes);
-        rowCount = acceptedRowIndexes.length;
+        counts = { inserted: upsertFirings.inserts.length, updated: upsertFirings.updates.length };
+        const filtered = applyUpsertConflictFilter(input, rowCount, upsertFirings);
+        acceptedRowIndexes = filtered.acceptedRowIndexes;
+        skippedRowCount = filtered.skippedRowCount;
+        input = filtered.batch;
+        rowCount = filtered.rowCount;
+        upsertFirings = filtered.firings;
         logicalBytes = estimateBatchBytes(input);
         resolvedKeys = batchKeys(table, input);
-        counts = {
-          inserted: classifiedFirings.inserts.length,
-          updated: classifiedFirings.updates.length
-        };
-        const acceptedPosition = new Map(acceptedRowIndexes.map((original, position) => [original, position]));
-        upsertFirings = {
-          inserts: classifiedFirings.inserts.map((index) => acceptedPosition.get(index) ?? -1),
-          updates: classifiedFirings.updates.map((index) => acceptedPosition.get(index) ?? -1),
-          skipped: [],
-          oldImages: acceptedRowIndexes.map((index) => classifiedFirings.oldImages[index])
-        };
         if (rowCount === 0) {
           await transaction.abort();
           const version = transaction.snapshotVersion;
@@ -3973,23 +3963,38 @@ class MinnowDatabase {
       return;
     const hintVersion = transaction?.snapshotVersion ?? await this.store.getCurrentManifestVersion();
     const hint = this.#autoCompactionHints.get(table.id);
-    if (hint?.version === hintVersion && hint.levelZero < MAX_LEVEL_ZERO_SEGMENTS)
-      return;
-    let segments = await this.#currentVisibleSegments(table);
-    let levelZero = 0;
-    for (const segment of segments) {
-      if (segment.level === 0)
-        levelZero += 1;
+    let visible;
+    let levelZero;
+    if (hint?.version === hintVersion) {
+      ({ visible, levelZero } = hint);
+    } else {
+      const segments = await this.#currentVisibleSegments(table);
+      visible = segments.length;
+      levelZero = countLevelZeroSegments(segments);
     }
-    if (levelZero < MAX_LEVEL_ZERO_SEGMENTS)
+    if (levelZero < AUTO_COMPACT_BACKPRESSURE_LEVEL_ZERO_SEGMENTS)
+      return;
+    const atCeiling = levelZero >= MAX_LEVEL_ZERO_SEGMENTS;
+    if (!atCeiling && this.#autoCompactionBackoff.get(table.id)?.retryTimer !== void 0)
       return;
     await this.#publishPendingCompactions();
-    await this.compactTableStep(table.name, {
-      maxBlocks: AUTO_COMPACT_STEP_BLOCKS,
-      maxLevel0Segments: AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS
-    });
-    segments = await this.#currentVisibleSegments(table);
-    levelZero = segments.reduce((count, segment) => count + (segment.level === 0 ? 1 : 0), 0);
+    try {
+      const progress = await this.compactTableStep(table.name, {
+        maxBlocks: AUTO_COMPACT_STEP_BLOCKS,
+        maxLevel0Segments: AUTO_COMPACT_MAX_LEVEL_ZERO_SEGMENTS
+      });
+      if (!atCeiling && progress.result !== null && !progress.result.compacted) {
+        this.#backOffAutoCompaction(table.id, visible);
+      }
+    } catch (error) {
+      if (atCeiling)
+        throw error;
+      this.#backOffAutoCompaction(table.id, visible);
+      return;
+    }
+    if (!atCeiling)
+      return;
+    levelZero = countLevelZeroSegments(await this.#currentVisibleSegments(table));
     if (levelZero >= MAX_LEVEL_ZERO_SEGMENTS) {
       throw new CompactionBacklogError(table.name, levelZero, MAX_LEVEL_ZERO_SEGMENTS);
     }
@@ -4299,9 +4304,9 @@ class MinnowDatabase {
       }
     }
   }
-  async #upsertTriggerFirings(table, keyColumn, batch, rowCount, readRows, conflictWhere, force = false) {
+  async #upsertTriggerFirings(table, keyColumn, batch, rowCount, readRows, conflictWhere) {
     const fires = (table.triggers ?? []).some((trigger) => trigger.event === "insert" || trigger.event === "update");
-    if (!fires && !force || rowCount === 0)
+    if (!fires && conflictWhere === void 0 || rowCount === 0)
       return void 0;
     const quote = (name) => `"${name.replaceAll('"', '""')}"`;
     const keyValues = batch.columns[keyColumn.name] ?? [];
@@ -4415,8 +4420,15 @@ class MinnowDatabase {
     if (target.uniqueKeyColumnId !== void 0) {
       throw new TypeError(`Trigger bodies insert into keyless tables only: ${target.name}`);
     }
+    let columns = compiled.columns;
+    if (columns.length === 0 && compiled.defaultValues !== true) {
+      columns = visibleTableColumns(target).map((column) => column.name);
+      if (derivedRows.some((row) => row.length !== columns.length)) {
+        throw new TypeError(`Trigger body INSERT must match the ${String(columns.length)} visible columns of ${target.name}`);
+      }
+    }
     const statementNow = this.#now();
-    const materialized = await this.#materializeInsertExpressions({ ...compiled, rows: derivedRows }, statementNow);
+    const materialized = await this.#materializeInsertExpressions({ ...compiled, columns, rows: derivedRows }, statementNow);
     const input = insertStatementBatch(materialized);
     coerceSqlWriteBatch(target, input);
     const filled = await fillColumnDefaults(target, input, (sql) => this.#evaluateDefaultExpression(sql, statementNow), derivedRows.length);
@@ -4629,10 +4641,10 @@ class MinnowDatabase {
         staged += 1;
         return guarded(() => this.#sessionInsert(transaction, tableName, input, "insert"));
       },
-      upsertBatch: async (tableName, input) => {
+      upsertBatch: async (tableName, input, options2) => {
         open();
         staged += 1;
-        return guarded(() => this.#sessionInsert(transaction, tableName, input, "upsert"));
+        return guarded(() => this.#sessionInsert(transaction, tableName, input, "upsert", options2));
       },
       updateBatch: async (tableName, input) => {
         open();
@@ -4866,10 +4878,12 @@ class MinnowDatabase {
     }
     return { added, removed };
   }
-  async #sessionInsert(transaction, tableName, input, kind, cascadeBudget = 1) {
+  async #sessionInsert(transaction, tableName, input, kind, options, cascadeBudget = 1) {
     const table = await this.#findTable(tableName);
-    await this.#assertCompactionCapacity(table, transaction);
-    const { batch, generated, autoIncrement, rowCount } = await this.#fillDefaults(table, input);
+    const filled = await this.#fillDefaults(table, input);
+    const { generated, autoIncrement } = filled;
+    let batch = filled.batch;
+    let rowCount = filled.rowCount;
     if (autoIncrement !== void 0 && autoIncrement.missingIndexes.length > 0) {
       const values = await this.store.reserveAutoIncrement(table.id, autoIncrement.column.id, autoIncrement.missingIndexes.length, autoIncrement.atLeast);
       patchAutoIncrementValues(batch, autoIncrement, values);
@@ -4878,6 +4892,36 @@ class MinnowDatabase {
         validateValue(autoIncrement.column, patched[rowIndex] ?? null, rowIndex);
       }
     }
+    const sessionUpsertKeyColumn = kind === "upsert" ? getUniqueKeyColumn(table) : void 0;
+    const normalizedConflictWhere = kind === "upsert" && options?.conflictWhere !== void 0 ? normalizeUpsertConflictWhere(table, options.conflictWhere) : void 0;
+    if (normalizedConflictWhere !== void 0 && sessionUpsertKeyColumn === void 0) {
+      throw new TypeError(`Table needs a unique key before it can be upserted: ${table.name}`);
+    }
+    let sessionUpsertFirings = sessionUpsertKeyColumn === void 0 ? void 0 : await this.#upsertTriggerFirings(table, sessionUpsertKeyColumn, batch, rowCount, (sql, params) => this.#sessionQuery(transaction, sql, { params }), normalizedConflictWhere);
+    let skippedRowCount = 0;
+    if (normalizedConflictWhere !== void 0) {
+      if (sessionUpsertFirings === void 0) {
+        throw new Error("Upsert conflict classification is missing");
+      }
+      const filtered = applyUpsertConflictFilter(batch, rowCount, sessionUpsertFirings);
+      skippedRowCount = filtered.skippedRowCount;
+      if (filtered.rowCount !== rowCount) {
+        remapGeneratedColumns(generated, filtered.acceptedRowIndexes);
+      }
+      batch = filtered.batch;
+      rowCount = filtered.rowCount;
+      sessionUpsertFirings = filtered.firings;
+      if (rowCount === 0) {
+        return {
+          tableName: table.name,
+          segmentId: null,
+          rowCount: 0,
+          skippedRowCount,
+          ...generated.size === 0 ? {} : { generatedColumns: Object.fromEntries(generated) }
+        };
+      }
+    }
+    await this.#assertCompactionCapacity(table, transaction);
     const keys = batchKeys(table, batch);
     if (keys !== void 0) {
       transaction.setUniqueKeyChanges({
@@ -4889,8 +4933,6 @@ class MinnowDatabase {
     await this.#assertForeignKeysPresent(table, (column) => batch.columns[column] ?? [], (sql, params) => this.#sessionQuery(transaction, sql, { params }), transaction);
     const rowIds = await this.store.reserveRowIds(table.id, rowCount);
     const insertValueAt = (source, column, rowIndex) => source === "new" ? batch.columns[column]?.[rowIndex] ?? null : null;
-    const sessionUpsertKeyColumn = kind === "upsert" ? getUniqueKeyColumn(table) : void 0;
-    const sessionUpsertFirings = sessionUpsertKeyColumn === void 0 ? void 0 : await this.#upsertTriggerFirings(table, sessionUpsertKeyColumn, batch, rowCount, (sql, params) => this.#sessionQuery(transaction, sql, { params }));
     stageSecondaryUniqueInsertChanges(transaction, table, batch, kind === "upsert" ? sessionUpsertFirings?.oldImages : void 0);
     if (kind === "insert") {
       await this.#stageTriggerDerivedInserts(transaction, table, "insert", rowCount, insertValueAt, "before", cascadeBudget);
@@ -4908,6 +4950,7 @@ class MinnowDatabase {
       tableName: table.name,
       segmentId,
       rowCount,
+      skippedRowCount,
       ...generated.size === 0 ? {} : { generatedColumns: Object.fromEntries(generated) }
     };
   }
@@ -9275,7 +9318,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
       }
       const priorAttemptOutputStoredBytes = await this.#priorCompactionAttemptOutputStoredBytes(table.id, baseJobId);
       const maximumOutputStoredBytes = Math.max(0, floorWholeNumberProduct(selection.level0SourceStoredBytes, maxWriteAmplification, "Compaction maximum output stored bytes") - priorAttemptOutputStoredBytes);
-      const plannedOutputStoredBytesUpperBound = await this.#plannedPhysicalOutputStoredBytesUpperBound(rewritePlan, snapshot);
+      const plannedOutputStoredBytesUpperBound = await this.#plannedPhysicalOutputStoredBytesUpperBound(rewritePlan, memoryBudgetBytes, snapshot);
       levelTwoBudget = {
         outputPartitionOrdinal,
         maxWriteAmplification,
@@ -9731,17 +9774,15 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
     const columnIndexById = new Map(table.columns.map((column, index) => [column.id, index]));
     const touched = /* @__PURE__ */ new Set();
     const deltaKeys = /* @__PURE__ */ new Map();
-    for (const segment of segments) {
-      if (!mergeSourceReferencesKeys(segment.kind))
-        continue;
-      const keys = [];
-      await this.#forEachMergeSourceKey(segment, keyColumn, snapshot, (value) => {
-        const key = overlayKeyOf(keyColumn.type, value);
-        keys.push(key);
-        touched.add(key);
-      });
-      deltaKeys.set(segment.segmentId, keys);
-    }
+    const deltas = segments.filter((segment) => mergeSourceReferencesKeys(segment.kind));
+    for (const segment of deltas)
+      deltaKeys.set(segment.segmentId, []);
+    const unusedBudgetBytes = memoryBudgetBytes - plannerMemoryBytes;
+    await this.#forEachMergeSourceKey(deltas, keyColumn, unusedBudgetBytes, snapshot, (segment, value) => {
+      const key = overlayKeyOf(keyColumn.type, value);
+      deltaKeys.get(segment.segmentId)?.push(key);
+      touched.add(key);
+    });
     let slotCount = 0;
     for (const segment of segments) {
       if (mergeSourceBearsRows(segment.kind))
@@ -9813,7 +9854,7 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
       if (keys !== void 0) {
         keys.forEach(visit);
       } else if (touched.size > 0) {
-        await this.#forEachMergeSourceKey(segment, keyColumn, snapshot, (value, rowIndex) => {
+        await this.#forEachMergeSourceKey([segment], keyColumn, unusedBudgetBytes, snapshot, (_, value, rowIndex) => {
           visit(overlayKeyOf(keyColumn.type, value), rowIndex);
         });
       }
@@ -9850,28 +9891,48 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
     }
     return { ...output.finish(), sourceOutputRowStarts };
   }
-  async #forEachMergeSourceKey(segment, column, snapshot, action) {
-    const planned = segment.columns.find((candidate) => candidate.columnId === column.id);
-    if (planned === void 0) {
-      throw new Error(`Mutation segment has no key column: ${segment.segmentId}`);
+  async #forEachMergeSourceKey(segments, column, unusedBudgetBytes, snapshot, action) {
+    const blocks = [];
+    let largestEncodedBytes = 0;
+    for (const segment of segments) {
+      const planned = segment.columns.find((candidate) => candidate.columnId === column.id);
+      if (planned === void 0) {
+        throw new Error(`Mutation segment has no key column: ${segment.segmentId}`);
+      }
+      for (const source of planned.sourceBlocks) {
+        blocks.push({
+          blockId: source.blockId,
+          storedBytes: source.storedBytes,
+          encodedBytes: source.encodedBytes,
+          segment,
+          source
+        });
+        largestEncodedBytes = Math.max(largestEncodedBytes, source.encodedBytes);
+      }
     }
-    let rowIndex = 0;
-    for (const source of planned.sourceBlocks) {
-      await this.#renewInternalLeaseIfNeeded(snapshot);
-      const bytes = await this.store.getBlock(source.blockId);
-      if (bytes === void 0)
-        throw new Error(`Compaction source block is missing: ${source.blockId}`);
+    const limitBytes = safeWholeNumberSum([
+      Math.max(0, unusedBudgetBytes),
+      safeWholeNumberProduct(largestEncodedBytes, MERGE_PLANNER_DECODED_KEY_BLOCK_FACTOR, "Mutation compaction key decode memory")
+    ], "Mutation compaction key decode memory");
+    const rowIndexes = /* @__PURE__ */ new Map();
+    await this.#decodeCompactionBlocks(blocks, { limitBytes, retainsDecoded: false }, snapshot, async (bytes, { source }) => {
       const description = inspectBlock(bytes);
       if (bytes.byteLength !== source.storedBytes || description.encodedLength !== source.encodedBytes || description.checksum !== source.checksum || description.rowCount !== source.rowCount || description.type !== column.type) {
         throw new Error(`Compaction source block differs from its plan: ${source.blockId}`);
       }
-      for (const value of (await decodeBlock(bytes)).column.values) {
-        action(value, rowIndex);
+      return (await decodeBlock(bytes)).column.values;
+    }, (values, { segment }) => {
+      let rowIndex = rowIndexes.get(segment.segmentId) ?? 0;
+      for (const value of values) {
+        action(segment, value, rowIndex);
         rowIndex += 1;
       }
-    }
-    if (rowIndex !== segment.rowCount) {
-      throw new Error(`Mutation segment key rows differ: ${segment.segmentId}`);
+      rowIndexes.set(segment.segmentId, rowIndex);
+    });
+    for (const segment of segments) {
+      if ((rowIndexes.get(segment.segmentId) ?? 0) !== segment.rowCount) {
+        throw new Error(`Mutation segment key rows differ: ${segment.segmentId}`);
+      }
     }
   }
   async #refinePhysicalOutputWindows(columns, estimatedOutputs, targetBlockBytes, outputCompression, memoryBudgetBytes, snapshot) {
@@ -9930,12 +9991,12 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
     const loaded = await this.#loadPhysicalCompactionRanges(column, output, outputCompression, memoryBudgetBytes, snapshot);
     return measurePhysicalColumnRanges(column.type, loaded.ranges);
   }
-  async #plannedPhysicalOutputStoredBytesUpperBound(plan, snapshot) {
+  async #plannedPhysicalOutputStoredBytesUpperBound(plan, memoryBudgetBytes, snapshot) {
     const layout = physicalRewriteLayout(plan);
     let total = 0;
     for (const output of layout.outputs) {
       for (const column of layout.columns) {
-        const measurement = await this.#measurePhysicalCompactionOutput(column, output, plan.outputCompression, Number.MAX_SAFE_INTEGER, snapshot);
+        const measurement = await this.#measurePhysicalCompactionOutput(column, output, plan.outputCompression, memoryBudgetBytes, snapshot);
         total = safeWholeNumberSum([
           total,
           maximumPhysicalBlockByteLength(measurement.encodedByteLength, measurement.metadata, plan.outputCompression)
@@ -10296,26 +10357,84 @@ ${notes.map((note) => `-- ${note}`).join("\n")}`;
       throw new CompactionMemoryBudgetError(memoryBudgetBytes, memoryBound);
     }
     const outputEnd = safeWholeNumberSum([output.rowStart, output.rowCount], "Compaction output row range");
-    const ranges = [];
-    for (const sourceBlock of overlappingPhysicalSourceRanges(column, output)) {
-      if (snapshot !== void 0)
-        await this.#renewInternalLeaseIfNeeded(snapshot);
-      const bytes = await this.store.getBlock(sourceBlock.blockId);
-      if (bytes === void 0) {
-        throw new Error(`A compaction source block is missing: ${sourceBlock.blockId}`);
-      }
+    const sourceBlocks = overlappingPhysicalSourceRanges(column, output);
+    const distinct = [];
+    const seen = /* @__PURE__ */ new Set();
+    for (const sourceBlock of sourceBlocks) {
+      if (seen.has(sourceBlock.blockId))
+        continue;
+      seen.add(sourceBlock.blockId);
+      distinct.push(sourceBlock);
+    }
+    const decoded = /* @__PURE__ */ new Map();
+    const peakWorkingBytes = await this.#decodeCompactionBlocks(distinct, { limitBytes: memoryBudgetBytes, retainsDecoded: true }, snapshot, async (bytes, sourceBlock) => {
       const description = inspectBlock(bytes);
       if (bytes.byteLength !== sourceBlock.storedBytes || description.encodedLength !== sourceBlock.encodedBytes || description.checksum !== sourceBlock.checksum || description.rowCount !== sourceBlock.sourceBlockRowCount || description.type !== column.type) {
         throw new Error(`A compaction source block differs from its plan: ${sourceBlock.blockId}`);
       }
-      const decoded = await decodePhysicalBlock(bytes);
+      return decodePhysicalBlock(bytes);
+    }, (block, sourceBlock) => {
+      decoded.set(sourceBlock.blockId, block);
+    });
+    const ranges = [];
+    for (const sourceBlock of sourceBlocks) {
+      const block = decoded.get(sourceBlock.blockId);
+      if (block === void 0) {
+        throw new Error(`A compaction source block is missing: ${sourceBlock.blockId}`);
+      }
       const sourceEnd = safeWholeNumberSum([sourceBlock.outputRowStart, sourceBlock.rowCount], "Compaction source row range");
       const start = sourceBlock.sourceRowStart + Math.max(output.rowStart, sourceBlock.outputRowStart) - sourceBlock.outputRowStart;
       const end = sourceBlock.sourceRowStart + Math.min(outputEnd, sourceEnd) - sourceBlock.outputRowStart;
-      const slice = slicePhysicalColumn(decoded.column, start, end);
+      const slice = slicePhysicalColumn(block.column, start, end);
       ranges.push({ column: slice, start: 0, end: slice.rowCount });
     }
-    return { ranges, peakWorkingBytes: memoryBound };
+    return { ranges, peakWorkingBytes: Math.max(memoryBound, peakWorkingBytes) };
+  }
+  async #decodeCompactionBlocks(blocks, budget, snapshot, decode, consume) {
+    let retainedBytes = 0;
+    let peakBytes = 0;
+    let start = 0;
+    while (start < blocks.length) {
+      let end = start;
+      let transientBytes = 0;
+      let storedBytes = 0;
+      for (; end < blocks.length && end - start < MAX_STORAGE_BULK_READ_ITEMS; end += 1) {
+        const block = blocks[end];
+        if (block === void 0)
+          break;
+        const blockTransientBytes = safeWholeNumberSum([
+          safeWholeNumberProduct(block.storedBytes, 2, "Compaction decode memory"),
+          safeWholeNumberProduct(block.encodedBytes, 2, "Compaction decode memory")
+        ], "Compaction decode memory");
+        if (end > start && (transientBytes + blockTransientBytes > budget.limitBytes - retainedBytes || storedBytes + block.storedBytes > MAX_BLOCK_READ_BATCH_BYTES)) {
+          break;
+        }
+        transientBytes += blockTransientBytes;
+        storedBytes += block.storedBytes;
+      }
+      const group = blocks.slice(start, end);
+      if (snapshot !== void 0)
+        await this.#renewInternalLeaseIfNeeded(snapshot);
+      const bytes = await this.#readBlockWindow(group.map((block) => block.blockId));
+      const decoded = await Promise.all(group.map((block, index) => {
+        const stored = bytes[index];
+        if (stored === void 0) {
+          throw new Error(`A compaction source block is missing: ${block.blockId}`);
+        }
+        return decode(stored, block);
+      }));
+      peakBytes = Math.max(peakBytes, retainedBytes + transientBytes);
+      for (const [index, block] of group.entries()) {
+        const value = decoded[index];
+        if (value === void 0)
+          throw new Error("Compaction decode lost a block");
+        consume(value, block);
+        if (budget.retainsDecoded)
+          retainedBytes += block.encodedBytes;
+      }
+      start = end;
+    }
+    return peakBytes;
   }
   async #loadCompactionSources(job) {
     const segments = await Promise.all(job.sourceSegmentIds.map((id) => this.store.getSegment(id)));
@@ -14152,6 +14271,49 @@ function normalizeColumnLogicalValue(column, value) {
   }
   return value;
 }
+function mergeAscendingIndexes(left, right) {
+  const merged = [];
+  let leftIndex = 0;
+  let rightIndex = 0;
+  while (leftIndex < left.length || rightIndex < right.length) {
+    const a = left[leftIndex];
+    const b = right[rightIndex];
+    if (b === void 0 || a !== void 0 && a <= b) {
+      if (a !== void 0)
+        merged.push(a);
+      leftIndex += 1;
+    } else {
+      merged.push(b);
+      rightIndex += 1;
+    }
+  }
+  return merged;
+}
+function applyUpsertConflictFilter(batch, rowCount, classified) {
+  const acceptedRowIndexes = mergeAscendingIndexes(classified.inserts, classified.updates);
+  const skippedRowCount = classified.skipped.length;
+  if (acceptedRowIndexes.length === rowCount) {
+    return { batch, rowCount, skippedRowCount, acceptedRowIndexes, firings: classified };
+  }
+  const acceptedPosition = new Map(acceptedRowIndexes.map((original, position) => [original, position]));
+  return {
+    batch: selectBatchRows(batch, acceptedRowIndexes),
+    rowCount: acceptedRowIndexes.length,
+    skippedRowCount,
+    acceptedRowIndexes,
+    firings: {
+      inserts: classified.inserts.map((index) => acceptedPosition.get(index) ?? -1),
+      updates: classified.updates.map((index) => acceptedPosition.get(index) ?? -1),
+      skipped: [],
+      oldImages: acceptedRowIndexes.map((index) => classified.oldImages[index])
+    }
+  };
+}
+function remapGeneratedColumns(generated, acceptedRowIndexes) {
+  for (const [name, values] of generated) {
+    generated.set(name, acceptedRowIndexes.map((index) => values[index] ?? null));
+  }
+}
 function selectBatchRows(input, indexes) {
   return {
     columns: Object.fromEntries(Object.entries(input.columns).map(([name, values]) => [
@@ -14431,6 +14593,14 @@ function autoCompactionHint(version, segments) {
     deltas
   };
 }
+function countLevelZeroSegments(segments) {
+  let levelZero = 0;
+  for (const segment of segments) {
+    if (segment.level === 0)
+      levelZero += 1;
+  }
+  return levelZero;
+}
 function autoCompactionDueHint(hint) {
   return hint.levelZero >= AUTO_COMPACT_SCAN_SEGMENTS || hint.deltas >= AUTO_COMPACT_DELTA_SEGMENTS;
 }
@@ -15881,6 +16051,7 @@ function sourceOrderTuple(segment, transactions, label) {
   };
 }
 const MERGE_PLANNER_KEY_BYTES = 96;
+const MERGE_PLANNER_DECODED_KEY_BLOCK_FACTOR = 4;
 function planLinearOutputPartitions(totalRows, partitionRows, firstOrder, nextOrder) {
   const count = Math.max(1, Math.ceil(totalRows / partitionRows));
   const orders = fractionalLogicalOrders(firstOrder, nextOrder, count);
@@ -15996,7 +16167,6 @@ function mergePlannerMemoryBound(table, segments, keyColumnId) {
   const PATCH_ROW_BYTES = 64;
   const PATCH_CELL_BYTES = 48;
   const RANGE_BYTES = 80;
-  const DECODED_KEY_BLOCK_FACTOR = 4;
   let slotRows = 0;
   let deltaKeys = 0;
   let patchRows = 0;
@@ -16027,7 +16197,7 @@ function mergePlannerMemoryBound(table, segments, keyColumnId) {
       safeWholeNumberProduct(columns, PATCH_CELL_BYTES, "Mutation patch cells")
     ], "Mutation compaction patch row"), "Mutation compaction patches"),
     safeWholeNumberProduct(safeWholeNumberSum([sourceBlocks, safeWholeNumberProduct(patchRows, columns, "Mutation patched cells")], "Mutation compaction ranges"), RANGE_BYTES, "Mutation compaction range bytes"),
-    safeWholeNumberProduct(largestKeyBlockBytes, DECODED_KEY_BLOCK_FACTOR, "Mutation compaction decoded key block")
+    safeWholeNumberProduct(largestKeyBlockBytes, MERGE_PLANNER_DECODED_KEY_BLOCK_FACTOR, "Mutation compaction decoded key block")
   ], "Mutation compaction planner memory");
 }
 function mergeSourceBearsRows(kind) {

package/dist/engine/worker-server.js

@@ -554,10 +554,13 @@ class DatabaseRpcServer {
       return handle.session.execute(sql, params);
     }
     if (method === "stage") {
-      const [op, tableName, input] = args;
+      const [op, tableName, input, options] = args;
       if (!isStageOp(op)) {
         throw new Error(`Unsupported write stage operation: ${String(op)}`);
       }
+      if (op === "upsertBatch") {
+        return handle.session.upsertBatch(tableName, input, options);
+      }
       return handle.session[op](tableName, input);
     }
     if (method === "commit")

package/dist/storage/indexeddb.js

@@ -3919,7 +3919,12 @@ class IndexedDbBlockStore {
         await assertActiveGarbageCollectionMarker(gcStore, current);
       }
       const updated = updateGarbageCollectionPlanningRecord(current, input);
-      await assertGarbageCollectionCandidateProvenanceInTransaction(transaction, updated);
+      await assertGarbageCollectionCandidateProvenanceInTransaction(transaction, {
+        candidateManifestVersions: input.candidateManifestVersions ?? [],
+        candidateSegmentIds: input.candidateSegmentIds ?? [],
+        candidateBlockIds: input.candidateBlockIds ?? [],
+        candidateTransactionIds: input.candidateTransactionIds ?? []
+      });
       gcStore.put(garbageCollectionJobEnvelope(updated), key);
       await transactionDone(transaction);
       return structuredClone(updated);
@@ -9717,15 +9722,15 @@ function assertGenericTransactionUpdateAllowed(record, update) {
     throw new TypeError("Only commitTransaction can set a committed transaction version");
   }
 }
-async function assertGarbageCollectionCandidateProvenanceInTransaction(transaction, job) {
+async function assertGarbageCollectionCandidateProvenanceInTransaction(transaction, candidates) {
   const manifestStore = transaction.objectStore("manifests");
-  for (const version of job.candidateManifestVersions) {
+  for (const version of candidates.candidateManifestVersions) {
     const value = await requestResult(manifestStore.get(version));
     if (value === void 0) {
       throw new Error(`Garbage collection candidate manifest is missing: ${String(version)}`);
     }
   }
-  for (const id of job.candidateTransactionIds) {
+  for (const id of candidates.candidateTransactionIds) {
     const value = await requestResult(transaction.objectStore("transactions").get(id));
     const record = value === void 0 ? void 0 : asTransactionRecord(value);
     if (record === void 0 || record.status !== "aborted" && (record.status !== "committed" || record.committedVersion === null)) {
@@ -9733,10 +9738,10 @@ async function assertGarbageCollectionCandidateProvenanceInTransaction(transacti
     }
   }
   const manifestProvenBlockIds = /* @__PURE__ */ new Set();
-  const manifestValues = await Promise.all(job.candidateBlockIds.map((id) => requestResult(transaction.objectStore("catalog").get(manifestBlockKey(id)))));
+  const manifestValues = await Promise.all(candidates.candidateBlockIds.map((id) => requestResult(transaction.objectStore("catalog").get(manifestBlockKey(id)))));
   for (const [index, value] of manifestValues.entries()) {
     if (value !== void 0) {
-      const id = job.candidateBlockIds[index] ?? "";
+      const id = candidates.candidateBlockIds[index] ?? "";
       asManifestBlockRecord(value, id);
       manifestProvenBlockIds.add(id);
     }
@@ -9757,12 +9762,12 @@ async function assertGarbageCollectionCandidateProvenanceInTransaction(transacti
       return isTerminalCompactionJob(record) && (record.sourceBlockIds.includes(id) || record.outputBlockIds.includes(id));
     });
   };
-  for (const id of job.candidateBlockIds) {
+  for (const id of candidates.candidateBlockIds) {
     if (await blockHasProvenance(id))
       continue;
     throw new Error(`Garbage collection block candidate has no persisted provenance: ${id}`);
   }
-  for (const id of job.candidateSegmentIds) {
+  for (const id of candidates.candidateSegmentIds) {
     const segmentValue = await requestResult(transaction.objectStore("segments").get(id));
     if (segmentValue !== void 0)
       continue;

package/dist/storage/toolkit/record-core.js

@@ -3567,7 +3567,12 @@ class RecordCore {
       throw new GarbageCollectionJobConflictError(input.jobId, input.expectedRevision, current?.revision ?? null);
     }
     const updated = updateGarbageCollectionPlanningRecord(current, input);
-    assertGarbageCollectionCandidateProvenance(updated, this.#manifests, this.#manifestBlocks, this.#segments, this.#transactions, this.#roots);
+    assertGarbageCollectionCandidateProvenance({
+      candidateManifestVersions: input.candidateManifestVersions ?? [],
+      candidateSegmentIds: input.candidateSegmentIds ?? [],
+      candidateBlockIds: input.candidateBlockIds ?? [],
+      candidateTransactionIds: input.candidateTransactionIds ?? []
+    }, this.#manifests, this.#manifestBlocks, this.#segments, this.#transactions, this.#roots);
     this.#garbageCollectionJobs.set(updated.id, updated);
     return cloneRecord(updated);
   }
@@ -5661,23 +5666,23 @@ function assertPendingArtifactsAvailable(transaction, physical, segments, valida
     }
   }
 }
-function assertGarbageCollectionCandidateProvenance(job, manifests, manifestBlocks, segments, transactions, roots) {
-  for (const version of job.candidateManifestVersions) {
+function assertGarbageCollectionCandidateProvenance(candidates, manifests, manifestBlocks, segments, transactions, roots) {
+  for (const version of candidates.candidateManifestVersions) {
     if (!manifests.has(version)) {
       throw new Error(`Garbage collection candidate manifest is missing: ${String(version)}`);
     }
   }
-  for (const id of job.candidateTransactionIds) {
+  for (const id of candidates.candidateTransactionIds) {
     const transaction = transactions.get(id);
     if (transaction === void 0 || transaction.status !== "aborted" && (transaction.status !== "committed" || transaction.committedVersion === null)) {
       throw new Error(`Garbage collection transaction candidate is not terminal: ${id}`);
     }
   }
-  const unprovenBlockId = job.candidateBlockIds.find((id) => !manifestBlocks.has(id) && roots.abortedTransactionBlockCount(id) === 0 && roots.terminalJobBlockCount(id) === 0);
+  const unprovenBlockId = candidates.candidateBlockIds.find((id) => !manifestBlocks.has(id) && roots.abortedTransactionBlockCount(id) === 0 && roots.terminalJobBlockCount(id) === 0);
   if (unprovenBlockId !== void 0) {
     throw new Error(`Garbage collection block candidate has no persisted provenance: ${unprovenBlockId}`);
   }
-  const unprovenSegmentId = job.candidateSegmentIds.find((id) => {
+  const unprovenSegmentId = candidates.candidateSegmentIds.find((id) => {
     return !segments.has(id);
   });
   if (unprovenSegmentId !== void 0) {

package/dist/storage/types.d.ts

@@ -815,6 +815,17 @@ export interface UpdateGarbageCollectionPlanningInput {
     discovery: GarbageCollectionDiscovery;
     updatedAt: string;
 }
+/**
+ * One page's (or one job's full) proposed garbage-collection candidates, checked for persisted
+ * provenance before being accepted. Shared by every adapter's provenance assertion so a candidate
+ * kind added to one cannot be forgotten in another.
+ */
+export interface GarbageCollectionCandidateSet {
+    readonly candidateManifestVersions: readonly number[];
+    readonly candidateSegmentIds: readonly string[];
+    readonly candidateBlockIds: readonly string[];
+    readonly candidateTransactionIds: readonly string[];
+}
 export interface GarbageCollectionJobRecord {
     id: string;
     candidateManifestVersions: number[];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@minnowdb/core",
-  "version": "0.7.5",
+  "version": "0.7.7",
   "description": "A columnar SQL database for the browser: PostgreSQL-style SQL over durable IndexedDB or OPFS data, with no server or WebAssembly module.",
   "license": "MIT",
   "author": "Eric Wilhite",