@harperfast/harper 5.3.0-beta.3 → 5.3.0-beta.4

package/dist/resources/Table.js

@@ -46,6 +46,7 @@ exports.acquireUpdateAttributesLock = acquireUpdateAttributesLock;
 exports.tryUpdateAttributesLock = tryUpdateAttributesLock;
 exports.releaseUpdateAttributesLock = releaseUpdateAttributesLock;
 exports.withUpdateAttributesLock = withUpdateAttributesLock;
+exports.withUpdateAttributesLockNonBlocking = withUpdateAttributesLockNonBlocking;
 exports.ignoreAlreadyDropped = ignoreAlreadyDropped;
 exports.freezeRecord = freezeRecord;
 exports.frozenRecordView = frozenRecordView;
@@ -67,6 +68,7 @@ const recordLock_ts_1 = require("./recordLock.js");
 const nodeName_ts_1 = require("../server/nodeName.js");
 const envMngr = __importStar(require("../utility/environment/environmentManager.js"));
 const transactionBroadcast_ts_1 = require("./transactionBroadcast.js");
+const databaseDropPreparation_ts_1 = require("./databaseDropPreparation.js");
 const hdbError_ts_1 = require("../utility/errors/hdbError.js");
 const signalling = __importStar(require("../utility/signalling.js"));
 const itc_js_1 = require("../server/threads/itc.js");
@@ -83,6 +85,7 @@ const auditStore_ts_1 = require("./auditStore.js");
 const derivedIndexRegistry_ts_1 = require("./derivedIndexRegistry.js");
 const recordLockCoordinator_ts_1 = require("./recordLockCoordinator.js");
 const embedHook_ts_1 = require("./models/embedHook.js");
+const decideHook_ts_1 = require("./models/decideHook.js");
 const common_utils_ts_1 = require("../utility/common_utils.js");
 const RecordEncoder_ts_1 = require("./RecordEncoder.js");
 const write_ts_1 = require("./analytics/write.js");
@@ -98,6 +101,7 @@ const rocksdb_js_1 = require("@harperfast/rocksdb-js");
 const LMDBTransaction_1 = require("./LMDBTransaction");
 const contentTypes_1 = require("../server/serverHelpers/contentTypes");
 const jsonSchemaTypes_ts_1 = require("./jsonSchemaTypes.js");
+const fullTextSchema_ts_1 = require("./fullTextSchema.js");
 const { sortBy } = lodash_1.default;
 const { validateAttribute } = lmdbProcessRows_js_1.default;
 const NULL_WITH_TIMESTAMP = new Uint8Array(9);
@@ -189,8 +193,28 @@ function tryUpdateAttributesLock(rootStore) {
 function releaseUpdateAttributesLock(rootStore) {
     rootStore.unlock(updateAttributesLockKey);
 }
-function withUpdateAttributesLock(rootStore, scopeDescription, callback) {
-    acquireUpdateAttributesLock(rootStore, scopeDescription);
+async function acquireUpdateAttributesLockAsync(rootStore, scopeDescription, timeout = exports.UPDATE_ATTRIBUTES_LOCK_TIMEOUT) {
+    if (rootStore.tryLock(updateAttributesLockKey))
+        return;
+    const startTime = node_perf_hooks_1.performance.now();
+    let waitTime = 1;
+    while (!rootStore.tryLock(updateAttributesLockKey)) {
+        const elapsed = node_perf_hooks_1.performance.now() - startTime;
+        if (elapsed >= timeout) {
+            throw new hdbError_ts_1.UpdateAttributesLockTimeoutError(`Timed out after ${Math.round(elapsed)}ms waiting for the exclusive '${UPDATE_ATTRIBUTES_LOCK}' lock on ${scopeDescription}; the lock holder did not release it before the deadline, so this schema/attribute update cannot proceed`);
+        }
+        await new Promise((resolve) => setTimeout(resolve, Math.min(waitTime, timeout - elapsed)));
+        if (waitTime < 16)
+            waitTime *= 2;
+    }
+    const waited = node_perf_hooks_1.performance.now() - startTime;
+    if (waited >= exports.UPDATE_ATTRIBUTES_LOCK_SLOW_WAIT)
+        try {
+            logger_ts_1.logger.warn?.(`Acquired the exclusive '${UPDATE_ATTRIBUTES_LOCK}' lock on ${scopeDescription} after waiting ${Math.round(waited)}ms`);
+        }
+        catch { }
+}
+function runWithUpdateAttributesLock(rootStore, scopeDescription, callback) {
     try {
         const result = callback();
         if (typeof result?.then === 'function') {
@@ -203,6 +227,15 @@ function withUpdateAttributesLock(rootStore, scopeDescription, callback) {
         releaseUpdateAttributesLock(rootStore);
     }
 }
+function withUpdateAttributesLock(rootStore, scopeDescription, callback) {
+    acquireUpdateAttributesLock(rootStore, scopeDescription);
+    return runWithUpdateAttributesLock(rootStore, scopeDescription, callback);
+}
+function withUpdateAttributesLockNonBlocking(rootStore, scopeDescription, callback) {
+    if (rootStore.tryLock(updateAttributesLockKey))
+        return runWithUpdateAttributesLock(rootStore, scopeDescription, callback);
+    return acquireUpdateAttributesLockAsync(rootStore, scopeDescription).then(() => runWithUpdateAttributesLock(rootStore, scopeDescription, callback));
+}
 // Tolerate a redundant column family drop. Drops are broadcast to every worker
 // thread and each holds its own handle to the same underlying family, so a
 // concurrent worker may already have dropped it; the storage engine reports
@@ -521,7 +554,9 @@ function isPlainOptions(value) {
 });
 function makeTable(options) {
     var _a;
-    const { primaryKey, indices, tableId, tableName, primaryStore, databasePath, databaseName, auditStore, schemaDefined, dbisDB: dbisDb, sealed, splitSegments, replicate, description, hidden, cacheControl, isBranch, fullTextIndexes = [], } = options;
+    const { primaryKey, indices, tableId, tableName, primaryStore, databasePath, databaseName, auditStore, schemaDefined, dbisDB: dbisDb, sealed, splitSegments, replicate, description, hidden, cacheControl, isBranch, fullTextIndexes = [], fullTextIndexGenerations = Object.create(null), fullTextIndexRetirements = [], } = options;
+    const tableRootStore = primaryStore.rootStore;
+    const tableRootPath = tableRootStore.path;
     let { expirationMS: expirationMs, evictionMS: evictionMs, audit, trackDeletes } = options;
     // Set when the TTL exists only on this thread: either application code configured it at runtime, or
     // an isolated application's schema was declared here. Hydrating persisted metadata does not set it:
@@ -548,6 +583,9 @@ function makeTable(options) {
     let hasSourceGet;
     let primaryKeyAttribute;
     let lastEvictionCompletion = Promise.resolve();
+    let recordExpirationCompletion = Promise.resolve();
+    const maintenanceCommits = new Set();
+    let maintenanceClosed = false;
     let droppingTable = false;
     let createdTimeProperty, updatedTimeProperty, expiresAtProperty;
     for (const attribute of attributes) {
@@ -761,9 +799,10 @@ function makeTable(options) {
         }
         return { txnLogKey: version, nodeId };
     }
-    // Canonical-source applies (sourceApply), replay and replication notifications are never shed;
-    // dropping one would advance the source cursor past a write that never landed.
+    // Teardown rejects every producer; source/replay paths only bypass derived-index lag shedding.
     function assertDerivedIndexAdmission(options, transaction) {
+        if ((0, databaseDropPreparation_ts_1.databaseDropPrepared)(tableRootPath) || (0, DatabaseTransaction_ts_1.databaseCommitsSuspended)(tableRootStore))
+            throw new hdbError_ts_1.DatabaseClosingError(databaseName, !transaction?.root && !transaction?.snapshotFree);
         if (options?.isNotification || transaction?.sourceApply || transaction?.isReplay)
             return;
         const reason = (0, derivedIndexRegistry_ts_1.derivedIndexWriteRejection)(auditStore, tableId);
@@ -788,6 +827,17 @@ function makeTable(options) {
         error.statusCode = 404;
         throw error;
     }
+    function currentFullTextDescriptor() {
+        const namedKey = `${tableName}/${primaryKey}`;
+        const namedDescriptor = dbisDb.getSync(namedKey);
+        const key = namedDescriptor?.isPrimaryKey ? namedKey : `${tableName}/`;
+        const descriptor = dbisDb.getSync(key);
+        if (!descriptor ||
+            (descriptor.tableId != null && descriptor.tableId !== tableId) ||
+            descriptor.generation !== tableGeneration)
+            return;
+        return { key, descriptor };
+    }
     class TableResource extends Resource_ts_1.Resource {
         #record; // the stored/frozen record from the database and stored in the cache (should not be modified directly)
         #changes; // the changes to the record that have been made (should not be modified directly)
@@ -840,6 +890,38 @@ function makeTable(options) {
         static derivedIndexRuntime;
         static audit = audit;
         static fullTextIndexes = fullTextIndexes;
+        static fullTextIndexGenerations = fullTextIndexGenerations;
+        static fullTextIndexRetirements = fullTextIndexRetirements;
+        static hasCurrentFullTextIndexRetirements(names) {
+            if (names.length === 0 || !(primaryStore.rootStore instanceof rocksdb_js_1.RocksDatabase))
+                return false;
+            return withUpdateAttributesLockNonBlocking(primaryStore.rootStore, `verify full-text retirement for '${databaseName}.${tableName}'`, () => {
+                const current = currentFullTextDescriptor();
+                if (!current)
+                    return false;
+                const pending = new Set((0, fullTextSchema_ts_1.persistedFullTextIndexNames)(current.descriptor.fullTextIndexRetirements));
+                return names.every((name) => pending.has(name));
+            });
+        }
+        static completeFullTextIndexRetirements(names) {
+            if (names.length === 0 || !(primaryStore.rootStore instanceof rocksdb_js_1.RocksDatabase))
+                return;
+            const completed = new Set(names);
+            return withUpdateAttributesLockNonBlocking(primaryStore.rootStore, `complete full-text retirement for '${databaseName}.${tableName}'`, () => {
+                const current = currentFullTextDescriptor();
+                if (!current)
+                    return;
+                const { key, descriptor } = current;
+                const remaining = (0, fullTextSchema_ts_1.persistedFullTextIndexNames)(descriptor.fullTextIndexRetirements).filter((name) => !completed.has(name));
+                const updated = { ...descriptor };
+                if (remaining.length > 0)
+                    updated.fullTextIndexRetirements = remaining.map((name) => ({ name }));
+                else
+                    delete updated.fullTextIndexRetirements;
+                dbisDb.putSync(key, updated);
+                this.fullTextIndexRetirements = remaining;
+            });
+        }
         static databasePath = databasePath;
         static databaseName = databaseName;
         static attributes = attributes;
@@ -863,6 +945,9 @@ function makeTable(options) {
         static userEmbedders = {};
         static userSetEmbedders = new Set();
         static embedAttributes = attributes.filter((a) => a?.embed);
+        static userDeciders = {};
+        static userSetDeciders = new Set();
+        static decideAttributes = attributes.filter((a) => a?.decide);
         static source;
         static getResidencyById;
         static get expirationMS() {
@@ -1224,7 +1309,7 @@ function makeTable(options) {
                                                         catch { }
                                                         throw error;
                                                     }
-                                                    return seqTransaction.commit().catch((error) => {
+                                                    return (0, DatabaseTransaction_ts_1.commitTrackedRocksTransaction)(seqTransaction, dbisDb, primaryStore.rootStore).catch((error) => {
                                                         // A rejected commit leaves the handle open too, so release it here as well —
                                                         // same reason as the staging failure above, and the same shape as the
                                                         // eviction paths' commit failures (see evict/commitItems below).
@@ -1453,6 +1538,8 @@ function makeTable(options) {
          * @returns
          */
         static getResource(target, request, resourceOptions) {
+            if ((0, databaseDropPreparation_ts_1.databaseDropPrepared)(tableRootPath) || (0, DatabaseTransaction_ts_1.databaseCommitsSuspended)(tableRootStore))
+                throw new hdbError_ts_1.DatabaseClosingError(databaseName, !request?.transaction?.root && !request?.transaction?.snapshotFree);
             const resource = super.getResource(target, request, resourceOptions);
             if (this.loadAsInstance !== false) {
                 return resource._loadRecord(target, request, resourceOptions);
@@ -1842,14 +1929,25 @@ function makeTable(options) {
             // same-name recreate must not race an owner still applying to the old generation.
             const derivedIndexRuntime = _a.derivedIndexRuntime;
             const restoreDerivedIndexesAfterFailedDrop = () => {
+                _a.resumeMaintenance();
                 try {
-                    _a.derivedIndexRuntime = derivedIndexRuntime?.restoreAfterFailedDrop?.();
+                    const restored = derivedIndexRuntime?.restoreAfterFailedDrop?.();
+                    if (_a.derivedIndexRuntime === derivedIndexRuntime)
+                        _a.derivedIndexRuntime = restored;
                 }
                 catch (restoreError) {
-                    _a.derivedIndexRuntime = undefined;
+                    if (_a.derivedIndexRuntime === derivedIndexRuntime)
+                        _a.derivedIndexRuntime = undefined;
                     logger_ts_1.logger.error?.(`Could not restore derived indexes after failed drop of ${databaseName}.${_a.tableName}`, restoreError);
                 }
             };
+            try {
+                await _a.closeMaintenance();
+            }
+            catch (error) {
+                _a.resumeMaintenance();
+                throw error;
+            }
             try {
                 await derivedIndexRuntime?.close(true);
             }
@@ -1857,13 +1955,23 @@ function makeTable(options) {
                 restoreDerivedIndexesAfterFailedDrop();
                 throw error;
             }
+            let releaseFullTextRetirementFence;
+            const releaseFullTextRetirement = () => {
+                const release = releaseFullTextRetirementFence;
+                releaseFullTextRetirementFence = undefined;
+                release?.();
+            };
             const abortStaleDrop = () => {
+                releaseFullTextRetirement();
                 derivedIndexRuntime?.completeDrop?.(false);
                 _a.derivedIndexRuntime = undefined;
                 _a.cleanup();
                 if (databases_ts_1.databases[databaseName]?.[tableName] === _a)
                     delete databases_ts_1.databases[databaseName][tableName];
             };
+            let fullTextDefinitionsForRetirement = [
+                ..._a.fullTextIndexes,
+            ];
             let dropIdentityConfirmed = databaseName !== databasePath;
             let primaryCatalogKey = _a.tableName + '/';
             let storeGeneration;
@@ -1892,6 +2000,23 @@ function makeTable(options) {
                         return false;
                     dropGeneration = primaryMeta.dropGeneration;
                     storeGeneration = primaryMeta.generation;
+                    const durableFullTextDefinitions = rootStore instanceof rocksdb_js_1.RocksDatabase
+                        ? [
+                            ...(0, fullTextSchema_ts_1.persistedFullTextIndexNames)(primaryMeta.fullTextIndexes),
+                            ...(0, fullTextSchema_ts_1.persistedFullTextIndexNames)(primaryMeta.fullTextIndexRetirements),
+                        ].map((name) => ({ name }))
+                        : [];
+                    const attachedFullTextDefinitions = derivedIndexRuntime?.fullTextDefinitions?.() ?? [];
+                    const definitionsByName = new Map([...durableFullTextDefinitions, ...attachedFullTextDefinitions].map((definition) => [
+                        definition.name,
+                        definition,
+                    ]));
+                    fullTextDefinitionsForRetirement = [...definitionsByName.values()];
+                    if (fullTextDefinitionsForRetirement.length > 0 && !releaseFullTextRetirementFence) {
+                        releaseFullTextRetirementFence = (0, derivedIndexRegistry_ts_1.acquireFullTextRetirementFence)(rootStore, _a.tableName);
+                        if (!releaseFullTextRetirementFence)
+                            throw new hdbError_ts_1.ClientError(`Cannot drop '${databaseName}.${_a.tableName}' while its full-text storage is being retired`, 409);
+                    }
                     if (primaryMeta.dropping)
                         return true;
                     primaryMeta.dropping = true;
@@ -1923,11 +2048,13 @@ function makeTable(options) {
                     }
                 }
                 catch (error) {
+                    releaseFullTextRetirement();
                     restoreDerivedIndexesAfterFailedDrop();
                     throw error;
                 }
             }
             if (!dropIdentityConfirmed) {
+                releaseFullTextRetirement();
                 abortStaleDrop();
                 return;
             }
@@ -1951,13 +2078,20 @@ function makeTable(options) {
                 try {
                     if (!dropGeneration)
                         throw new Error(`Cannot drop ${databaseName}.${tableName}: its catalog tombstone has no drop generation`);
-                    await retireRocksStores(storeGeneration, dropGeneration);
+                    const retired = await retireRocksStores(storeGeneration, dropGeneration);
+                    if (!retired) {
+                        derivedIndexRuntime?.completeDrop?.();
+                        releaseFullTextRetirement();
+                        return;
+                    }
                 }
                 catch (error) {
+                    releaseFullTextRetirement();
                     derivedIndexRuntime?.completeDrop?.();
                     throw error;
                 }
                 derivedIndexRuntime?.completeDrop?.();
+                releaseFullTextRetirement();
                 return;
             }
             try {
@@ -1968,6 +2102,7 @@ function makeTable(options) {
                 }
             }
             catch (error) {
+                releaseFullTextRetirement();
                 derivedIndexRuntime?.completeDrop?.();
                 throw error;
             }
@@ -2009,6 +2144,7 @@ function makeTable(options) {
                         await dbisDb.committed;
                 }
                 catch (error) {
+                    releaseFullTextRetirement();
                     derivedIndexRuntime?.completeDrop?.();
                     throw error;
                 }
@@ -2029,14 +2165,31 @@ function makeTable(options) {
                     node_fs_1.default.unlinkSync(primaryStore.path);
                 }
                 catch (error) {
+                    releaseFullTextRetirement();
                     derivedIndexRuntime?.completeDrop?.();
                     throw error;
                 }
             }
-            derivedIndexRuntime?.completeDrop?.();
-            const message = new itc_js_1.SchemaEventMsg(process.pid, hdbTerms_ts_1.OPERATIONS_ENUM.DROP_TABLE, databaseName, tableName);
-            message.dropTableId = tableId;
-            signalling.signalSchemaChange(message);
+            try {
+                const message = new itc_js_1.SchemaEventMsg(process.pid, hdbTerms_ts_1.OPERATIONS_ENUM.DROP_TABLE, databaseName, tableName);
+                message.dropTableId = tableId;
+                await signalling.signalSchemaChange(message);
+                await retireFullTextStorage();
+                derivedIndexRuntime?.completeDrop?.();
+            }
+            finally {
+                releaseFullTextRetirement();
+            }
+            async function retireFullTextStorage() {
+                if (fullTextDefinitionsForRetirement.length === 0)
+                    return true;
+                if (derivedIndexRuntime?.retireAfterConfirmedDrop)
+                    return derivedIndexRuntime.retireAfterConfirmedDrop(fullTextDefinitionsForRetirement);
+                else {
+                    const { retireFullTextIndexes } = await import("./derivedIndexes.js");
+                    return retireFullTextIndexes(_a, fullTextDefinitionsForRetirement);
+                }
+            }
             async function retireRocksStores(generation, dropGeneration) {
                 const releaseDropMark = (0, databases_ts_1.markDropInProgress)(dropGeneration);
                 try {
@@ -2044,6 +2197,10 @@ function makeTable(options) {
                     message.dropGeneration = dropGeneration;
                     message.dropTableId = tableId;
                     await signalling.signalSchemaChange(message);
+                    // Keep the tombstone's native index names durable until every peer writer
+                    // has stopped and the wrapper has retired their storage.
+                    if (!(await retireFullTextStorage()))
+                        return false;
                     const removed = withUpdateAttributesLock(rootStore, `table '${databaseName}.${tableName}'`, () => {
                         const stores = (0, databases_ts_1.storeNamesFor)(dbisDb, tableName, generation);
                         const retiredStores = (0, databases_ts_1.recordRetiredGeneration)(dbisDb, tableName, dropGeneration, stores, primaryStore.name ?? (0, databases_ts_1.storeNameFor)(primaryCatalogKey, generation));
@@ -2092,6 +2249,7 @@ function makeTable(options) {
                     await (0, databases_ts_1.sweepDroppedTableBlobs)(primaryStore, label);
                     if (!settled)
                         finishDroppedTableBlobSweep(rootStore, primaryStore, label);
+                    return true;
                 }
                 finally {
                     releaseDropMark();
@@ -2724,6 +2882,8 @@ function makeTable(options) {
          * Evicting a record will remove it from a caching table. This is not considered a canonical data change, and it is assumed that retrieving this record from the source will still yield the same record, this is only removing the local copy of the record.
          */
         static evict(id, existingRecord, existingVersion) {
+            if (maintenanceClosed)
+                return Promise.resolve();
             let entry;
             const lmdbTransaction = txnForContext({ transaction: new DatabaseTransaction_ts_1.DatabaseTransaction() });
             let transaction = lmdbTransaction.getReadTxn();
@@ -2775,15 +2935,15 @@ function makeTable(options) {
                     // a plain resolution object rather than a promise — return the store's write promises instead, so
                     // the caller gets a real thenable that resolves once the removal is durable.
                     lmdbTransaction.commit();
-                    return Promise.resolve(lmdbCompletion).catch((error) => {
+                    return trackMaintenanceCommit(Promise.resolve(lmdbCompletion).catch((error) => {
                         logger_ts_1.logger.warn?.('Error evicting record', id, error);
-                    });
+                    }));
                 }
                 // RocksDB: eviction writes went directly into the raw transaction via options; commit it directly,
                 // as DatabaseTransaction.commit() would abort it (no tracked writes). The raw commit bypasses
                 // DatabaseTransaction's ERR_BUSY retry, so a concurrent-write conflict rejects here — swallow it
                 // (abandon the eviction) and log anything unexpected, rather than letting it crash the process.
-                return transaction.commit().catch((error) => {
+                return trackMaintenanceCommit((0, DatabaseTransaction_ts_1.commitTrackedRocksTransaction)(transaction, primaryStore).catch((error) => {
                     // The commit failed, so the read-snapshot/transaction handle is still open — release it, as the
                     // batched-eviction path does on its own commit failures. committed===true skips the finally abort.
                     try {
@@ -2794,7 +2954,7 @@ function makeTable(options) {
                         logger_ts_1.logger.trace?.('Abandoned eviction of busy record', id);
                     else
                         logger_ts_1.logger.warn?.('Error evicting record', id, error);
-                });
+                }));
             }
             finally {
                 if (!committed) {
@@ -4076,23 +4236,26 @@ function makeTable(options) {
                 },
             };
             this.#savingOperation = write;
-            // `@embed` hook must run before `addWrite` so the embedder's vector is on the
-            // record when `commit` runs. (The txn `before` slot runs after commit, which
-            // suits blob writes but not embedding, where the vector must be present at commit.)
-            // Known limitation of this write-time placement (a validate-time alternative was
-            // tried and reverted as a Harper-foreign pattern): the embedder sees this write's
-            // payload, before table validation — so a write that later fails validation still
-            // calls the backend, and a tracked-instance mutation (update(id,{}); row.source=…;
-            // save()) that sets the source via accessors after update() won't re-embed. A
-            // resource-layer re-embed is the proper fix; tracked as a follow-up.
-            const embedBefore = (0, embedHook_ts_1.buildEmbedBefore)(recordUpdate, context, options, _a.embedAttributes, _a.userEmbedders);
+            // The hooks run before `addWrite` so the derived values are on the record at commit (the
+            // txn `before` slot runs after commit). They see the payload before table validation, and a
+            // tracked-instance mutation that sets the source via accessors after update() is not seen.
+            let modelHooksBefore;
+            try {
+                modelHooksBefore =
+                    (_a.embedAttributes.length || _a.decideAttributes.length) && !(0, databases_ts_1.isReadOnlyMode)()
+                        ? (0, embedHook_ts_1.combineWriteHooks)((0, embedHook_ts_1.buildEmbedBefore)(recordUpdate, context, options, _a.embedAttributes, _a.userEmbedders), (0, decideHook_ts_1.buildDecideBefore)(recordUpdate, context, options, _a.decideAttributes, _a.userDeciders))
+                        : undefined;
+            }
+            catch (err) {
+                return Promise.reject(err);
+            }
             const proceed = () => {
                 // On a source/replication apply (`isNotification`), the record's already-saved blobs were
                 // received out-of-band for THIS write, so track them for skip/abort cleanup (harper-pro#406).
                 write.beforeIntermediate = preCommitBlobsForRecordBefore(write, recordUpdate, undefined, undefined, options?.isNotification);
                 return transaction.addWrite(write);
             };
-            return embedBefore ? embedBefore().then(proceed) : proceed();
+            return modelHooksBefore ? modelHooksBefore(context?.signal).then(proceed) : proceed();
         }
         async delete(target) {
             if (isSearchTarget(target)) {
@@ -5326,6 +5489,10 @@ function makeTable(options) {
                         // messages during this loop because the snapshot:false cursor will pick them up itself.
                         pendingRealTimeQueue = null;
                         dropDuringReplay = true;
+                        // subscription.startTime is the resume cursor (exclusive) and RocksDB gives every record of a
+                        // transaction the same txnLogKey, so it only moves to a key once all of that key's records are
+                        // handled; an early return leaves it before a partly delivered transaction.
+                        let handledTxnLogKey;
                         try {
                             for (const auditRecord of auditStore.getRange({
                                 start: startTime,
@@ -5342,20 +5509,25 @@ function makeTable(options) {
                                     continue;
                                 if ((0, auditStore_ts_1.isLockControlType)(auditRecord.type))
                                     continue;
+                                if (handledTxnLogKey !== undefined && auditRecord.txnLogKey !== handledTxnLogKey) {
+                                    subscription.startTime = handledTxnLogKey;
+                                }
                                 const id = auditRecord.recordId;
-                                subscription.startTime = auditRecord.txnLogKey;
                                 if (thisId == null || isDescendantId(thisId, id)) {
                                     const event = eventFromAudit(id, auditRecord, auditRecord.txnLogKey);
-                                    if (!event)
-                                        continue;
-                                    if (!send(event))
-                                        return;
-                                    if (subscription.queue?.length > EVENT_HIGH_WATER_MARK) {
-                                        if ((await subscription.waitForDrain()) === false)
+                                    if (event) {
+                                        if (!send(event))
                                             return;
+                                        if (subscription.queue?.length > EVENT_HIGH_WATER_MARK) {
+                                            if ((await subscription.waitForDrain()) === false)
+                                                return;
+                                        }
                                     }
                                 }
+                                handledTxnLogKey = auditRecord.txnLogKey;
                             }
+                            if (handledTxnLogKey !== undefined)
+                                subscription.startTime = handledTxnLogKey;
                         }
                         finally {
                             // replay is done, we can start sending real-time messages again
@@ -6318,10 +6490,19 @@ function makeTable(options) {
         static updatedAttributes() {
             // Refresh on every call: schema reload mutates `attributes` in place, so the
             // class-construction snapshot would otherwise go stale.
+            // Declarations are refused before they are saved; here a descriptor an earlier build
+            // accepted must still load, so a violation is logged and the table keeps working.
+            try {
+                (0, embedHook_ts_1.assertDerivedFieldOwnership)(this.attributes);
+            }
+            catch (error) {
+                console.error(`Derived attributes of table "${tableName}" conflict: ${error.message}`);
+            }
             this.embedAttributes = this.attributes.filter((a) => a?.embed);
+            this.decideAttributes = this.attributes.filter((a) => a?.decide);
             expiresAtProperty = this.attributes.find((attribute) => attribute.expiresAt);
-            // Drop registry entries for attributes that are no longer `@embed`, so a dropped
-            // directive doesn't leave a stale embedder or block a default refresh on re-add.
+            // Drop registry entries for attributes that are no longer `@embed` / `@decide`, so a dropped
+            // directive doesn't leave a stale hook or block a default refresh on re-add.
             const embedNames = new Set(this.embedAttributes.map((a) => a.name));
             for (const name of Object.keys(this.userEmbedders))
                 if (!embedNames.has(name))
@@ -6329,6 +6510,13 @@ function makeTable(options) {
             for (const name of this.userSetEmbedders)
                 if (!embedNames.has(name))
                     this.userSetEmbedders.delete(name);
+            const decideNames = new Set(this.decideAttributes.map((a) => a.name));
+            for (const name of Object.keys(this.userDeciders))
+                if (!decideNames.has(name))
+                    delete this.userDeciders[name];
+            for (const name of this.userSetDeciders)
+                if (!decideNames.has(name))
+                    this.userSetDeciders.delete(name);
             propertyResolvers = this.propertyResolvers = {
                 $id: (object, context, entry) => ({ value: entry.key }),
                 $updatedtime: (object, context, entry) => entry.version,
@@ -6368,9 +6556,12 @@ function makeTable(options) {
                 const computed = attribute.computed;
                 // Register the default embedder unless an author override is set. Sits outside
                 // the resolver chain below so `@embed` fields still flow through auto-HNSW indexing.
-                if (attribute.embed && !_a.userSetEmbedders.has(attribute.name)) {
+                if (attribute.embed && !this.userSetEmbedders.has(attribute.name)) {
                     this.userEmbedders[attribute.name] = (0, embedHook_ts_1.createDefaultEmbedder)(attribute.embed);
                 }
+                if (attribute.decide && !this.userSetDeciders.has(attribute.name)) {
+                    this.userDeciders[attribute.name] = (0, decideHook_ts_1.createDefaultDecider)(attribute.decide);
+                }
                 if (relationship) {
                     if (attribute.indexed) {
                         console.error(`A relationship property can not be directly indexed, (but you may want to index the foreign key attribute)`);
@@ -6597,6 +6788,26 @@ function makeTable(options) {
             this.userEmbedders[attribute_name] = embedder;
             this.userSetEmbedders.add(attribute_name);
         }
+        /**
+         * Override the default decider for a `@decide` attribute. Return `{ value, probability }`
+         * to store at the attribute and its confidence attribute, or `null` to clear both. The
+         * value must be one the directive allows, and the probability is required when the
+         * directive names a confidence attribute. Like an embedder, the decider receives the write
+         * payload, not the post-merge record, and a `signal` that aborts when a sibling hook fails.
+         */
+        static setDecideAttribute(attribute_name, decider) {
+            const attribute = (0, search_ts_1.findAttribute)(attributes, attribute_name);
+            if (!attribute) {
+                console.error(`The attribute "${attribute_name}" does not exist in the table "${tableName}"`);
+                return;
+            }
+            if (!attribute.decide) {
+                console.error(`The attribute "${attribute_name}" is not declared with @decide in the table "${tableName}"`);
+                return;
+            }
+            this.userDeciders[attribute_name] = decider;
+            this.userSetDeciders.add(attribute_name);
+        }
         static async deleteHistory(endTime = 0, cleanupDeletedRecords = false) {
             const maxConcurrentRemovals = isRocksDB ? MAX_CONCURRENT_HISTORY_REMOVALS : MAX_CONCURRENT_LMDB_HISTORY_REMOVALS;
             const inFlightRemovals = new Set();
@@ -6760,31 +6971,105 @@ function makeTable(options) {
             } while (count < 1000 && nextVersion);
             return history.reverse();
         }
-        static clear() {
-            // clear the primary store and every secondary index dbi (same pattern used by
-            // runIndexing when rebuilding from scratch), so clear() doesn't leave stale
-            // index entries pointing at records that no longer exist.
-            const promises = [primaryStore.clear()];
-            for (const key in indices) {
-                const index = indices[key];
-                index.customIndex?.resetDerivedStorage?.();
-                promises.push(index.clearAsync ? index.clearAsync() : index.clear());
+        static async clear() {
+            const rootStore = primaryStore.rootStore;
+            const assertNoFullTextDeclaration = () => {
+                const namedPrimaryDescriptor = dbisDb.getSync(`${tableName}/${primaryKey}`);
+                const primaryDescriptor = namedPrimaryDescriptor?.isPrimaryKey
+                    ? namedPrimaryDescriptor
+                    : dbisDb.getSync(`${tableName}/`);
+                if (_a.fullTextIndexes.length > 0 ||
+                    (rootStore instanceof rocksdb_js_1.RocksDatabase &&
+                        (0, fullTextSchema_ts_1.persistedFullTextIndexNames)(primaryDescriptor?.fullTextIndexes).length > 0))
+                    throw new hdbError_ts_1.ClientError(`Table.clear() is not supported on full-text table '${databaseName}.${tableName}' until whole-table invalidation is crash-safe`, 501);
+            };
+            let releaseFullTextClearFence;
+            let waitForExistingClear;
+            let admission;
+            if (rootStore instanceof rocksdb_js_1.RocksDatabase) {
+                admission = withUpdateAttributesLockNonBlocking(rootStore, `clear table '${databaseName}.${tableName}'`, () => {
+                    assertNoFullTextDeclaration();
+                    releaseFullTextClearFence = (0, derivedIndexRegistry_ts_1.acquireFullTextClearFence)(rootStore, tableId);
+                    if (!releaseFullTextClearFence)
+                        waitForExistingClear = (0, derivedIndexRegistry_ts_1.waitForFullTextClear)(rootStore, tableId);
+                });
             }
-            return Promise.all(promises);
+            else
+                assertNoFullTextDeclaration();
+            const startClear = () => {
+                if (waitForExistingClear)
+                    return waitForExistingClear.then(() => _a.clear());
+                // clear the primary store and every secondary index dbi (same pattern used by
+                // runIndexing when rebuilding from scratch), so clear() doesn't leave stale
+                // index entries pointing at records that no longer exist.
+                const promises = [];
+                const settleStartedClears = (synchronousFailure) => Promise.allSettled(promises)
+                    .then((results) => {
+                    if (synchronousFailure)
+                        throw synchronousFailure.error;
+                    const values = [];
+                    for (const result of results) {
+                        if (result.status === 'rejected')
+                            throw result.reason;
+                        values.push(result.value);
+                    }
+                    return values;
+                })
+                    .finally(releaseFullTextClearFence);
+                try {
+                    promises.push(primaryStore.clear());
+                    for (const key in indices) {
+                        const index = indices[key];
+                        index.customIndex?.resetDerivedStorage?.();
+                        promises.push(index.clearAsync ? index.clearAsync() : index.clear());
+                    }
+                    return settleStartedClears();
+                }
+                catch (error) {
+                    if (promises.length > 0)
+                        return settleStartedClears({ error });
+                    releaseFullTextClearFence?.();
+                    throw error;
+                }
+            };
+            return admission instanceof Promise ? admission.then(startClear) : startClear();
         }
         /** Release everything makeTable() registered process-wide; the class must not be used afterwards. */
         static cleanup() {
             disposed = true;
             _a.getResource = unavailableResource;
-            _a.derivedIndexRuntime
+            void _a.derivedIndexRuntime
                 ?.close()
                 .catch((error) => logger_ts_1.logger.warn?.(`Derived index shutdown failed for ${databaseName}.${tableName}`, error));
-            clearTimeout(cleanupTimer);
-            settlePendingCleanup();
-            clearInterval(recordExpirationInterval);
+            stopMaintenance();
             deleteCallbackHandle?.remove();
             (0, storageReclamation_ts_1.removeStorageReclamationHandler)(primaryStore.path, reclamationHandler);
         }
+        static async closeMaintenance(deadline) {
+            const startedAt = Date.now();
+            const defaultTimeout = (0, DatabaseTransaction_ts_1.getDatabaseCommitDrainTimeoutMilliseconds)();
+            const timeoutMilliseconds = deadline === undefined ? defaultTimeout : Math.max(0, deadline - startedAt);
+            const resolvedDeadline = deadline ?? startedAt + timeoutMilliseconds;
+            stopMaintenance();
+            const timeoutError = () => new hdbError_ts_1.DatabaseDrainTimeoutError(databaseName, timeoutMilliseconds);
+            await (0, when_ts_1.settleBeforeDeadline)([lastEvictionCompletion, recordExpirationCompletion], resolvedDeadline, timeoutError);
+            for (;;) {
+                const pending = [...maintenanceCommits];
+                if (pending.length === 0)
+                    return;
+                await (0, when_ts_1.settleBeforeDeadline)(pending, resolvedDeadline, timeoutError);
+            }
+        }
+        static resumeMaintenance() {
+            if (disposed || !maintenanceClosed)
+                return;
+            maintenanceClosed = false;
+            lastCleanupInterval = undefined;
+            if (expirationScanScheduled || evictionMs)
+                scheduleCleanup();
+            if (expiresAtProperty && !recordExpirationInterval)
+                runRecordExpirationEviction();
+        }
         static _readTxnForContext(context) {
             return txnForContext(context).getReadTxn();
         }
@@ -6802,7 +7087,7 @@ function makeTable(options) {
         }
     }, () => {
         throw new hdbError_ts_1.ServerError('Service unavailable, exceeded request queue limit for resolving cache record', 503);
-    });
+    }, undefined, `cache resolution for ${tableName}`);
     try {
         TableResource.updatedAttributes(); // on creation, update accessors as well
         if (expirationMs) {
@@ -7445,7 +7730,8 @@ function makeTable(options) {
             replacingVersion: existingVersion,
             // Once dropTable() has started, no new source-fill write may begin; still resolve the
             // caller's read with fresh source data, just don't cache it into a table that's going away.
-            noCacheStore: droppingTable,
+            // No write can commit on a read-only node, so a fill there is served but never staged.
+            noCacheStore: droppingTable || (0, databases_ts_1.isReadOnlyMode)(),
             source: null,
             transaction: undefined,
             expiresAt: undefined,
@@ -7769,14 +8055,11 @@ function makeTable(options) {
                         }
                     },
                 };
-                // The cache-from-source write bypasses `_writeUpdate`, so wire the embed hook here
-                // too (always the originating node). It runs after the client GET has resolved with
-                // fresh source data, so it's a background commit: an embedder failure aborts the cache
-                // write via the outer error handler (row re-embeds next read) and never reaches the
-                // caller. Source-resolution errors are handled earlier, with the stale-data fallback.
-                const embedBefore = (0, embedHook_ts_1.buildEmbedBefore)(updatedRecord, sourceContext, undefined, TableResource.embedAttributes, TableResource.userEmbedders);
-                if (embedBefore)
-                    await embedBefore();
+                // The fill is shared by every reader, so it takes no request signal; a hook failure aborts
+                // only the cache write.
+                const modelHooksBefore = (0, embedHook_ts_1.combineWriteHooks)((0, embedHook_ts_1.buildEmbedBefore)(updatedRecord, sourceContext, undefined, TableResource.embedAttributes, TableResource.userEmbedders), (0, decideHook_ts_1.buildDecideBefore)(updatedRecord, sourceContext, undefined, TableResource.decideAttributes, TableResource.userDeciders));
+                if (modelHooksBefore)
+                    await modelHooksBefore();
                 if (droppingTable) {
                     // Re-check right before staging the write: dropTable() may have started
                     // while we were awaiting the embed step above (harper#1381).
@@ -7811,6 +8094,19 @@ function makeTable(options) {
             throw new hdbError_ts_1.ClientError('Can not specify replication confirmation without super user permissions', 403);
         return true;
     }
+    function trackMaintenanceCommit(commit) {
+        const tracked = commit.finally(() => maintenanceCommits.delete(tracked));
+        maintenanceCommits.add(tracked);
+        return tracked;
+    }
+    function stopMaintenance() {
+        maintenanceClosed = true;
+        clearTimeout(cleanupTimer);
+        cleanupTimer = undefined;
+        settlePendingCleanup();
+        clearInterval(recordExpirationInterval);
+        recordExpirationInterval = undefined;
+    }
     // RocksDB-only: coalesces eviction/tombstone removals into shared transactions so the cleanup
     // scan pays one commit per batch instead of one per record. Descriptors hold only the decoded
     // primary key and the version seen during the scan (both stable primitives — the scanned record
@@ -7875,7 +8171,7 @@ function makeTable(options) {
                     return;
                 }
                 try {
-                    await transaction.commit();
+                    await (0, DatabaseTransaction_ts_1.commitTrackedRocksTransaction)(transaction, primaryStore);
                     return;
                 }
                 catch (error) {
@@ -7926,7 +8222,7 @@ function makeTable(options) {
     }
     function scheduleCleanup(priority) {
         // a reclamation run may still hold this class's handler after cleanup(); a promise here would never settle
-        if (disposed)
+        if (disposed || maintenanceClosed)
             return;
         let runImmediately = false;
         if (priority) {
@@ -7938,7 +8234,10 @@ function makeTable(options) {
         // Periodically evict expired records and deleted records searching for records who expiresAt timestamp is before now
         if (cleanupInterval === lastCleanupInterval && !runImmediately)
             return;
-        lastCleanupInterval = cleanupInterval;
+        // Left unrecorded by a thread with no worker index yet: under threads: 0 the main thread loads tables
+        // before it becomes worker 0, and its next call for this interval has to arm the scan then.
+        if ((0, manageThreads_js_1.getWorkerIndex)() !== undefined)
+            lastCleanupInterval = cleanupInterval;
         if ((0, manageThreads_js_1.ownsStoreMaintenance)(primaryStore.path) || (ttlConfiguredByApplication && (0, manageThreads_js_1.isDedicatedWorker)())) {
             // run on the last thread so we aren't overloading lower-numbered threads
             if (cleanupTimer)
@@ -7968,11 +8267,13 @@ function makeTable(options) {
                     ? Date.now()
                     : Math.ceil((Date.now() - startOfYear.getTime()) / nextInterval) * nextInterval + startOfYear.getTime();
                 const startNextTimer = (nextScheduled) => {
-                    if (disposed)
+                    if (disposed || maintenanceClosed)
                         return;
                     logger_ts_1.logger.trace?.(`Scheduled next cleanup scan at ${new Date(nextScheduled)}`);
                     // noinspection JSVoidFunctionReturnValueUsed
                     cleanupTimer = setTimeout(() => (lastEvictionCompletion = lastEvictionCompletion.then(async () => {
+                        if (disposed || maintenanceClosed)
+                            return;
                         // schedule the next run for when the next cleanup interval should occur (or now if it is in the past)
                         startNextTimer(Math.max(nextScheduled + cleanupInterval, Date.now()));
                         const rootStore = primaryStore.rootStore;
@@ -8022,6 +8323,8 @@ function makeTable(options) {
                                 versions: true,
                                 lazy: true, // only want to access metadata most of the time
                             })) {
+                                if (maintenanceClosed)
+                                    break;
                                 const { key, value: record, version, expiresAt, metadataFlags } = entry;
                                 // if there is no auditing cleanup and we are tracking deletion, need to do cleanup of
                                 // these deletion entries (LMDB audit cleanup has its own scheduled job for this)
@@ -8055,6 +8358,7 @@ function makeTable(options) {
                                 }
                                 await rest();
                             }
+                            await Promise.all(outstandingCleanupOperations.filter(Boolean));
                             if (batcher)
                                 await batcher.drain();
                             logger_ts_1.logger.debug?.(`Finished cleanup scan for ${tableName}, evicted ${count} entries`);
@@ -8083,45 +8387,59 @@ function makeTable(options) {
         // Periodically evict expired records, searching for records who expiresAt timestamp is before now
         if ((0, manageThreads_js_1.ownsStoreExpiration)(primaryStore.path) || (ttlConfiguredByApplication && (0, manageThreads_js_1.isDedicatedWorker)())) {
             // we want to run the pruning of expired records on only one thread so we don't have conflicts in evicting
-            recordExpirationInterval = setInterval(async () => {
+            recordExpirationInterval = setInterval(() => {
                 // go through each database and table and then search for expired entries
                 // find any entries that are set to expire before now
                 // updatedAttributes() clears expiresAtProperty when a live redeclaration drops the directive,
                 // and there is nothing left for this interval to scan by
-                if (disposed || runningRecordExpiration || !expiresAtProperty)
+                if (disposed || maintenanceClosed || runningRecordExpiration || !expiresAtProperty)
                     return;
                 runningRecordExpiration = true;
-                try {
+                recordExpirationCompletion = (async () => {
                     const expiresAtName = expiresAtProperty.name;
                     const index = indices[expiresAtName];
                     if (!index)
                         throw new Error(`expiresAt attribute ${expiresAtProperty} must be indexed`);
-                    for (const key of index.getRange({
+                    const inFlight = new Set();
+                    const trackInFlight = (operation) => {
+                        const tracked = operation.finally(() => inFlight.delete(tracked));
+                        inFlight.add(tracked);
+                        if (inFlight.size >= 50)
+                            return Promise.race(inFlight);
+                    };
+                    expirationScan: for (const key of index.getRange({
                         start: true,
                         values: false,
                         end: Date.now(),
                         snapshot: false,
                     })) {
                         for (const id of index.getValues(key)) {
+                            if (maintenanceClosed)
+                                break expirationScan;
                             const recordEntry = primaryStore.getEntry(id);
                             if (!recordEntry?.value) {
                                 // cleanup the index if the record is gone
-                                primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id));
+                                const repair = trackMaintenanceCommit(Promise.resolve(primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id))).catch((error) => logger_ts_1.logger.warn?.('Error removing stale expiration index entry', id, error)));
+                                const backpressure = trackInFlight(repair);
+                                if (backpressure)
+                                    await backpressure;
                             }
                             else if (recordEntry.value[expiresAtName] < Date.now()) {
                                 // make sure the record hasn't changed and won't change while removing
-                                TableResource.evict(id, recordEntry.value, recordEntry.version);
+                                const eviction = TableResource.evict(id, recordEntry.value, recordEntry.version);
+                                if (eviction) {
+                                    const backpressure = trackInFlight(eviction);
+                                    if (backpressure)
+                                        await backpressure;
+                                }
                             }
                         }
                         await rest();
                     }
-                }
-                catch (error) {
-                    logger_ts_1.logger.error?.('Error in evicting old records', error);
-                }
-                finally {
-                    runningRecordExpiration = false;
-                }
+                    await Promise.all(inFlight);
+                })()
+                    .catch((error) => logger_ts_1.logger.error?.('Error in evicting old records', error))
+                    .finally(() => (runningRecordExpiration = false));
             }, RECORD_PRUNING_INTERVAL).unref();
         }
     }