@harperfast/harper 5.0.25 → 5.1.0-beta.1

package/dist/resources/Table.js

@@ -45,7 +45,8 @@ exports.EVICTED = exports.INVALIDATED = void 0;
 exports.makeTable = makeTable;
 exports.coerceType = coerceType;
 const hdbTerms_ts_1 = require("../utility/hdbTerms.js");
-const commonUtility_js_1 = require("../utility/lmdb/commonUtility.js");
+const node_vm_1 = require("node:vm");
+const commonUtility_ts_1 = require("../utility/lmdb/commonUtility.js");
 const nodeIdMapping_ts_1 = require("./nodeIdMapping.js");
 const lodash_1 = __importDefault(require("lodash"));
 const extended_iterable_1 = require("@harperfast/extended-iterable");
@@ -55,7 +56,7 @@ const when_ts_1 = require("../utility/when.js");
 const DatabaseTransaction_ts_1 = require("./DatabaseTransaction.js");
 const envMngr = __importStar(require("../utility/environment/environmentManager.js"));
 const transactionBroadcast_ts_1 = require("./transactionBroadcast.js");
-const hdbError_js_1 = require("../utility/errors/hdbError.js");
+const hdbError_ts_1 = require("../utility/errors/hdbError.js");
 const signalling = __importStar(require("../utility/signalling.js"));
 const itc_js_1 = require("../server/threads/itc.js");
 const databases_ts_1 = require("./databases.js");
@@ -66,7 +67,8 @@ const transaction_ts_1 = require("./transaction.js");
 const ordered_binary_1 = require("ordered-binary");
 const manageThreads_js_1 = require("../server/threads/manageThreads.js");
 const auditStore_ts_1 = require("./auditStore.js");
-const common_utils_js_1 = require("../utility/common_utils.js");
+const embedHook_ts_1 = require("./models/embedHook.js");
+const common_utils_ts_1 = require("../utility/common_utils.js");
 const RecordEncoder_ts_1 = require("./RecordEncoder.js");
 const write_ts_1 = require("./analytics/write.js");
 const crdt_ts_1 = require("./crdt.js");
@@ -74,7 +76,7 @@ const Headers_ts_1 = require("../server/serverHelpers/Headers.js");
 const node_fs_1 = __importDefault(require("node:fs"));
 const blob_ts_1 = require("./blob.js");
 const storageReclamation_ts_1 = require("../server/storageReclamation.js");
-const harper_logger_js_1 = __importDefault(require("../utility/logging/harper_logger.js"));
+const harper_logger_ts_1 = __importDefault(require("../utility/logging/harper_logger.js"));
 const throttle_ts_1 = require("../server/throttle.js");
 const rocksdb_js_1 = require("@harperfast/rocksdb-js");
 const LMDBTransaction_1 = require("./LMDBTransaction");
@@ -85,6 +87,7 @@ const NULL_WITH_TIMESTAMP = new Uint8Array(9);
 NULL_WITH_TIMESTAMP[8] = 0xc0; // null
 const UNCACHEABLE_TIMESTAMP = Infinity; // we use this when dynamic content is accessed that we can't safely cache, and this prevents earlier timestamps from change the "last" modification
 const RECORD_PRUNING_INTERVAL = 60000; // one minute
+const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]);
 envMngr.initSync();
 const LMDB_PREFETCH_WRITES = envMngr.get(hdbTerms_ts_1.CONFIG_PARAMS.STORAGE_PREFETCHWRITES);
 const LOCK_TIMEOUT = 10000;
@@ -106,6 +109,7 @@ const FULL_PERMISSIONS = {
  * Instances of the returned class are Resource instances, intended to provide a consistent view or transaction of the table
  * @param options
  */
+// #section: setup-and-factory
 function makeTable(options) {
     const { primaryKey, indices, tableId, tableName, primaryStore, databasePath, databaseName, auditStore, schemaDefined, dbisDB: dbisDb, sealed, splitSegments, replicate, } = options;
     let { expirationMS: expirationMs, evictionMS: evictionMs, audit, trackDeletes } = options;
@@ -116,7 +120,7 @@ function makeTable(options) {
     const updateRecord = (0, RecordEncoder_ts_1.recordUpdater)(primaryStore, tableId, auditStore);
     let sourceLoad; // if a source has a load function (replicator), record it here
     let hasSourceGet;
-    let primaryKeyAttribute = {};
+    let primaryKeyAttribute;
     let lastEvictionCompletion = Promise.resolve();
     let createdTimeProperty, updatedTimeProperty, expiresAtProperty;
     for (const attribute of attributes) {
@@ -186,6 +190,7 @@ function makeTable(options) {
         #version; // version of the record
         #entry; // the entry from the database
         #savingOperation; // operation for the record is currently being saved
+        // #section: static-config
         static name = tableName; // for display/debugging purposes
         static primaryStore = primaryStore;
         static auditStore = auditStore;
@@ -204,6 +209,11 @@ function makeTable(options) {
         static updatedTimeProperty = updatedTimeProperty;
         static propertyResolvers;
         static userResolvers = {};
+        // `@embed` hook registry. `userSetEmbedders` records names set explicitly via
+        // `setEmbedAttribute` so a schema reload refreshes defaults without clobbering them.
+        static userEmbedders = {};
+        static userSetEmbedders = new Set();
+        static embedAttributes = attributes.filter((a) => a?.embed);
         static source;
         static getResidencyById;
         static get expirationMS() {
@@ -219,6 +229,7 @@ function makeTable(options) {
          * @param options
          * @returns
          */
+        // #section: resource-registry
         static sourcedFrom(source, options) {
             // define a source for retrieving invalidated entries for caching purposes
             if (options) {
@@ -463,7 +474,7 @@ function makeTable(options) {
                                 });
                                 if (txnInProgress)
                                     txnInProgress.committed = commitResolution;
-                                if (userRoleUpdate && commitResolution && !commitResolution?.waitingForUserChange) {
+                                if (userRoleUpdate && commitResolution && !commitResolution.waitingForUserChange) {
                                     // if the user role changed, asynchronously signal the user change (but don't block this function)
                                     commitResolution.then(() => signalling.signalUserChange(new itc_js_1.UserEventMsg(process.pid)));
                                     commitResolution.waitingForUserChange = true; // only need to send one signal per transaction
@@ -539,7 +550,9 @@ function makeTable(options) {
                         // return 504 (rather than 404) if there is no content and the cache-control header
                         // dictates not to go to source
                         if (!this.doesExist())
-                            throw new hdbError_js_1.ServerError('Entry is not cached', 504);
+                            throw new hdbError_ts_1.ServerError('Entry is not cached', 504);
+                        if (hasSourceGet && target)
+                            target.loadedFromSource = false; // mark it as cached
                     }
                     else if (resourceOptions?.ensureLoaded) {
                         const loadingFromSource = ensureLoadedFromSource(this.constructor.source, id, entry, request, this, target);
@@ -582,6 +595,7 @@ function makeTable(options) {
                 });
             }
         }
+        // #section: lifecycle-admin
         static getNewId() {
             const type = primaryKeyAttribute?.type;
             // the default Resource behavior is to return a GUID, but for a table we can return incrementing numeric keys if the type is (or can be) numeric
@@ -867,6 +881,11 @@ function makeTable(options) {
             }
             signalling.signalSchemaChange(new itc_js_1.SchemaEventMsg(process.pid, hdbTerms_ts_1.OPERATIONS_ENUM.DROP_TABLE, databaseName, tableName));
         }
+        // #section: read-path
+        /**
+         * This retrieves the data of this resource.
+         * @param target - If included, is an identifier/query that specifies the requested target to retrieve and query
+         */
         get(target) {
             const constructor = this.constructor;
             if (typeof target === 'string' && constructor.loadAsInstance !== false)
@@ -905,13 +924,13 @@ function makeTable(options) {
                 const id = requestTargetToId(target);
                 checkValidId(id);
                 let allowed = true;
-                if (target.checkPermission) {
+                if (target?.checkPermission) {
                     // requesting authorization verification
                     allowed = this.allowRead(context.user, target, context);
                 }
                 return (0, when_ts_1.promiseNormalize)((0, when_ts_1.when)((0, when_ts_1.when)(allowed, (allowed) => {
                     if (!allowed) {
-                        throw new hdbError_js_1.AccessViolation(context.user);
+                        throw new hdbError_ts_1.AccessViolation(context.user);
                     }
                     const ensureLoaded = true;
                     return loadLocalRecord(id, context, { transaction: readTxn, ensureLoaded }, false, (entry) => {
@@ -920,7 +939,7 @@ function makeTable(options) {
                             // return 504 (rather than 404) if there is no content and the cache-control header
                             // dictates not to go to source
                             if (!entry?.value)
-                                throw new hdbError_js_1.ServerError('Entry is not cached', 504);
+                                throw new hdbError_ts_1.ServerError('Entry is not cached', 504);
                         }
                         else if (ensureLoaded) {
                             const loadingFromSource = ensureLoadedFromSource(constructor.source, id, entry, context, this, target);
@@ -960,6 +979,7 @@ function makeTable(options) {
             }
             return undefined;
         }
+        // #section: authz-hooks
         /**
          * Determine if the user is allowed to get/read data from the current resource
          */
@@ -1124,13 +1144,13 @@ function makeTable(options) {
                     let allowed = true;
                     if (target == undefined)
                         throw new TypeError('Can not put a record without a target');
-                    if (target.checkPermission) {
+                    if (target?.checkPermission) {
                         // requesting authorization verification
                         allowed = this.allowUpdate(context.user, updates, context);
                     }
                     return (0, when_ts_1.when)(allowed, (allowed) => {
                         if (!allowed) {
-                            throw new hdbError_js_1.AccessViolation(context.user);
+                            throw new hdbError_ts_1.AccessViolation(context.user);
                         }
                         let loading;
                         if (!this.#entry && this.constructor.loadAsInstance === false) {
@@ -1139,31 +1159,33 @@ function makeTable(options) {
                         }
                         return (0, when_ts_1.when)(loading, () => {
                             this.#changes = updates;
-                            this._writeUpdate(id, this.#changes, false);
-                            return this;
+                            // `when` awaits the embed hook (when `@embed` is active) before resolving,
+                            // so the caller's `save()` doesn't run before the write is staged.
+                            return (0, when_ts_1.when)(this._writeUpdate(id, this.#changes, false), () => this);
                         });
                     });
                 }
             }
-            this._writeUpdate(id, this.#changes, fullUpdate);
-            return this;
+            return (0, when_ts_1.when)(this._writeUpdate(id, this.#changes, fullUpdate), () => this);
         }
         /**
          * Save any changes into this instance to the current transaction
          */
         save() {
             if (this.#savingOperation) {
-                const transaction = txnForContext(this.getContext());
-                if (transaction.save) {
-                    try {
-                        return transaction.save(this.#savingOperation);
-                    }
-                    finally {
-                        this.#savingOperation = null;
-                    }
+                try {
+                    return this.#saveOperation(this.#savingOperation);
+                }
+                finally {
+                    this.#savingOperation = null;
                 }
             }
         }
+        #saveOperation(operation) {
+            const transaction = txnForContext(this.getContext());
+            if (transaction.save)
+                return transaction.save(operation) || operation.promise || operation.result;
+        }
         addTo(property, value) {
             if (typeof value === 'number' || typeof value === 'bigint') {
                 if (this.#savingOperation?.fullUpdate)
@@ -1210,7 +1232,7 @@ function makeTable(options) {
             }
             return (0, when_ts_1.when)(allowed, (allowed) => {
                 if (!allowed) {
-                    throw new hdbError_js_1.AccessViolation(context.user);
+                    throw new hdbError_ts_1.AccessViolation(context.user);
                 }
                 this._writeInvalidate(target ? requestTargetToId(target) : this.getId());
             });
@@ -1287,7 +1309,7 @@ function makeTable(options) {
                     }
                     logger_ts_1.logger.trace?.(`Relocating entry id: ${id}, timestamp: ${new Date(txnTime).toISOString()}`);
                     updateRecord(id, newRecord, existingEntry, txnTime, metadata, audit, {
-                        user: context.user,
+                        user: context?.user,
                         residencyId: options.residencyId,
                         nodeId: options.nodeId,
                         viaNodeId: options?.viaNodeId,
@@ -1370,7 +1392,7 @@ function makeTable(options) {
                 // RocksDB: eviction writes went directly into the raw transaction via options;
                 // commit it directly, as DatabaseTransaction.commit() would abort it (no tracked writes).
                 // Wrap in Promise.resolve so callers can rely on a thenable return regardless of engine.
-                return Promise.resolve(transaction.commit());
+                return transaction.commit();
             }
             finally {
                 if (!committed) {
@@ -1400,11 +1422,12 @@ function makeTable(options) {
          * Store the provided record data into the current resource. This is not written
          * until the corresponding transaction is committed.
          */
+        // @ts-expect-error The implementation intentionally uses a different argument order for back-compat
         put(target, record) {
             if (record === undefined || record instanceof URLSearchParams) {
-                // legacy argument position, shift the arguments and go through the update method for back-compat
-                this.update(target, true);
-                return this.save();
+                // legacy argument position, shift the arguments and go through the update method for back-compat.
+                // `when` settles the embed hook before `save()` so the write is staged first.
+                return (0, when_ts_1.when)(this.update(target, true), () => this.save());
             }
             else {
                 let allowed = true;
@@ -1417,20 +1440,26 @@ function makeTable(options) {
                 }
                 return (0, when_ts_1.when)(allowed, (allowed) => {
                     if (!allowed) {
-                        throw new hdbError_js_1.AccessViolation(context.user);
+                        throw new hdbError_ts_1.AccessViolation(context.user);
                     }
                     // standard path, handle arrays as multiple updates, and otherwise do a direct update
                     if (Array.isArray(record)) {
-                        return Promise.all(record.map((element) => {
+                        // Capture each element's operation synchronously (before any async `@embed`
+                        // hook resolves): `#savingOperation` is a single field that parallel writes
+                        // would otherwise clobber, so a deferred `save()` would commit the wrong op
+                        // — e.g. one element's save running before a later element's vector is written.
+                        const writes = record.map((element) => {
                             const id = element[primaryKey];
-                            this._writeUpdate(id, element, true);
-                            return this.save();
-                        }));
+                            const writePromise = this._writeUpdate(id, element, true);
+                            const operation = this.#savingOperation;
+                            return (0, when_ts_1.when)(writePromise, () => this.#saveOperation(operation));
+                        });
+                        this.#savingOperation = null;
+                        return Promise.all(writes);
                     }
                     else {
                         const id = requestTargetToId(target);
-                        this._writeUpdate(id, record, true);
-                        return this.save();
+                        return (0, when_ts_1.when)(this._writeUpdate(id, record, true), () => this.save());
                     }
                 });
             }
@@ -1453,7 +1482,7 @@ function makeTable(options) {
             }
             return (0, when_ts_1.when)(allowed, (allowed) => {
                 if (!allowed) {
-                    throw new hdbError_js_1.AccessViolation(context.user);
+                    throw new hdbError_ts_1.AccessViolation(context.user);
                 }
                 let id = requestTargetToId(target) ?? record[primaryKey];
                 if (id === undefined) {
@@ -1463,19 +1492,21 @@ function makeTable(options) {
                 else {
                     const existing = primaryStore.getSync(id);
                     if (existing) {
-                        throw new hdbError_js_1.ClientError('Record already exists', 409);
+                        throw new hdbError_ts_1.ClientError('Record already exists', 409);
                     }
                 }
-                this._writeUpdate(id, record, true);
-                return record;
+                // `_writeUpdate` may return a promise when an `@embed` directive
+                // requires running an embedder before the per-write `commit(...)`
+                // closure. `when()` passes through synchronous returns.
+                return (0, when_ts_1.when)(this._writeUpdate(id, record, true), () => record);
             });
         }
         // @ts-expect-error The implementation handles the possibility of target and recordUpdate being swapped
         patch(target, recordUpdate) {
             if (recordUpdate === undefined || recordUpdate instanceof URLSearchParams) {
-                // legacy argument position, shift the arguments and go through the update method for back-compat
-                this.update(target, false);
-                return this.save();
+                // legacy argument position, shift the arguments and go through the update method for back-compat.
+                // `when` settles the embed hook before `save()` so the write is staged first.
+                return (0, when_ts_1.when)(this.update(target, false), () => this.save());
             }
             else {
                 // standard path, ensure there is no return object
@@ -1484,6 +1515,7 @@ function makeTable(options) {
                 });
             }
         }
+        // #section: write-path-internals
         // perform the actual write operation; this may come from a user request to write (put, post, etc.), or
         // a notification that a write has already occurred in the canonical data source, we need to update our
         // local copy
@@ -1796,8 +1828,21 @@ function makeTable(options) {
                 },
             };
             this.#savingOperation = write;
-            write.beforeIntermediate = preCommitBlobsForRecordBefore(write, recordUpdate);
-            return transaction.addWrite(write);
+            // `@embed` hook must run before `addWrite` so the embedder's vector is on the
+            // record when `commit` runs. (The txn `before` slot runs after commit, which
+            // suits blob writes but not embedding, where the vector must be present at commit.)
+            // Known limitation of this write-time placement (a validate-time alternative was
+            // tried and reverted as a Harper-foreign pattern): the embedder sees this write's
+            // payload, before table validation — so a write that later fails validation still
+            // calls the backend, and a tracked-instance mutation (update(id,{}); row.source=…;
+            // save()) that sets the source via accessors after update() won't re-embed. A
+            // resource-layer re-embed is the proper fix; tracked as a follow-up.
+            const embedBefore = (0, embedHook_ts_1.buildEmbedBefore)(recordUpdate, context, options, TableResource.embedAttributes, TableResource.userEmbedders);
+            const proceed = () => {
+                write.beforeIntermediate = preCommitBlobsForRecordBefore(write, recordUpdate);
+                return transaction.addWrite(write);
+            };
+            return embedBefore ? embedBefore().then(proceed) : proceed();
         }
         async delete(target) {
             if (isSearchTarget(target)) {
@@ -1810,13 +1855,13 @@ function makeTable(options) {
             if (target) {
                 let allowed = true;
                 const context = this.getContext();
-                if (target.checkPermission) {
+                if (target?.checkPermission) {
                     // requesting authorization verification
                     allowed = this.allowDelete(context.user, target, context);
                 }
                 return (0, when_ts_1.when)(allowed, (allowed) => {
                     if (!allowed) {
-                        throw new hdbError_js_1.AccessViolation(context.user);
+                        throw new hdbError_ts_1.AccessViolation(context.user);
                     }
                     const id = requestTargetToId(target);
                     this._writeDelete(id);
@@ -1868,6 +1913,7 @@ function makeTable(options) {
             });
             return true;
         }
+        // #section: search-query
         search(target) {
             const context = this.getContext();
             const txn = txnForContext(context);
@@ -1879,7 +1925,7 @@ function makeTable(options) {
                 // requesting authorization verification
                 const allowed = this.allowRead(context.user, target, context);
                 if (!allowed) {
-                    throw new hdbError_js_1.AccessViolation(context.user);
+                    throw new hdbError_ts_1.AccessViolation(context.user);
                 }
             }
             if (context)
@@ -1922,18 +1968,39 @@ function makeTable(options) {
                         condition.conditions = prepareConditions(condition.conditions, condition.operator);
                         continue;
                     }
+                    // Normalize `not_X` comparator forms passed in via structured queries.
+                    // The REST parser already does this, but programmatic callers may
+                    // pass `not_in`, `not_starts_with`, etc. directly.
+                    if (condition.comparator) {
+                        const resolved = (0, search_ts_1.resolveComparator)(condition.comparator);
+                        if (resolved.negated) {
+                            condition.comparator = resolved.comparator;
+                            condition.negated = true;
+                        }
+                    }
                     const attribute_name = condition[0] ?? condition.attribute;
-                    const attribute = attribute_name == null ? primaryKeyAttribute : (0, search_ts_1.findAttribute)(attributes, attribute_name);
+                    let attribute = attribute_name == null ? primaryKeyAttribute : (0, search_ts_1.findAttribute)(attributes, attribute_name);
+                    if (!attribute && Array.isArray(attribute_name) && attribute_name.length > 1) {
+                        // Plain JSON nested path: the leaf may not be declared in the
+                        // schema. Fall back to the root attribute so we can validate
+                        // existence without requiring the inner structure to be typed.
+                        attribute = (0, search_ts_1.findAttribute)(attributes, attribute_name[0]);
+                    }
                     if (!attribute) {
                         if (attribute_name != null && !target.allowConditionsOnDynamicAttributes)
-                            throw (0, hdbError_js_1.handleHDBError)(new Error(), `${attribute_name} is not a defined attribute`, 404);
+                            throw (0, hdbError_ts_1.handleHDBError)(new Error(), `${attribute_name} is not a defined attribute`, 404);
                     }
                     else if (attribute.type || search_ts_1.COERCIBLE_OPERATORS[condition.comparator]) {
-                        // Do auto-coercion or coercion as required by the attribute type
-                        if (condition[1] === undefined)
-                            condition.value = coerceTypedValues(condition.value, attribute);
-                        else
-                            condition[1] = coerceTypedValues(condition[1], attribute);
+                        // Do auto-coercion or coercion as required by the attribute type.
+                        // Skipped for nested paths into plain JSON — the root attribute's
+                        // type is not the leaf type, so coercion would be wrong.
+                        const isNestedPathRoot = Array.isArray(attribute_name) && attribute_name.length > 1 && !attribute.relationship;
+                        if (!isNestedPathRoot) {
+                            if (condition[1] === undefined)
+                                condition.value = coerceTypedValues(condition.value, attribute);
+                            else
+                                condition[1] = coerceTypedValues(condition[1], attribute);
+                        }
                     }
                     if (condition.chainedConditions) {
                         if (condition.chainedConditions.length === 1 && (!condition.operator || condition.operator == 'and')) {
@@ -1958,7 +2025,7 @@ function makeTable(options) {
                             }
                             const isGe = lower.comparator === 'ge' || lower.comparator === 'greater_than_equal';
                             const isLe = upper.comparator === 'le' || upper.comparator === 'less_than_equal';
-                            condition.comparator = (isGe ? 'ge' : 'gt') + (isLe ? 'le' : 'lt');
+                            condition.comparator = ((isGe ? 'ge' : 'gt') + (isLe ? 'le' : 'lt'));
                             condition.value = [lower.value, upper.value];
                         }
                         else
@@ -1998,7 +2065,7 @@ function makeTable(options) {
                 if (operator !== 'or') {
                     const attribute_name = sort.attribute;
                     if (attribute_name == undefined)
-                        throw new hdbError_js_1.ClientError('Sort requires an attribute');
+                        throw new hdbError_ts_1.ClientError('Sort requires an attribute');
                     orderAlignedCondition = conditions.find((condition) => (0, search_ts_1.flattenKey)(condition.attribute) === (0, search_ts_1.flattenKey)(attribute_name));
                     if (orderAlignedCondition) {
                         // if there is a condition on the same attribute as the first sort, we can use it to align the sort
@@ -2007,14 +2074,14 @@ function makeTable(options) {
                     else {
                         const attribute = (0, search_ts_1.findAttribute)(attributes, attribute_name);
                         if (!attribute)
-                            throw (0, hdbError_js_1.handleHDBError)(new Error(), `${Array.isArray(attribute_name) ? attribute_name.join('.') : attribute_name} is not a defined attribute`, 404);
+                            throw (0, hdbError_ts_1.handleHDBError)(new Error(), `${Array.isArray(attribute_name) ? attribute_name.join('.') : attribute_name} is not a defined attribute`, 404);
                         if (attribute.indexed) {
                             // if it is indexed, we add a pseudo-condition to align with the natural sort order of the index
                             orderAlignedCondition = { ...sort, comparator: 'sort' };
                             conditions.push(orderAlignedCondition);
                         }
                         else if (conditions.length === 0 && !target.allowFullScan)
-                            throw (0, hdbError_js_1.handleHDBError)(new Error(), `${Array.isArray(attribute_name) ? attribute_name.join('.') : attribute_name} is not indexed and not combined with any other conditions`, 404);
+                            throw (0, hdbError_ts_1.handleHDBError)(new Error(), `${Array.isArray(attribute_name) ? attribute_name.join('.') : attribute_name} is not indexed and not combined with any other conditions`, 404);
                     }
                     if (orderAlignedCondition)
                         orderAlignedCondition.descending = Boolean(sort.descending);
@@ -2422,7 +2489,16 @@ function makeTable(options) {
                         else {
                             value = record[attribute_name];
                             if (value && typeof value === 'object' && attribute_name !== attribute) {
-                                value = TableResource.transformEntryForSelect(attribute.select || attribute, context, readTxn, null)({ value });
+                                const subTransform = TableResource.transformEntryForSelect(attribute.select || attribute, context, readTxn, null);
+                                // Plain JSON nested values: arrays project per-element so that
+                                // `select: [{ name: 'addresses', select: ['city'] }]` returns
+                                // `addresses: [{ city }, { city }]` rather than a single object.
+                                if (Array.isArray(value)) {
+                                    value = value.map((item) => item && typeof item === 'object' ? subTransform({ value: item }) : item);
+                                }
+                                else if (!(value instanceof Date)) {
+                                    value = subTransform({ value });
+                                }
                             }
                         }
                         callback(value, attribute_name);
@@ -2461,7 +2537,7 @@ function makeTable(options) {
                         }
                     }
                     else
-                        throw new hdbError_js_1.ClientError('Invalid select' + select);
+                        throw new hdbError_ts_1.ClientError('Invalid select' + select);
                     if (promises) {
                         return Promise.all(promises).then(() => selected);
                     }
@@ -2471,6 +2547,7 @@ function makeTable(options) {
             };
             return transform;
         }
+        // #section: pub-sub
         async subscribe(request) {
             if (!auditStore)
                 throw new Error('Can not subscribe to a table without an audit log');
@@ -2561,7 +2638,7 @@ function makeTable(options) {
                     // a collection should retrieve all descendant ids
                     if (startTime) {
                         if (count)
-                            throw new hdbError_js_1.ClientError('startTime and previousCount can not be combined for a table level subscription');
+                            throw new hdbError_ts_1.ClientError('startTime and previousCount can not be combined for a table level subscription');
                         // start time specified, get the audit history for this time range. We drop real-time
                         // messages during this loop because the snapshot:false cursor will pick them up itself.
                         pendingRealTimeQueue = null;
@@ -2772,7 +2849,7 @@ function makeTable(options) {
                 }
             })();
             result.catch((error) => {
-                harper_logger_js_1.default.error?.('Error in real-time subscription:', error);
+                harper_logger_ts_1.default.error?.('Error in real-time subscription:', error);
                 subscription.send(error);
             });
             function send(event) {
@@ -2809,13 +2886,13 @@ function makeTable(options) {
             else {
                 let allowed = true;
                 const context = this.getContext();
-                if (target.checkPermission) {
+                if (target?.checkPermission) {
                     // requesting authorization verification
-                    allowed = this.allowCreate(context.user, message, context);
+                    allowed = this.allowDelete(context.user, target, context);
                 }
                 return (0, when_ts_1.when)(allowed, (allowed) => {
                     if (!allowed) {
-                        throw new hdbError_js_1.AccessViolation(context.user);
+                        throw new hdbError_ts_1.AccessViolation(context.user);
                     }
                     const id = requestTargetToId(target);
                     this._writePublish(id, message, options);
@@ -2867,6 +2944,7 @@ function makeTable(options) {
             write.beforeIntermediate = preCommitBlobsForRecordBefore(write, message, undefined, true);
             transaction.addWrite(write);
         }
+        // #section: validation
         validate(record, patch) {
             let validationErrors;
             const validateValue = (value, attribute, name) => {
@@ -3001,9 +3079,10 @@ function makeTable(options) {
                 }
             }
             if (validationErrors) {
-                throw new hdbError_js_1.ClientError(validationErrors.join('. '));
+                throw new hdbError_ts_1.ClientError(validationErrors.join('. '));
             }
         }
+        // #section: stats-admin
         getUpdatedTime() {
             return this.#version;
         }
@@ -3011,9 +3090,9 @@ function makeTable(options) {
             const new_attributes = attributes.slice(0);
             for (const attribute of attributesToAdd) {
                 if (!attribute.name)
-                    throw new hdbError_js_1.ClientError('Attribute name is required');
+                    throw new hdbError_ts_1.ClientError('Attribute name is required');
                 if (attribute.name.match(/[`/]/))
-                    throw new hdbError_js_1.ClientError('Attribute names cannot include backticks or forward slashes');
+                    throw new hdbError_ts_1.ClientError('Attribute names cannot include backticks or forward slashes');
                 validateAttribute(attribute.name);
                 new_attributes.push(attribute);
             }
@@ -3126,6 +3205,18 @@ function makeTable(options) {
          * When attributes have been changed, we update the accessors that are assigned to this table
          */
         static updatedAttributes() {
+            // Refresh on every call: schema reload mutates `attributes` in place, so the
+            // class-construction snapshot would otherwise go stale.
+            this.embedAttributes = this.attributes.filter((a) => a?.embed);
+            // Drop registry entries for attributes that are no longer `@embed`, so a dropped
+            // directive doesn't leave a stale embedder or block a default refresh on re-add.
+            const embedNames = new Set(this.embedAttributes.map((a) => a.name));
+            for (const name of Object.keys(this.userEmbedders))
+                if (!embedNames.has(name))
+                    delete this.userEmbedders[name];
+            for (const name of this.userSetEmbedders)
+                if (!embedNames.has(name))
+                    this.userSetEmbedders.delete(name);
             propertyResolvers = this.propertyResolvers = {
                 $id: (object, context, entry) => ({ value: entry.key }),
                 $updatedtime: (object, context, entry) => entry.version,
@@ -3142,6 +3233,11 @@ function makeTable(options) {
                 attribute.resolve = null; // reset this
                 const relationship = attribute.relationship;
                 const computed = attribute.computed;
+                // Register the default embedder unless an author override is set. Sits outside
+                // the resolver chain below so `@embed` fields still flow through auto-HNSW indexing.
+                if (attribute.embed && !TableResource.userSetEmbedders.has(attribute.name)) {
+                    this.userEmbedders[attribute.name] = (0, embedHook_ts_1.createDefaultEmbedder)(attribute.embed);
+                }
                 if (relationship) {
                     if (attribute.indexed) {
                         console.error(`A relationship property can not be directly indexed, (but you may want to index the foreign key attribute)`);
@@ -3242,6 +3338,14 @@ function makeTable(options) {
                     if (typeof computed.from === 'function') {
                         this.setComputedAttribute(attribute.name, computed.from);
                     }
+                    else if (attribute.computedFromExpression) {
+                        // build a fallback scope object with all attribute names set to undefined,
+                        // matching the behavior in graphql.ts to prevent ReferenceErrors
+                        const attributesFallback = {};
+                        for (const attr of this.attributes)
+                            attributesFallback[attr.name] = undefined;
+                        this.setComputedAttribute(attribute.name, createComputedFrom(attribute.computedFromExpression, attributesFallback));
+                    }
                     propertyResolvers[attribute.name] = attribute.resolve = (object, context, entry) => {
                         const value = typeof computed.from === 'string' ? object[computed.from] : object;
                         const userResolver = this.userResolvers[attribute.name];
@@ -3295,6 +3399,7 @@ function makeTable(options) {
                 }
             }
         }
+        // #section: computed-history
         static setComputedAttribute(attribute_name, resolver) {
             const attribute = (0, search_ts_1.findAttribute)(attributes, attribute_name);
             if (!attribute) {
@@ -3307,6 +3412,25 @@ function makeTable(options) {
             }
             this.userResolvers[attribute_name] = resolver;
         }
+        /**
+         * Override the default embedder for an `@embed` attribute. Return the vector to
+         * store at `attribute_name`. The embedder receives the write payload (the fields
+         * present in the PUT/PATCH body), not the post-merge record, so multi-field
+         * concatenation only works when all source fields are in the same write.
+         */
+        static setEmbedAttribute(attribute_name, embedder) {
+            const attribute = (0, search_ts_1.findAttribute)(attributes, attribute_name);
+            if (!attribute) {
+                console.error(`The attribute "${attribute_name}" does not exist in the table "${tableName}"`);
+                return;
+            }
+            if (!attribute.embed) {
+                console.error(`The attribute "${attribute_name}" is not declared with @embed in the table "${tableName}"`);
+                return;
+            }
+            this.userEmbedders[attribute_name] = embedder;
+            this.userSetEmbedders.add(attribute_name);
+        }
         static async deleteHistory(endTime = 0, cleanupDeletedRecords = false) {
             let completion;
             for (const auditRecord of auditStore.getRange({
@@ -3408,7 +3532,7 @@ function makeTable(options) {
             }
         }
     }, () => {
-        throw new hdbError_js_1.ServerError('Service unavailable, exceeded request queue limit for resolving cache record', 503);
+        throw new hdbError_ts_1.ServerError('Service unavailable, exceeded request queue limit for resolving cache record', 503);
     });
     TableResource.updatedAttributes(); // on creation, update accessors as well
     if (expirationMs)
@@ -3438,8 +3562,8 @@ function makeTable(options) {
             hasChanges = true;
             const indexNulls = index.indexNulls;
             // determine what index values need to be removed and added
-            let valuesToAdd = (0, commonUtility_js_1.getIndexedValues)(value, indexNulls);
-            let valuesToRemove = (0, commonUtility_js_1.getIndexedValues)(existingValue, indexNulls);
+            let valuesToAdd = (0, commonUtility_ts_1.getIndexedValues)(value, indexNulls);
+            let valuesToRemove = (0, commonUtility_ts_1.getIndexedValues)(existingValue, indexNulls);
             let isLMDB = !!index.prefetch;
             if (valuesToRemove?.length > 0) {
                 // put this in a conditional so we can do a faster version for new records
@@ -3483,19 +3607,21 @@ function makeTable(options) {
     function checkValidId(id) {
         switch (typeof id) {
             case 'number':
+                if (isNaN(id))
+                    throw new hdbError_ts_1.ClientError('Invalid primary key of NaN', 400);
                 return true;
             case 'string':
                 if (id.length < 659)
                     return true; // max number of characters that can't expand our key size limit
                 if (id.length > MAX_KEY_BYTES) {
                     // we can quickly determine this is too big
-                    throw new Error('Primary key size is too large: ' + id.length);
+                    throw new hdbError_ts_1.ClientError('Primary key size is too large: ' + id.length, 400);
                 }
                 // TODO: We could potentially have a faster test here, Buffer.byteLength is close, but we have to handle characters < 4 that are escaped in ordered-binary
                 break; // otherwise we have to test it, in this range, unicode characters could put it over the limit
             case 'object':
                 if (id === null) {
-                    throw new Error('Invalid primary key of null');
+                    throw new hdbError_ts_1.ClientError('Invalid primary key of null', 400);
                 }
                 break; // otherwise we have to test it
             case 'bigint':
@@ -3503,13 +3629,13 @@ function makeTable(options) {
                     return true;
                 break; // otherwise we have to test it
             default:
-                throw new Error('Invalid primary key type: ' + typeof id);
+                throw new hdbError_ts_1.ClientError('Invalid primary key type: ' + typeof id, 400);
         }
         // otherwise it is difficult to determine if the key size is too large
         // without actually attempting to serialize it
         const length = (0, ordered_binary_1.writeKey)(id, TEST_WRITE_KEY_BUFFER, 0);
         if (length > MAX_KEY_BYTES)
-            throw new Error('Primary key size is too large: ' + id.length);
+            throw new hdbError_ts_1.ClientError('Primary key size is too large: ' + id.length, 400);
         return true;
     }
     function requestTargetToId(target) {
@@ -3550,7 +3676,7 @@ function makeTable(options) {
             // skip recording reads for most system tables except hdb_analytics
             // we want to track analytics reads in licensing, etc.
             if (databaseName !== 'system' && (options.type === 'read' || !options.type)) {
-                harper_logger_js_1.default.trace?.('Recording db-read action for', `${databaseName}.${tableName}`);
+                harper_logger_ts_1.default.trace?.('Recording db-read action for', `${databaseName}.${tableName}`);
                 (0, write_ts_1.recordAction)(entry?.size ?? 1, 'db-read', tableName, null);
             }
             // we need to freeze entry records to ensure the integrity of the cache;
@@ -3681,6 +3807,11 @@ function makeTable(options) {
         }
     }
     function ensureLoadedFromSource(source, id, entry, context, resource, target) {
+        if (context?.onlyIfCached) {
+            if (!entry?.value)
+                throw new hdbError_ts_1.ServerError('Entry is not cached', 504);
+            return;
+        }
         if (hasSourceGet) {
             let needsSourceData = false;
             if (context.noCache)
@@ -3711,11 +3842,9 @@ function makeTable(options) {
                     return entry;
                 });
                 // if the resource defines a method for indicating if stale-while-revalidate is allowed for a record
-                if (context?.onlyIfCached || (entry?.value && resource?.allowStaleWhileRevalidate?.(entry, id))) {
+                if (entry?.value && resource?.allowStaleWhileRevalidate?.(entry, id)) {
                     // since we aren't waiting for it any errors won't propagate so we should at least log them
                     loadingFromSource.catch((error) => logger_ts_1.logger.warn?.(error));
-                    if (context?.onlyIfCached && !resource.doesExist())
-                        throw new hdbError_js_1.ServerError('Entry is not cached', 504);
                     return; // go ahead and return and let the current stale value be used while we re-validate
                 }
                 else
@@ -3767,7 +3896,7 @@ function makeTable(options) {
             } while (true);
         }
         else {
-            transaction = isRocksDB ? new DatabaseTransaction_ts_1.ImmediateTransaction(primaryStore) : new LMDBTransaction_1.ImmediateTransaction(primaryStore);
+            transaction = (isRocksDB ? new DatabaseTransaction_ts_1.ImmediateTransaction(primaryStore) : new LMDBTransaction_1.ImmediateTransaction(primaryStore));
             if (context) {
                 context.transaction = transaction;
                 if (context.timestamp)
@@ -3972,16 +4101,15 @@ function makeTable(options) {
                             throw new Error('Only objects can be cached and stored in tables');
                         if (updatedRecord.status > 0 && updatedRecord.headers) {
                             // if the source has a status code and headers, treat it as a response
-                            if (updatedRecord.status >= 300) {
-                                if (updatedRecord.status === 304) {
-                                    // revalidation of our current cached record
-                                    updatedRecord = existingRecord;
-                                    version = existingVersion;
-                                }
-                                else {
-                                    // if the source has an error status, we need to throw an error
-                                    throw new hdbError_js_1.ServerError(updatedRecord.body || 'Error from source', updatedRecord.status);
-                                } // there are definitely more status codes to handle
+                            const status = updatedRecord.status;
+                            if (status === 304) {
+                                // revalidation of our current cached record
+                                updatedRecord = existingRecord;
+                                version = existingVersion;
+                            }
+                            else if (!CACHEABLE_STATUS_CODES.has(status)) {
+                                // non-cacheable status - propagate to client without caching
+                                throw new hdbError_ts_1.ServerError(updatedRecord.body || 'Error from source', status);
                             }
                             else {
                                 let headers;
@@ -4010,17 +4138,13 @@ function makeTable(options) {
                                 if (data !== undefined) {
                                     // we have structured data that we have parsed
                                     delete headers['content-type']; // don't store the content type if we have already parsed it
-                                    updatedRecord = {
-                                        headers,
-                                        data,
-                                    };
+                                    updatedRecord = { headers, data };
                                 }
                                 else {
-                                    updatedRecord = {
-                                        headers,
-                                        body: createBlob(updatedRecord.body),
-                                    };
+                                    updatedRecord = { headers, body: createBlob(updatedRecord.body) };
                                 }
+                                if (status !== 200)
+                                    updatedRecord.status = status;
                             }
                         }
                         if (typeof updatedRecord.toJSON === 'function')
@@ -4029,11 +4153,26 @@ function makeTable(options) {
                             updatedRecord[primaryKey] = id;
                     }
                     resolved = true;
-                    resolve({
+                    const resolvedEntry = {
                         key: id,
                         version,
                         value: updatedRecord,
-                    });
+                        expiresAt: sourceContext.expiresAt,
+                        metadataFlags: 0,
+                        size: 0,
+                        localTime: 0,
+                        nodeId: 0,
+                        residencyId: 0,
+                    };
+                    // Give the plain object the RecordObject prototype so getExpiresAt/getUpdatedTime
+                    // are available on the immediately-resolved entry. We mutate the prototype
+                    // in-place rather than copying so that the commit callback (which adds
+                    // createdAt/updatedAt to updatedRecord) is still reflected in the entry value.
+                    if (updatedRecord && updatedRecord.constructor === Object) {
+                        Object.setPrototypeOf(updatedRecord, primaryStore.encoder.structPrototype);
+                        RecordEncoder_ts_1.entryMap.set(updatedRecord, resolvedEntry);
+                    }
+                    resolve(resolvedEntry);
                 }
                 catch (error) {
                     error.message += ` while resolving record ${id} for ${tableName}`;
@@ -4164,6 +4303,14 @@ function makeTable(options) {
                         }
                     },
                 };
+                // The cache-from-source write bypasses `_writeUpdate`, so wire the embed hook here
+                // too (always the originating node). It runs after the client GET has resolved with
+                // fresh source data, so it's a background commit: an embedder failure aborts the cache
+                // write via the outer error handler (row re-embeds next read) and never reaches the
+                // caller. Source-resolution errors are handled earlier, with the stale-data fallback.
+                const embedBefore = (0, embedHook_ts_1.buildEmbedBefore)(updatedRecord, sourceContext, undefined, TableResource.embedAttributes, TableResource.userEmbedders);
+                if (embedBefore)
+                    await embedBefore();
                 sourceWrite.before = preCommitBlobsForRecordBefore(sourceWrite, updatedRecord);
                 dbTxn.addWrite(sourceWrite);
             }), () => {
@@ -4186,9 +4333,9 @@ function makeTable(options) {
         if (context.user?.role?.permission?.super_user)
             return true;
         if (context.replicateTo)
-            throw new hdbError_js_1.ClientError('Can not specify replication parameters without super user permissions', 403);
+            throw new hdbError_ts_1.ClientError('Can not specify replication parameters without super user permissions', 403);
         if (context.replicatedConfirmation)
-            throw new hdbError_js_1.ClientError('Can not specify replication confirmation without super user permissions', 403);
+            throw new hdbError_ts_1.ClientError('Can not specify replication confirmation without super user permissions', 403);
         return true;
     }
     function scheduleCleanup(priority) {
@@ -4392,7 +4539,9 @@ function makeTable(options) {
             return callSources
                 ? async () => {
                     // if we are calling the sources first and waiting for blobs, do those in order
-                    await callSources();
+                    const result = callSources();
+                    if (result && result.then)
+                        await result;
                     await preCommit.complete();
                 }
                 : () => preCommit.complete();
@@ -4414,6 +4563,17 @@ function attributesAsObject(attribute_permissions, type) {
 function noop() {
     // prefetch callback
 }
+/**
+ * Recreate a computed "from" function from a stored expression string. This is used when a table
+ * is loaded from metadata on a thread that hasn't loaded the GraphQL schema, so the computed
+ * function needs to be reconstructed from the persisted expression.
+ */
+function createComputedFrom(computedFromExpression, attributesFallback) {
+    const script = new node_vm_1.Script(attributesFallback
+        ? `function computed(attributes) { return function(record) { with(attributes) { with (record) { return ${computedFromExpression}; } } } } computed;`
+        : `function computed() { return function(record) { with (record) { return ${computedFromExpression}; } } } computed;`);
+    return script.runInThisContext()(attributesFallback);
+}
 const ENDS_WITH_TIMEZONE = /[+-][0-9]{2}:[0-9]{2}|[a-zA-Z]$/;
 /**
  * Coerce a string to the type defined by the attribute
@@ -4448,7 +4608,7 @@ function coerceType(value, attribute) {
             case 'BigInt':
                 return value === 'null' ? null : BigInt(value);
             case 'Boolean':
-                return (0, common_utils_js_1.autoCastBooleanStrict)(value);
+                return (0, common_utils_ts_1.autoCastBooleanStrict)(value);
             case 'Date':
                 if (isNaN(value)) {
                     if (value === 'null')
@@ -4464,7 +4624,7 @@ function coerceType(value, attribute) {
                 return new Date(+value); // epoch ms number
             case undefined:
             case 'Any':
-                return (0, common_utils_js_1.autoCast)(value);
+                return (0, common_utils_ts_1.autoCast)(value);
             default:
                 return value;
         }