@harperfast/harper 5.3.0-beta.4 → 5.3.0

package/dist/resources/Table.js

@@ -379,6 +379,7 @@ const MAX_OUT_OF_ORDER_AUDIT_DEPTH = 1000;
 const MAX_PREVIOUS_COUNT_SCAN = 10_000;
 const AUTHORIZATION_SELECT = Symbol.for('harper.authorizationSelect');
 const SEARCH_AUTHORIZATION_TRANSFORMS = Symbol.for('harper.searchAuthorizationTransforms');
+const FULL_TEXT_READ_PERMISSION = Symbol('fullTextReadPermission');
 const AUTHORIZATION_TRANSFORM_METHODS = ['map', 'filter', 'concat', 'flatMap', 'slice', 'mapError'];
 function propagateSearchAuthorization(iterable, authorization, source) {
     if (!iterable || typeof iterable !== 'object')
@@ -439,6 +440,30 @@ function cloneConditions(conditions) {
         return copy;
     });
 }
+function selectRequestsProperty(select, propertyName) {
+    if (!select)
+        return false;
+    const selected = Array.isArray(select) ? select : [select];
+    return selected.some((property) => (typeof property === 'string' ? property : property?.name) === propertyName);
+}
+function appendSelectProperty(select, propertyName) {
+    if (!Array.isArray(select))
+        return [select, propertyName];
+    return Object.assign([...select, propertyName], select);
+}
+function conditionsContainFullText(entries, definitions) {
+    for (const entry of entries) {
+        if (entry.conditions) {
+            if (conditionsContainFullText(entry.conditions, definitions))
+                return true;
+        }
+        else if ((0, search_ts_1.fullTextComparatorMode)(entry.comparator) &&
+            typeof (entry[0] ?? entry.attribute) === 'string' &&
+            definitions.some(({ name }) => name === (entry[0] ?? entry.attribute)))
+            return true;
+    }
+    return false;
+}
 // Ambient, path-scoped cycle guard for the enumerable-struct `toJSON` serialization path. A record on
 // a cyclically-enumerable table can (transitively) reference itself, which would recurse forever through
 // JSON.stringify. The struct getters resolve related records by id — and without a shared cache each
@@ -554,7 +579,9 @@ function isPlainOptions(value) {
 });
 function makeTable(options) {
     var _a;
-    const { primaryKey, indices, tableId, tableName, primaryStore, databasePath, databaseName, auditStore, schemaDefined, dbisDB: dbisDb, sealed, splitSegments, replicate, description, hidden, cacheControl, isBranch, fullTextIndexes = [], fullTextIndexGenerations = Object.create(null), fullTextIndexRetirements = [], } = options;
+    const { primaryKey, indices, tableId, tableName, primaryStore, databasePath, databaseName, auditStore, schemaDefined, dbisDB: dbisDb, sealed, splitSegments, replicate, description, hidden, cacheControl, isBranch, fullTextIndexes = [], fullTextFields = [], fullTextIndexGenerations = Object.create(null), fullTextIndexRetirements = [], } = options;
+    let declaredFullTextFields = Object.freeze([...fullTextFields]);
+    let fullTextFieldNames = fullTextFields.length > 0 ? new Set(fullTextFields) : undefined;
     const tableRootStore = primaryStore.rootStore;
     const tableRootPath = tableRootStore.path;
     let { expirationMS: expirationMs, evictionMS: evictionMs, audit, trackDeletes } = options;
@@ -890,6 +917,26 @@ function makeTable(options) {
         static derivedIndexRuntime;
         static audit = audit;
         static fullTextIndexes = fullTextIndexes;
+        static get fullTextFields() {
+            return declaredFullTextFields;
+        }
+        static set fullTextFields(names) {
+            declaredFullTextFields = Object.freeze([...names]);
+            fullTextFieldNames = names.length > 0 ? new Set(names) : undefined;
+        }
+        static assertFullTextSelection(select, sort) {
+            assertFullTextSelection(select);
+            for (let order = sort; order; order = order.next)
+                assertRecordField(order.attribute);
+        }
+        static assertFullTextRecordField(name) {
+            assertFullTextRecordField(name);
+        }
+        static isFullTextSearchEntryCurrent(entry) {
+            return !(entry.metadataFlags & (exports.INVALIDATED | exports.EVICTED | RecordEncoder_ts_1.VERSION_REUSED));
+        }
+        static fullTextQueryIndexes = Object.create(null);
+        static hasFullTextQueryIndexes = false;
         static fullTextIndexGenerations = fullTextIndexGenerations;
         static fullTextIndexRetirements = fullTextIndexRetirements;
         static hasCurrentFullTextIndexRetirements(names) {
@@ -1446,13 +1493,14 @@ function makeTable(options) {
                                                 hasChanges = true;
                                             }
                                         }
-                                        if (hasChanges || event.fullTextIndexes !== undefined) {
+                                        if (hasChanges || event.fullTextIndexes !== undefined || event.fullTextFields !== undefined) {
                                             const schemaVersion = this.schemaVersion;
                                             const definedTable = (0, databases_ts_1.table)({
                                                 table: tableName,
                                                 database: databaseName,
                                                 attributes: updatedAttributes,
                                                 fullTextIndexes: event.fullTextIndexes,
+                                                fullTextFields: event.fullTextFields,
                                                 origin: 'cluster',
                                             });
                                             if (definedTable.schemaVersion !== schemaVersion)
@@ -2263,6 +2311,11 @@ function makeTable(options) {
          */
         get(target) {
             const constructor = this.constructor;
+            if (fullTextFieldNames || hasRelationships)
+                assertFullTextSelection(target?.select);
+            if (fullTextFieldNames) {
+                assertRecordField(target?.property ?? (typeof target === 'string' && constructor.loadAsInstance !== false ? target : undefined));
+            }
             if (typeof target === 'string' && constructor.loadAsInstance !== false)
                 return this.getProperty(target);
             if (isSearchTarget(target)) {
@@ -2371,6 +2424,9 @@ function makeTable(options) {
          */
         allowRead(user, target, context) {
             const tablePermission = getTablePermissions(user, target);
+            // Resource.search consumes checkPermission before full-text source-field authorization runs.
+            if (_a.fullTextIndexes.length > 0 && target?.checkPermission && tablePermission)
+                target[FULL_TEXT_READ_PERMISSION] = tablePermission;
             if (tablePermission?.read) {
                 if (tablePermission.isSuperUser)
                     return true;
@@ -2388,7 +2444,11 @@ function makeTable(options) {
                         target.select = selectArray
                             .map((property) => {
                             const propertyName = property.name || property;
-                            if (!attrsForType || attrsForType[propertyName]) {
+                            if (!attrsForType ||
+                                attrsForType[propertyName] ||
+                                fullTextFieldNames?.has(propertyName) ||
+                                propertyName === '$score' ||
+                                propertyName === '$highlights') {
                                 const relatedTable = propertyResolvers[propertyName]?.definition?.tableClass;
                                 if (relatedTable) {
                                     // if there is a related table, we need to ensure the user has permission to read from that table and that attributes are properly restricted
@@ -2411,7 +2471,9 @@ function makeTable(options) {
                     }
                     else {
                         target.select = attribute_permissions
-                            .filter((attribute) => attribute.read && !propertyResolvers[attribute.attribute_name])
+                            .filter((attribute) => attribute.read &&
+                            !fullTextFieldNames?.has(attribute.attribute_name) &&
+                            !propertyResolvers[attribute.attribute_name])
                             .map((attribute) => attribute.attribute_name);
                     }
                     target[AUTHORIZATION_SELECT] = true;
@@ -2443,7 +2505,7 @@ function makeTable(options) {
                     // that the user doesn't have permission to remove
                     for (const permission of attribute_permissions) {
                         const key = permission.attribute_name;
-                        if (!permission.update && !(key in updatedData)) {
+                        if (!permission.update && !fullTextFieldNames?.has(key) && !(key in updatedData)) {
                             updatedData[key] = this.getProperty(key);
                         }
                     }
@@ -3522,6 +3584,8 @@ function makeTable(options) {
                 captureChanges,
                 validate: (txnTime, committedBy = transaction) => {
                     write.captureChanges?.();
+                    if (context?.source && !committedBy.isReplay)
+                        assertFullTextWrite(recordUpdate, true);
                     if (fullUpdate || (recordUpdate && (0, tracked_ts_1.hasChanges)(this.#changes === recordUpdate ? this : recordUpdate))) {
                         if (!context?.source) {
                             committedBy.checkOverloaded();
@@ -4393,6 +4457,8 @@ function makeTable(options) {
                 throw new Error('No query provided');
             if (target.parseError)
                 throw target.parseError; // if there was a parse error, we can throw it now
+            if (fullTextFieldNames || hasRelationships)
+                assertFullTextSelection(target.select);
             const getColumns = () => {
                 const select = target.select;
                 if (select) {
@@ -4536,6 +4602,7 @@ function makeTable(options) {
             conditions = cloneConditions(conditions);
             let orderAlignedCondition;
             let syntheticOrderCondition;
+            let includeFullTextHighlights = false;
             const filtered = {};
             function prepareConditions(conditions, operator) {
                 // some validation:
@@ -4568,6 +4635,50 @@ function makeTable(options) {
                         }
                     }
                     const attribute_name = condition[0] ?? condition.attribute;
+                    const fullTextMode = _a.fullTextIndexes.length > 0 ||
+                        Array.isArray(attribute_name) ||
+                        (typeof attribute_name === 'string' && fullTextFieldNames?.has(attribute_name))
+                        ? (0, search_ts_1.fullTextComparatorMode)(condition.comparator)
+                        : undefined;
+                    if (!fullTextMode && (fullTextFieldNames || hasRelationships))
+                        assertFullTextRecordField(attribute_name);
+                    const fullTextDefinition = fullTextMode && typeof attribute_name === 'string'
+                        ? _a.fullTextIndexes.find((definition) => definition.name === attribute_name)
+                        : undefined;
+                    if (fullTextMode && Array.isArray(attribute_name))
+                        throw new hdbError_ts_1.ClientError('Full-text predicates must directly name an index on the queried table', 400);
+                    if (fullTextMode && !fullTextDefinition)
+                        throwUnknownFullTextIndex(context, target, attribute_name);
+                    if (fullTextDefinition) {
+                        const fields = condition.fields;
+                        if (fields !== undefined) {
+                            if (!Array.isArray(fields) || fields.length === 0 || fields.some((field) => typeof field !== 'string'))
+                                throw new hdbError_ts_1.ClientError(`Full-text index '${attribute_name}' requires a non-empty fields list`, 400);
+                            assertFullTextReadAccess(context, target, fullTextDefinition, fields);
+                        }
+                        else
+                            assertFullTextReadAccess(context, target, fullTextDefinition);
+                        const value = condition[1] ?? condition.value;
+                        if (typeof value !== 'string' || value.length === 0)
+                            throw new hdbError_ts_1.ClientError(`Full-text index '${attribute_name}' requires a full-text comparator and non-empty string value`, 400);
+                        if (fullTextMode === 'phrase' && !fullTextDefinition.positions)
+                            throw new hdbError_ts_1.ClientError(`Full-text index '${attribute_name}' does not store phrase positions`, 400);
+                        if ((fullTextMode === 'prefix' || fullTextMode === 'fuzzy-prefix') && !fullTextDefinition.surfaceTerms)
+                            throw new hdbError_ts_1.ClientError(`Full-text index '${attribute_name}' does not store surface terms`, 400);
+                        if (fields !== undefined) {
+                            const sourceNames = new Set(fullTextDefinition.fields.map(({ name }) => name));
+                            if (new Set(fields).size !== fields.length || fields.some((field) => !sourceNames.has(field)))
+                                throw new hdbError_ts_1.ClientError(`Full-text index '${attribute_name}' contains an unknown or duplicate field`, 400);
+                        }
+                        condition.includeHighlights =
+                            condition.includeHighlights === true || selectRequestsProperty(target.select, '$highlights');
+                        if (condition.includeHighlights && !fullTextDefinition.highlighting)
+                            throw new hdbError_ts_1.ClientError(`Full-text index '${attribute_name}' does not enable highlighting`, 400);
+                        if (!_a.fullTextQueryIndexes[attribute_name]?.customIndex)
+                            throw new hdbError_ts_1.IndexRebuildingError(`Full-text index '${attribute_name}' is not ready`);
+                        includeFullTextHighlights ||= condition.includeHighlights;
+                        continue;
+                    }
                     let attribute = attribute_name == null ? primaryKeyAttribute : (0, search_ts_1.findAttribute)(attributes, attribute_name);
                     if (!attribute && Array.isArray(attribute_name) && attribute_name.length > 1) {
                         // Plain JSON nested path: the leaf may not be declared in the
@@ -4647,8 +4758,18 @@ function makeTable(options) {
             const operator = target.operator;
             if (conditions.length > 0 || operator)
                 conditions = prepareConditions(conditions, operator);
-            const sort = typeof target.sort === 'object' && target.sort;
+            let sort = typeof target.sort === 'object' && target.sort;
+            if (_a.fullTextIndexes.length > 0 &&
+                conditionsContainFullText(conditions, _a.fullTextIndexes)) {
+                if (target.reverse)
+                    throw new hdbError_ts_1.ClientError('Full-text results can only use descending $score order', 400);
+                if (sort && (sort.attribute !== '$score' || sort.next || sort.descending !== true))
+                    throw new hdbError_ts_1.ClientError('Full-text results can only use descending $score order', 400);
+                sort = undefined;
+            }
             for (let order = sort; order; order = order.next) {
+                if (fullTextFieldNames)
+                    assertRecordField(order.attribute);
                 if (typeof order.attribute !== 'string')
                     continue;
                 const customIndex = indices[order.attribute]?.customIndex;
@@ -4715,7 +4836,13 @@ function makeTable(options) {
                     postOrdering = sort;
                 }
             }
-            const select = target.select;
+            const select = includeFullTextHighlights
+                ? target.select === undefined
+                    ? ['*', '$highlights']
+                    : selectRequestsProperty(target.select, '$highlights')
+                        ? target.select
+                        : appendSelectProperty(target.select, '$highlights')
+                : target.select;
             // Whether the caller supplied real filter conditions — read from the raw request, NOT the
             // planner-augmented `conditions` (which by now may carry a synthetic `sort` pseudo-condition and
             // injected full-scan condition). Used to pick the count-estimate source below.
@@ -4816,7 +4943,11 @@ function makeTable(options) {
                         if (c.conditions)
                             return touchesCustomIndex(c.conditions);
                         const attr = Array.isArray(c.attribute) ? c.attribute[0] : (c.attribute ?? c[0]);
-                        return typeof attr === 'string' && Boolean(indices[attr]?.customIndex);
+                        if (typeof attr !== 'string')
+                            return false;
+                        return (0, search_ts_1.fullTextComparatorMode)(c.comparator)
+                            ? Boolean(_a.fullTextQueryIndexes?.[attr]?.customIndex)
+                            : Boolean(indices[attr]?.customIndex);
                     });
                     const approximateResultSet = typeof target.vectorFilter === 'function' || touchesCustomIndex(conditions);
                     return (async () => {
@@ -6092,6 +6223,8 @@ function makeTable(options) {
         }
         // #section: validation
         validate(record, patch) {
+            if (fullTextFieldNames)
+                assertFullTextWrite(record);
             // Accumulate structured per-field issues so the 400 carries `{ path, code,
             // message }[]` matching the emitted OpenAPI, instead of a single joined string. The joined
             // message is still built for the HTTP title, preserving back-compat for callers that read it.
@@ -6523,6 +6656,8 @@ function makeTable(options) {
                 $updatedTime: (object, context, entry) => entry.version,
                 $expiresAt: (object, context, entry) => entry.expiresAt,
                 $record: (object, context, entry) => (entry ? { value: object } : object),
+                $score: (object, context, entry) => entry?.$score,
+                $highlights: (object, context, entry) => entry?.$highlights,
                 $distance: (object, context, entry, returnEntry, sort) => {
                     if (!entry)
                         return;
@@ -6545,6 +6680,7 @@ function makeTable(options) {
                     return customIndex.propertyResolver(vector, context, entry, distanceSort);
                 },
             };
+            propertyResolvers.$highlights.directReturn = true;
             for (const attribute of this.attributes) {
                 if (attribute.isPrimaryKey)
                     primaryKeyAttribute = attribute;
@@ -7380,6 +7516,67 @@ function makeTable(options) {
             return table;
         }
     }
+    function assertRecordField(name) {
+        const field = Array.isArray(name) ? name[0] : name;
+        if (typeof field === 'string' && fullTextFieldNames?.has(field))
+            throw new hdbError_ts_1.ClientError(`Full-text field "${field}" is query-only; use a full-text comparator and select $score or $highlights`, 400);
+    }
+    function assertFullTextRecordField(name) {
+        if (!Array.isArray(name))
+            return assertRecordField(name);
+        const [first, ...remaining] = name;
+        assertRecordField(first);
+        if (remaining.length === 0)
+            return;
+        propertyResolvers[first]?.definition?.tableClass?.assertFullTextRecordField?.(remaining.length === 1 ? remaining[0] : remaining);
+    }
+    function assertFullTextSelection(select) {
+        if (!fullTextFieldNames && !hasRelationships)
+            return;
+        if (typeof select === 'string')
+            assertRecordField(select);
+        else if (Array.isArray(select)) {
+            for (const property of select) {
+                const name = typeof property === 'object' ? property?.name : property;
+                assertRecordField(name);
+                if (property && typeof property === 'object')
+                    propertyResolvers[name]?.definition?.tableClass?.assertFullTextSelection?.(property.select || (Array.isArray(property) ? property : undefined), property.sort);
+            }
+        }
+    }
+    function assertFullTextWrite(record, sourceFill = false) {
+        if (!fullTextFieldNames || !record || typeof record !== 'object')
+            return;
+        for (const name in record) {
+            if (fullTextFieldNames.has(name)) {
+                if (sourceFill)
+                    throw new hdbError_ts_1.ServerError(`Source for ${tableName} returned query-only full-text field "${name}"`, 502);
+                throw new hdbError_ts_1.ClientError(`Full-text field "${name}" is query-only and cannot be written`, 400);
+            }
+        }
+    }
+    function assertFullTextReadAccess(context, target, definition, requestedFields) {
+        const user = context?.user;
+        const permission = target[FULL_TEXT_READ_PERMISSION] ?? getTablePermissions(user, target);
+        // Calls without a principal or explicit permission are trusted internal calls, matching allowRead.
+        if (!permission && !user)
+            return;
+        if (permission?.isSuperUser || !permission?.attribute_permissions?.length)
+            return;
+        const readable = attributesAsObject(permission.attribute_permissions, 'read');
+        const searched = requestedFields ?? definition.fields.map(({ name }) => name);
+        if (searched.some((name) => !readable[name]))
+            throw new hdbError_ts_1.AccessViolation(user);
+    }
+    function throwUnknownFullTextIndex(context, target, attributeName) {
+        const user = context?.user;
+        const permission = target[FULL_TEXT_READ_PERMISSION] ?? getTablePermissions(user, target);
+        if (permission?.attribute_permissions?.length)
+            throw new hdbError_ts_1.AccessViolation(user);
+        if (typeof attributeName === 'string' && fullTextFieldNames?.has(attributeName))
+            throw new hdbError_ts_1.IndexRebuildingError(`Full-text index '${attributeName}' is unavailable`);
+        throw new hdbError_ts_1.ClientError('Full-text comparator requires a declared @fullText index', 400);
+    }
     function setLoadedFromSource(target, loadedFromSource) {
         // cache disposition is a per-get result, recorded on the RequestTarget of the get (#1576)
         // target may be a primitive id on instance-API calls, which can't hold the flag
@@ -7745,6 +7942,7 @@ function makeTable(options) {
             const commitPromise = (0, transaction_ts_1.transaction)(sourceContext, async (_txn) => {
                 const start = node_perf_hooks_1.performance.now();
                 let updatedRecord, assignCreatedTime, sourceVersion;
+                let reusedCachedRecord = false;
                 let hasChanges, invalidated;
                 try {
                     updatedRecord = await throttledCallToSource(source, id, sourceContext, existingEntry);
@@ -7787,6 +7985,7 @@ function makeTable(options) {
                             if (status === 304) {
                                 // revalidation of our current cached record
                                 updatedRecord = existingRecord;
+                                reusedCachedRecord = true;
                                 sourceVersion = existingVersion;
                             }
                             else if (!CACHEABLE_STATUS_CODES.has(status)) {
@@ -7862,6 +8061,8 @@ function makeTable(options) {
                             }
                         }
                         updatedRecord = (0, RecordEncoder_ts_1.storedFieldsOnly)(primaryStore.encoder, updatedRecord);
+                        if (!reusedCachedRecord)
+                            assertFullTextWrite(updatedRecord, true);
                         if (primaryKey && updatedRecord[primaryKey] !== id)
                             updatedRecord[primaryKey] = id;
                     }