@harperfast/harper 5.3.0-beta.4 → 5.3.0

package/dist/resources/databases.js

@@ -1276,12 +1276,14 @@ function initStores(path, rootStore, databaseName, { defaultTable, auditPath, is
             else if (warnings.length === 0)
                 warnedFullTextStates.delete(warningKey);
         }
+        const fullTextFields = (0, fullTextSchemaLifecycle_ts_1.readPersistedFullTextFields)(primaryAttribute.fullTextFields, primaryAttribute.fullTextIndexes, attributes, (message) => logger.warn(`${databaseName}.${tableName}: ${message}`));
         const fullTextIndexGenerations = persistedFullTextIndexGenerations(primaryAttribute.fullTextIndexGenerations, fullTextIndexes);
         const fullTextIndexRetirements = (0, fullTextSchema_ts_1.persistedFullTextIndexNames)(primaryAttribute.fullTextIndexRetirements);
         if (table && !recreateTable) {
             if (primaryAttribute.audit === true && table.audit !== true)
                 table.enableAuditing();
             table.fullTextIndexes = fullTextIndexes;
+            table.fullTextFields = fullTextFields;
             table.fullTextIndexGenerations = fullTextIndexGenerations;
             table.fullTextIndexRetirements = fullTextIndexRetirements;
             indices = table.indices;
@@ -1430,6 +1432,7 @@ function initStores(path, rootStore, databaseName, { defaultTable, auditPath, is
                 indices,
                 attributes,
                 fullTextIndexes,
+                fullTextFields,
                 fullTextIndexGenerations,
                 fullTextIndexRetirements,
                 schemaDefined: primaryAttribute.schemaDefined,
@@ -3036,7 +3039,7 @@ function reloadBranchAt(path) {
     return branch.tables;
 }
 function declareTable(target, tableDefinition) {
-    let { table: tableName, database: databaseName, expiration, eviction, scanInterval, attributes, audit, sealed, splitSegments, replicate, randomAccessFields, trackDeletes, schemaDefined, schemaRelationshipsDefined, origin, description, properties, hidden, cacheControl, isolatedApplicationOwner, fullTextIndexes, } = tableDefinition;
+    let { table: tableName, database: databaseName, expiration, eviction, scanInterval, attributes, audit, sealed, splitSegments, replicate, randomAccessFields, trackDeletes, schemaDefined, schemaRelationshipsDefined, origin, description, properties, hidden, cacheControl, isolatedApplicationOwner, fullTextIndexes, fullTextFields, } = tableDefinition;
     const auditExplicitlyEnabled = audit === true;
     const auditExplicitlyDisabled = audit === false;
     const fullTextIndexesExplicit = tableDefinition.fullTextIndexes !== undefined;
@@ -3245,6 +3248,7 @@ function declareTable(target, tableDefinition) {
     let published = false;
     let fullTextValuesForPersistence;
     let fullTextPersistencePending = false;
+    let fullTextFieldsForPersistence;
     let activeFullTextIndexes;
     let fullTextIndexGenerationMap = Object.create(null);
     let fullTextIndexRetirementNames;
@@ -3289,20 +3293,33 @@ function declareTable(target, tableDefinition) {
                 validateHnswOptions(lockedAttributesDbi, auditExplicitlyEnabled || (!auditExplicitlyDisabled && persistedAuditUnderLock === true));
             }
             let persistedPrimary = persistedPrimaryDescriptor(Table.dbisDB);
-            let persistedFullTextValues = persistedPrimary.descriptor?.fullTextIndexes;
+            const persistedFullTextState = (name, liveValue) => persistedPrimary.descriptor === undefined ? liveValue : persistedPrimary.descriptor[name];
+            let persistedFullTextValues = persistedFullTextState('fullTextIndexes', Table.fullTextIndexes);
+            if (origin === 'cluster') {
+                const declaredFields = new Set((0, fullTextSchemaLifecycle_ts_1.readPersistedFullTextFields)(persistedFullTextState('fullTextFields', Table.fullTextFields), persistedFullTextValues, catalogAttributes(Table.dbisDB), fullTextWarning));
+                attributes = attributes.filter((attribute) => {
+                    if (!declaredFields.has(attribute.name))
+                        return true;
+                    fullTextWarning(`Ignoring peer attribute '${attribute.name}'; the local full-text field is authoritative`);
+                    return false;
+                });
+            }
             const incomingFullTextValues = fullTextIndexesExplicit ? fullTextIndexes : [];
             let fullTextValidationAttributes;
             if ((persistedFullTextValues !== undefined && (rootStore instanceof rocksdb_js_1.RocksDatabase || fullTextIndexesExplicit)) ||
-                persistedPrimary.descriptor?.fullTextIndexRetirements !== undefined ||
+                persistedFullTextState('fullTextIndexRetirements', Table.fullTextIndexRetirements) !== undefined ||
+                (persistedFullTextState('fullTextFields', Table.fullTextFields) !== undefined &&
+                    (rootStore instanceof rocksdb_js_1.RocksDatabase || fullTextIndexesExplicit || fullTextFields !== undefined)) ||
+                (fullTextFields !== undefined && (!Array.isArray(fullTextFields) || fullTextFields.length > 0)) ||
                 Table.fullTextIndexes?.length > 0 ||
                 (fullTextIndexesExplicit && (!Array.isArray(incomingFullTextValues) || incomingFullTextValues.length > 0))) {
                 if (!(rootStore instanceof rocksdb_js_1.RocksDatabase)) {
                     exclusiveLock();
                     persistedPrimary = persistedPrimaryDescriptor(Table.dbisDB);
-                    persistedFullTextValues = persistedPrimary.descriptor?.fullTextIndexes;
+                    persistedFullTextValues = persistedFullTextState('fullTextIndexes', Table.fullTextIndexes);
                 }
                 const originalAttributes = Table.attributes.slice();
-                fullTextIndexRetirementNames = (0, fullTextSchema_ts_1.persistedFullTextIndexNames)(persistedPrimary.descriptor?.fullTextIndexRetirements);
+                fullTextIndexRetirementNames = (0, fullTextSchema_ts_1.persistedFullTextIndexNames)(persistedFullTextState('fullTextIndexRetirements', Table.fullTextIndexRetirements));
                 const relationshipAttributes = originalAttributes.filter((attribute) => attribute.relationship);
                 const originalMetadata = {
                     description: Table.description,
@@ -3310,6 +3327,12 @@ function declareTable(target, tableDefinition) {
                     cacheControl: Table.cacheControl,
                     schemaDefined: Table.schemaDefined,
                 };
+                const originalFullTextState = {
+                    indexes: Table.fullTextIndexes.slice(),
+                    fields: [...Table.fullTextFields],
+                    generations: Object.assign(Object.create(null), Table.fullTextIndexGenerations),
+                    retirements: [...Table.fullTextIndexRetirements],
+                };
                 armFullTextLiveStateRestore = () => {
                     if (restoreFullTextLiveState)
                         return;
@@ -3323,12 +3346,23 @@ function declareTable(target, tableDefinition) {
                         Object.assign(Table, originalMetadata);
                         Table.properties = (0, jsonSchemaTypes_ts_1.projectAttributesToProperties)(restored);
                         const restoredPrimary = persistedPrimaryDescriptor(Table.dbisDB).descriptor;
-                        const restoredFullTextIndexes = rootStore instanceof rocksdb_js_1.RocksDatabase && restoredPrimary?.audit === true
-                            ? (0, fullTextSchemaLifecycle_ts_1.readPersistedFullTextDefinitions)(restoredPrimary?.fullTextIndexes, restored, fullTextWarning)
+                        if (restoredPrimary === undefined) {
+                            Table.fullTextIndexes = originalFullTextState.indexes;
+                            Table.fullTextFields = originalFullTextState.fields;
+                            Table.fullTextIndexGenerations = originalFullTextState.generations;
+                            Table.fullTextIndexRetirements = originalFullTextState.retirements;
+                            Table.schemaVersion++;
+                            Table.updatedAttributes();
+                            (0, derivedIndexes_ts_1.refreshDerivedIndexes)(Table);
+                            return;
+                        }
+                        const restoredFullTextIndexes = rootStore instanceof rocksdb_js_1.RocksDatabase && restoredPrimary.audit === true
+                            ? (0, fullTextSchemaLifecycle_ts_1.readPersistedFullTextDefinitions)(restoredPrimary.fullTextIndexes, restored, fullTextWarning)
                             : [];
                         Table.fullTextIndexes = restoredFullTextIndexes;
-                        Table.fullTextIndexGenerations = persistedFullTextIndexGenerations(restoredPrimary?.fullTextIndexGenerations, restoredFullTextIndexes);
-                        Table.fullTextIndexRetirements = (0, fullTextSchema_ts_1.persistedFullTextIndexNames)(restoredPrimary?.fullTextIndexRetirements);
+                        Table.fullTextFields = (0, fullTextSchemaLifecycle_ts_1.readPersistedFullTextFields)(restoredPrimary.fullTextFields, restoredPrimary.fullTextIndexes, restored, fullTextWarning);
+                        Table.fullTextIndexGenerations = persistedFullTextIndexGenerations(restoredPrimary.fullTextIndexGenerations, restoredFullTextIndexes);
+                        Table.fullTextIndexRetirements = (0, fullTextSchema_ts_1.persistedFullTextIndexNames)(restoredPrimary.fullTextIndexRetirements);
                         Table.schemaVersion++;
                         Table.updatedAttributes();
                         (0, derivedIndexes_ts_1.refreshDerivedIndexes)(Table);
@@ -3355,7 +3389,14 @@ function declareTable(target, tableDefinition) {
                 // Before the full-text branches can write an interim descriptor; the check on the merged list
                 // below stays authoritative.
                 (0, embedHook_ts_1.assertDerivedFieldOwnership)(validationAttributes);
-                const persistedAudit = persistedPrimary.descriptor?.audit;
+                const validateFullTextFields = (definitions) => {
+                    const names = (0, fullTextSchema_ts_1.persistedFullTextIndexNames)(definitions).map((name) => ({ name }));
+                    if (origin === 'cluster')
+                        return (0, fullTextSchemaLifecycle_ts_1.mergePeerFullTextFields)(persistedFullTextState('fullTextFields', Table.fullTextFields), fullTextFields, persistedFullTextValues, definitions, validationAttributes, fullTextWarning);
+                    const requestedNames = new Set(names.map(({ name }) => name));
+                    return (0, fullTextSchema_ts_1.compileFullTextFields)(fullTextFields === undefined ? [...requestedNames] : fullTextFields, names, validationAttributes, true);
+                };
+                const persistedAudit = persistedPrimary.descriptor === undefined ? Table.audit : persistedPrimary.descriptor.audit;
                 const durableAudit = persistedAudit === true;
                 const finalAudit = origin === 'cluster'
                     ? durableAudit
@@ -3378,11 +3419,12 @@ function declareTable(target, tableDefinition) {
                 }
                 else if (fullTextIndexesExplicit) {
                     const compiled = (0, fullTextSchema_ts_1.compileFullTextDefinitions)(incomingFullTextValues, validationAttributes);
+                    fullTextFieldsForPersistence = validateFullTextFields(compiled);
                     if (compiled.length > 0 && !(rootStore instanceof rocksdb_js_1.RocksDatabase))
                         throw new hdbError_ts_1.ClientError(`Table '${databaseName}.${tableName}' cannot use @fullText with the LMDB storage engine`, 400);
                     if (compiled.length > 0 && !finalAudit)
                         throw new hdbError_ts_1.ClientError(`Table '${databaseName}.${tableName}' must enable audit logging before using @fullText because its transaction log is the derived-index recovery source`, 400);
-                    const pinAudit = compiled.length > 0 && persistedPrimary.descriptor?.audit !== true;
+                    const pinAudit = compiled.length > 0 && persistedAudit !== true;
                     const requestedNames = new Set(compiled.map(({ name }) => name));
                     const durableDefinitions = (0, fullTextSchemaLifecycle_ts_1.readPersistedFullTextDefinitions)(persistedFullTextValues, durableAttributes, fullTextWarning);
                     const removedNames = durableDefinitions.map(({ name }) => name).filter((name) => !requestedNames.has(name));
@@ -3409,13 +3451,18 @@ function declareTable(target, tableDefinition) {
                             const interimDefinitions = durableDefinitions.filter(({ name }) => requestedNames.has(name));
                             if (interimDefinitions.length > 0) {
                                 interimPrimary.fullTextIndexes = interimDefinitions;
-                                interimPrimary.fullTextIndexGenerations = persistedFullTextIndexGenerations(persistedPrimary.descriptor?.fullTextIndexGenerations, interimDefinitions);
+                                interimPrimary.fullTextIndexGenerations = persistedFullTextIndexGenerations(persistedFullTextState('fullTextIndexGenerations', Table.fullTextIndexGenerations), interimDefinitions);
                             }
                             else {
                                 delete interimPrimary.fullTextIndexes;
                                 delete interimPrimary.fullTextIndexGenerations;
                             }
-                            const retirements = new Set((0, fullTextSchema_ts_1.persistedFullTextIndexNames)(persistedPrimary.descriptor?.fullTextIndexRetirements));
+                            const interimFullTextFields = (0, fullTextSchemaLifecycle_ts_1.readPersistedFullTextFields)(persistedFullTextState('fullTextFields', Table.fullTextFields), persistedFullTextValues, durableAttributes, fullTextWarning).filter((name) => requestedNames.has(name));
+                            if (interimFullTextFields.length > 0)
+                                interimPrimary.fullTextFields = interimFullTextFields;
+                            else
+                                delete interimPrimary.fullTextFields;
+                            const retirements = new Set((0, fullTextSchema_ts_1.persistedFullTextIndexNames)(persistedFullTextState('fullTextIndexRetirements', Table.fullTextIndexRetirements)));
                             for (const name of transitionRetirements)
                                 retirements.add(name);
                             for (const name of requestedNames)
@@ -3444,11 +3491,12 @@ function declareTable(target, tableDefinition) {
                     fullTextValuesForPersistence = persistedFullTextValues;
                     activeFullTextIndexes = rootStore instanceof rocksdb_js_1.RocksDatabase && finalAudit ? retained : [];
                 }
+                fullTextFieldsForPersistence ??= validateFullTextFields(fullTextValuesForPersistence);
             }
             if (activeFullTextIndexes !== undefined) {
                 const durableGenerationDefinitions = (0, fullTextSchemaLifecycle_ts_1.readPersistedFullTextDefinitions)(persistedFullTextValues, fullTextValidationAttributes ?? Table.attributes, fullTextWarning);
                 const finalGenerationDefinitions = (0, fullTextSchemaLifecycle_ts_1.readPersistedFullTextDefinitions)(fullTextValuesForPersistence ?? persistedFullTextValues, fullTextValidationAttributes ?? Table.attributes, fullTextWarning);
-                fullTextIndexGenerationMap = (0, fullTextSchema_ts_1.reconcileFullTextIndexGenerations)(durableGenerationDefinitions, persistedPrimary.descriptor?.fullTextIndexGenerations, finalGenerationDefinitions, createFullTextIndexGeneration);
+                fullTextIndexGenerationMap = (0, fullTextSchema_ts_1.reconcileFullTextIndexGenerations)(durableGenerationDefinitions, persistedFullTextState('fullTextIndexGenerations', Table.fullTextIndexGenerations), finalGenerationDefinitions, createFullTextIndexGeneration);
                 (0, derivedIndexes_ts_1.assertFullTextActivationSupported)(rootStore, databaseName, tableName, activeFullTextIndexes);
                 if (activeFullTextIndexes.length > 0 &&
                     rootStore instanceof rocksdb_js_1.RocksDatabase &&
@@ -3528,12 +3576,20 @@ function declareTable(target, tableDefinition) {
                     activeFullTextIndexes = compiled;
                 }
             }
+            fullTextFieldsForPersistence =
+                origin === 'cluster'
+                    ? (0, fullTextSchemaLifecycle_ts_1.mergePeerFullTextFields)(undefined, fullTextFields, [], fullTextValuesForPersistence, attributes, fullTextWarning)
+                    : (0, fullTextSchema_ts_1.compileFullTextFields)(fullTextFields === undefined ? (0, fullTextSchema_ts_1.persistedFullTextIndexNames)(fullTextValuesForPersistence) : fullTextFields, (0, fullTextSchema_ts_1.persistedFullTextIndexNames)(fullTextValuesForPersistence).map((name) => ({ name })), attributes, true);
             const auditStore = rootStore.auditStore;
             primaryKeyAttribute = attributes.find((attribute) => attribute.isPrimaryKey) || {};
             primaryKey = primaryKeyAttribute.name;
             primaryKeyAttribute.isPrimaryKey = true;
             primaryKeyAttribute.is_hash_attribute = true; // backward-compat: harperdb@4.x reads this field to open the DBI with correct flags
             primaryKeyAttribute.schemaDefined = schemaDefined;
+            if (fullTextFieldsForPersistence.length > 0)
+                primaryKeyAttribute.fullTextFields = fullTextFieldsForPersistence;
+            else
+                delete primaryKeyAttribute.fullTextFields;
             // Old readers treat every attribute row as live schema, so relationships stay on the ignored primary descriptor.
             if (relationshipDefinitions)
                 primaryKeyAttribute.relationships = relationshipDefinitions;
@@ -3676,6 +3732,7 @@ function declareTable(target, tableDefinition) {
                 indices: {},
                 attributes,
                 fullTextIndexes: activeFullTextIndexes ?? [],
+                fullTextFields: fullTextFieldsForPersistence,
                 fullTextIndexGenerations: fullTextIndexGenerationMap,
                 fullTextIndexRetirements: fullTextIndexRetirementNames ?? [],
                 schemaDefined,
@@ -4100,17 +4157,22 @@ function declareTable(target, tableDefinition) {
         }
         if (!deferredPrimaryRow &&
             (fullTextPersistencePending ||
+                fullTextFieldsForPersistence !== undefined ||
                 (rootStore instanceof rocksdb_js_1.RocksDatabase &&
                     activeFullTextIndexes !== undefined &&
                     Array.isArray(fullTextValuesForPersistence)))) {
             const { key, descriptor } = persistedPrimaryDescriptor(attributesDbi);
             if (descriptor && !tableIsDropping(descriptor, key)) {
                 const updatedPrimary = { ...descriptor };
+                if (fullTextFieldsForPersistence?.length)
+                    updatedPrimary.fullTextFields = fullTextFieldsForPersistence;
+                else if (fullTextFieldsForPersistence)
+                    delete updatedPrimary.fullTextFields;
                 if (Array.isArray(fullTextValuesForPersistence) && fullTextValuesForPersistence.length > 0) {
                     updatedPrimary.fullTextIndexes = fullTextValuesForPersistence;
                     updatedPrimary.fullTextIndexGenerations = fullTextIndexGenerationMap;
                 }
-                else {
+                else if (Array.isArray(fullTextValuesForPersistence)) {
                     delete updatedPrimary.fullTextIndexes;
                     delete updatedPrimary.fullTextIndexGenerations;
                 }
@@ -4127,6 +4189,7 @@ function declareTable(target, tableDefinition) {
                 else
                     delete updatedPrimary.fullTextIndexRetirements;
                 if (!(0, fullTextSchemaLifecycle_ts_1.definitionsEqual)(descriptor.fullTextIndexes, updatedPrimary.fullTextIndexes) ||
+                    !(0, fullTextSchemaLifecycle_ts_1.definitionsEqual)(descriptor.fullTextFields, updatedPrimary.fullTextFields) ||
                     JSON.stringify(descriptor.fullTextIndexGenerations ?? {}) !==
                         JSON.stringify(updatedPrimary.fullTextIndexGenerations ?? {}) ||
                     JSON.stringify(descriptor.fullTextIndexRetirements ?? []) !==
@@ -4199,6 +4262,8 @@ function declareTable(target, tableDefinition) {
     finally {
         releaseLock();
     }
+    if (fullTextFieldsForPersistence !== undefined)
+        Table.fullTextFields = fullTextFieldsForPersistence;
     if (activeFullTextIndexes !== undefined) {
         Table.fullTextIndexes = activeFullTextIndexes;
         Table.fullTextIndexGenerations = fullTextIndexGenerationMap;