@harperfast/harper 5.0.30 → 5.1.0-beta.2

package/dist/resources/search.js

@@ -4,13 +4,15 @@ exports.COERCIBLE_OPERATORS = void 0;
 exports.executeConditions = executeConditions;
 exports.searchByIndex = searchByIndex;
 exports.findAttribute = findAttribute;
+exports.resolveComparator = resolveComparator;
+exports.getNestedValue = getNestedValue;
 exports.filterByType = filterByType;
 exports.estimateCondition = estimateCondition;
 exports.parseQuery = parseQuery;
 exports.flattenKey = flattenKey;
 exports.intersectionEstimate = intersectionEstimate;
-const hdbError_js_1 = require("../utility/errors/hdbError.js");
-const terms_js_1 = require("../utility/lmdb/terms.js");
+const hdbError_ts_1 = require("../utility/errors/hdbError.js");
+const terms_ts_1 = require("../utility/lmdb/terms.js");
 const ordered_binary_1 = require("ordered-binary");
 const extended_iterable_1 = require("@harperfast/extended-iterable");
 const Table_ts_1 = require("./Table.js");
@@ -21,6 +23,15 @@ const rocksdb_js_1 = require("@harperfast/rocksdb-js");
 const OPEN_RANGE_ESTIMATE = 0.3;
 const BETWEEN_ESTIMATE = 0.1;
 const STARTS_WITH_ESTIMATE = 0.05;
+// Synthetic Table-like object used to recurse through plain JSON nested-path
+// segments. It has no attributes, indices, or property resolvers, so the
+// recursive filterByType call dispatches purely on the comparator.
+const NESTED_PATH_TABLE = Object.freeze({
+    attributes: [],
+    indices: {},
+    primaryKey: null,
+    propertyResolvers: null,
+});
 const SYMBOL_OPERATORS = {
     // these are coercing operators
     '<': 'lt',
@@ -114,16 +125,17 @@ function searchByIndex(searchCondition, transaction, reverse, Table, allowFullSc
     let value = searchCondition[1] ?? searchCondition.value;
     const comparator = searchCondition.comparator;
     if (value === undefined && comparator !== 'sort') {
-        throw new hdbError_js_1.ClientError(`Search condition for ${attribute_name} must have a value`);
+        throw new hdbError_ts_1.ClientError(`Search condition for ${attribute_name} must have a value`);
     }
+    let needFullScan;
     if (Array.isArray(attribute_name)) {
         const firstAttributeName = attribute_name[0];
         // get the potential relationship attribute
         const attribute = findAttribute(Table.attributes, firstAttributeName);
-        if (attribute.relationship) {
+        if (attribute?.relationship) {
             // it is a join/relational query
             if (attribute_name.length < 2)
-                throw new hdbError_js_1.ClientError('Can not directly query a relational attribute, must query an attribute within the target table');
+                throw new hdbError_ts_1.ClientError('Can not directly query a relational attribute, must query an attribute within the target table');
             const relatedTable = attribute.definition?.tableClass || attribute.elements?.definition?.tableClass;
             const joined = new Map();
             // search the related table
@@ -131,6 +143,7 @@ function searchByIndex(searchCondition, transaction, reverse, Table, allowFullSc
                 attribute: attribute_name.length > 2 ? attribute_name.slice(1) : attribute_name[1],
                 value,
                 comparator,
+                negated: searchCondition.negated,
             }, transaction, reverse, relatedTable, allowFullScan, joined);
             if (attribute.relationship.to) {
                 // this is one-to-many or many-to-many, so we need to track the filtering of related entries that match
@@ -161,7 +174,10 @@ function searchByIndex(searchCondition, transaction, reverse, Table, allowFullSc
             attribute_name = attribute_name[0];
         }
         else {
-            throw new hdbError_js_1.ClientError('Unable to query by attribute ' + JSON.stringify(attribute_name));
+            // Non-relationship nested path (plain JSON path). Phase 1: filter-only —
+            // we don't have an index for the nested path, so fall through to a full
+            // scan with `filterByType` walking the path on each record.
+            needFullScan = true;
         }
     }
     const isPrimaryKey = attribute_name === Table.primaryKey || attribute_name == null;
@@ -170,86 +186,102 @@ function searchByIndex(searchCondition, transaction, reverse, Table, allowFullSc
     let end, inclusiveEnd, exclusiveStart;
     if (value instanceof Date)
         value = value.getTime();
-    let needFullScan;
-    switch (ALTERNATE_COMPARATOR_NAMES[comparator] || comparator) {
-        case 'lt':
-            start = true;
-            end = value;
-            break;
-        case 'le':
-            start = true;
-            end = value;
-            inclusiveEnd = true;
-            break;
-        case 'gt':
-            start = value;
-            exclusiveStart = true;
-            break;
-        case 'ge':
-            start = value;
-            break;
-        case 'prefix': // this is form finding multi-part keys that start with the provided prefix
-            // this search needs to be of the form:
-            // start: [prefix, null], end: [prefix, MAXIMUM_KEY]
-            if (!Array.isArray(value))
-                value = [value, null];
-            else if (value[value.length - 1] != null)
-                value = value.concat(null);
-            start = value;
-            end = value.slice(0);
-            end[end.length - 1] = ordered_binary_1.MAXIMUM_KEY;
-            break;
-        case 'starts_with':
-            start = value.toString();
-            end = value + String.fromCharCode(0xffff);
-            break;
-        case 'between':
-        case 'gele':
-        case 'gelt':
-        case 'gtlt':
-        case 'gtle':
-            start = value[0];
-            if (start instanceof Date)
-                start = start.getTime();
-            end = value[1];
-            if (end instanceof Date)
-                end = end.getTime();
-            inclusiveEnd = comparator === 'gele' || comparator === 'gtle' || comparator === 'between';
-            exclusiveStart = comparator === 'gtlt' || comparator === 'gtle';
-            break;
-        case 'equals':
-        case undefined:
-            start = value;
-            end = value;
-            inclusiveEnd = true;
-            break;
-        case 'ne':
-            if (value === null) {
-                // since null is the lowest value in an index, we can treat anything higher as a non-null
+    if (searchCondition.negated) {
+        // Negated conditions are filter-only in Phase 1: scan the whole index/table
+        // and exclude matching rows in the filter. Without overriding the range,
+        // the bounded index iteration would only visit *included* rows.
+        start = true;
+        needFullScan = true;
+    }
+    else
+        switch (ALTERNATE_COMPARATOR_NAMES[comparator] || comparator) {
+            case 'lt':
+                start = true;
+                end = value;
+                break;
+            case 'le':
+                start = true;
+                end = value;
+                inclusiveEnd = true;
+                break;
+            case 'gt':
                 start = value;
                 exclusiveStart = true;
                 break;
-            }
-        case 'sort': // this is a special case for when we want to get all records for sorting
-        case 'contains':
-        case 'ends_with':
-            // we have to revert to full table scan here
-            start = true;
-            needFullScan = true;
-            break;
-        default:
-            throw new hdbError_js_1.ClientError(`Unknown query comparator "${comparator}"`);
-    }
+            case 'ge':
+                start = value;
+                break;
+            case 'prefix': // this is form finding multi-part keys that start with the provided prefix
+                // this search needs to be of the form:
+                // start: [prefix, null], end: [prefix, MAXIMUM_KEY]
+                if (!Array.isArray(value))
+                    value = [value, null];
+                else if (value[value.length - 1] != null)
+                    value = value.concat(null);
+                start = value;
+                end = value.slice(0);
+                end[end.length - 1] = ordered_binary_1.MAXIMUM_KEY;
+                break;
+            case 'starts_with':
+                start = value.toString();
+                end = value + String.fromCharCode(0xffff);
+                break;
+            case 'between':
+            case 'gele':
+            case 'gelt':
+            case 'gtlt':
+            case 'gtle':
+                start = value[0];
+                if (start instanceof Date)
+                    start = start.getTime();
+                end = value[1];
+                if (end instanceof Date)
+                    end = end.getTime();
+                inclusiveEnd =
+                    comparator === 'gele' || comparator === 'gtle' || comparator === 'between';
+                exclusiveStart = comparator === 'gtlt' || comparator === 'gtle';
+                break;
+            case 'equals':
+            case undefined:
+                start = value;
+                end = value;
+                inclusiveEnd = true;
+                break;
+            case 'in':
+                // Phase 1: route through filter — index-merge optimization is a Phase 2 follow-up.
+                // `value` is expected to be an array (empty array matches no rows).
+                if (!Array.isArray(value))
+                    throw new hdbError_ts_1.ClientError(`"in" comparator requires an array value`);
+                start = true;
+                needFullScan = true;
+                break;
+            case 'ne':
+                if (value === null) {
+                    // since null is the lowest value in an index, we can treat anything higher as a non-null
+                    start = value;
+                    exclusiveStart = true;
+                    break;
+                }
+            case 'sort': // this is a special case for when we want to get all records for sorting
+            case 'contains':
+            case 'ends_with':
+                // we have to revert to full table scan here
+                start = true;
+                needFullScan = true;
+                break;
+            default:
+                throw new hdbError_ts_1.ClientError(`Unknown query comparator "${comparator}"`);
+        }
     let filter;
-    if (typeof start === 'string' && start.length > terms_js_1.MAX_SEARCH_KEY_LENGTH) {
+    if (typeof start === 'string' && start.length > terms_ts_1.MAX_SEARCH_KEY_LENGTH) {
         // if the key is too long, we need to truncate it and filter the results
-        start = start.slice(0, terms_js_1.MAX_SEARCH_KEY_LENGTH) + terms_js_1.OVERFLOW_MARKER;
+        start = start.slice(0, terms_ts_1.MAX_SEARCH_KEY_LENGTH) + terms_ts_1.OVERFLOW_MARKER;
         exclusiveStart = false;
         filter = filterByType(searchCondition, Table, null, filtered, isPrimaryKey);
     }
-    if (typeof end === 'string' && end.length > terms_js_1.MAX_SEARCH_KEY_LENGTH) {
+    if (typeof end === 'string' && end.length > terms_ts_1.MAX_SEARCH_KEY_LENGTH) {
         // if the key is too long, we need to truncate it and filter the results
-        end = end.slice(0, terms_js_1.MAX_SEARCH_KEY_LENGTH) + terms_js_1.OVERFLOW_MARKER;
+        end = end.slice(0, terms_ts_1.MAX_SEARCH_KEY_LENGTH) + terms_ts_1.OVERFLOW_MARKER;
         inclusiveEnd = true;
         filter = filter ?? filterByType(searchCondition, Table, null, filtered, isPrimaryKey);
     }
@@ -261,19 +293,23 @@ function searchByIndex(searchCondition, transaction, reverse, Table, allowFullSc
         exclusiveStart = !inclusiveEnd;
         inclusiveEnd = newEnd;
     }
-    if (!index || index.isIndexing || needFullScan || (value === null && !index.indexNulls)) {
+    // For negated conditions we need to consider records whose attribute value is
+    // missing from the index (e.g. nulls when the index doesn't index nulls), so
+    // we bypass the secondary index and iterate over the primary store.
+    const skipIndex = searchCondition.negated && index && !isPrimaryKey;
+    if (!index || index.isIndexing || needFullScan || (value === null && !index.indexNulls) || skipIndex) {
         // no indexed searching available, need a full scan
         if (allowFullScan === false && !index)
-            throw new hdbError_js_1.ClientError(`"${attribute_name}" is not indexed, can not search for this attribute`, 404);
+            throw new hdbError_ts_1.ClientError(`"${attribute_name}" is not indexed, can not search for this attribute`, 404);
         if (allowFullScan === false && needFullScan)
-            throw new hdbError_js_1.ClientError(`Can not use ${comparator || 'equal'} operator without combining with a condition that uses an index, can not search for attribute ${attribute_name}`, 403);
+            throw new hdbError_ts_1.ClientError(`Can not use ${comparator || 'equal'} operator without combining with a condition that uses an index, can not search for attribute ${attribute_name}`, 403);
         if (index?.isIndexing)
-            throw new hdbError_js_1.ServerError(`"${attribute_name}" is not indexed yet, can not search for this attribute`, 503);
+            throw new hdbError_ts_1.ServerError(`"${attribute_name}" is not indexed yet, can not search for this attribute`, 503);
         if (value === null && index && !index.indexNulls)
-            throw new hdbError_js_1.ClientError(`"${attribute_name}" is not indexed for nulls, index needs to be rebuilt to search for nulls, can not search for this attribute`, 400);
+            throw new hdbError_ts_1.ClientError(`"${attribute_name}" is not indexed for nulls, index needs to be rebuilt to search for nulls, can not search for this attribute`, 400);
         filter = filter ?? filterByType(searchCondition, Table, null, filtered, isPrimaryKey);
         if (!filter) {
-            throw new hdbError_js_1.ClientError(`Unknown search operator ${searchCondition.comparator}`);
+            throw new hdbError_ts_1.ClientError(`Unknown search operator ${searchCondition.comparator}`);
         }
     }
     const rangeOptions = {
@@ -318,9 +354,9 @@ function searchByIndex(searchCondition, transaction, reverse, Table, allowFullSc
         results.hasEntries = true;
         return results;
     }
-    else if (index) {
+    else if (index && !skipIndex) {
         if (index.customIndex) {
-            return index.customIndex.search(searchCondition, context).map((entry) => {
+            const loaded = index.customIndex.search(searchCondition, context).map((entry) => {
                 // if the custom index returns an entry with metadata, merge it with the loaded entry
                 if (typeof entry === 'object' && entry) {
                     const { key, ...otherProps } = entry;
@@ -337,11 +373,29 @@ function searchByIndex(searchCondition, transaction, reverse, Table, allowFullSc
                 }
                 return entry;
             });
+            // Rerank: a quantized index navigates on approximate distances, so for a nearest-neighbor
+            // (sort) query, recompute the exact distance from each loaded record's full-precision vector
+            // and re-sort — restoring exact ordering and $distance. (lt/le threshold queries still use the
+            // index's approximate distance for now; exact threshold filtering needs over-fetch — follow-up.)
+            if (index.customIndex.int8 &&
+                index.customIndex.exactDistance &&
+                comparator === 'sort' &&
+                searchCondition.target &&
+                typeof attribute_name === 'string') {
+                const rescored = loaded.filter((e) => e !== extended_iterable_1.SKIP && e && e.value);
+                for (const e of rescored)
+                    e.distance = index.customIndex.exactDistance(searchCondition, e.value[attribute_name]);
+                // comparison-based (not subtraction) so Infinity sentinels for missing vectors
+                // sort last without producing NaN (Infinity - Infinity).
+                rescored.sort((a, b) => (a.distance === b.distance ? 0 : a.distance < b.distance ? -1 : 1));
+                return rescored;
+            }
+            return loaded;
         }
         return index.getRange(rangeOptions).map(filter
             ? function ({ key, value }) {
                 let recordMatcher;
-                if (typeof key === 'string' && key.length > terms_js_1.MAX_SEARCH_KEY_LENGTH) {
+                if (typeof key === 'string' && key.length > terms_ts_1.MAX_SEARCH_KEY_LENGTH) {
                     // if it is an overflow string, need to get the actual value from the database
                     recordMatcher = Table.primaryStore.getSync(value);
                 }
@@ -581,12 +635,59 @@ const ALTERNATE_COMPARATOR_NAMES = {
     'ew': 'ends_with',
     'endsWith': 'ends_with',
     'ct': 'contains',
+    'includes': 'in',
     '>': 'gt',
     '>=': 'ge',
     '<': 'lt',
     '<=': 'le',
     '...': 'between',
 };
+// Comparators whose value is a list (array) of values. Used to recognize the
+// REST `(v1,v2,...)` value syntax during parsing.
+const LIST_VALUE_COMPARATORS = new Set(['in', 'between']);
+// Base comparators that accept the `not_` prefix to produce a negated form.
+// `not_equal` is an existing alias for `ne` and keeps its existing semantics.
+const NEGATABLE_BASE_COMPARATORS = new Set(['in', 'between', 'starts_with', 'ends_with', 'contains', 'equals']);
+/**
+ * Resolve a comparator name to a (possibly stripped) base comparator and a
+ * `negated` flag. Existing aliases are preserved as-is — the execution layer
+ * resolves them via `ALTERNATE_COMPARATOR_NAMES`. Only the `not_` prefix is
+ * stripped here, and only when the base is a recognized negatable comparator
+ * and the full name is not itself an existing alias (so `not_equal` keeps its
+ * historical mapping to `ne`).
+ */
+function resolveComparator(comparator) {
+    if (comparator == null)
+        return { comparator, negated: false };
+    // Preserve existing aliases (e.g., `not_equal` -> `ne`) — let execution-time
+    // alias resolution handle them so we don't change comparator strings stored
+    // on the condition object.
+    if (ALTERNATE_COMPARATOR_NAMES[comparator])
+        return { comparator, negated: false };
+    if (typeof comparator === 'string' && comparator.startsWith('not_')) {
+        const base = comparator.slice(4);
+        const baseResolved = ALTERNATE_COMPARATOR_NAMES[base] || base;
+        if (NEGATABLE_BASE_COMPARATORS.has(baseResolved)) {
+            return { comparator: base, negated: true };
+        }
+    }
+    return { comparator, negated: false };
+}
+/**
+ * Walk a nested-property path on a record. Returns the value at the path,
+ * or undefined if any intermediate property is missing.
+ */
+function getNestedValue(record, path) {
+    if (typeof path === 'string')
+        return record?.[path];
+    let current = record;
+    for (let i = 0; i < path.length; i++) {
+        if (current == null)
+            return undefined;
+        current = current[path[i]];
+    }
+    return current;
+}
 /**
  * Create a filter based on the search condition that can be used to test each supplied record.
  * @param {SearchObject} searchCondition
@@ -594,6 +695,7 @@ const ALTERNATE_COMPARATOR_NAMES = {
  */
 function filterByType(searchCondition, Table, context, filtered, isPrimaryKey, estimatedIncomingCount) {
     const comparator = searchCondition.comparator;
+    const negated = searchCondition.negated;
     let attribute = searchCondition[0] ?? searchCondition.attribute;
     let value = searchCondition[1] ?? searchCondition.value;
     if (Array.isArray(attribute)) {
@@ -603,9 +705,25 @@ function filterByType(searchCondition, Table, context, filtered, isPrimaryKey, e
             attribute = attribute[0];
         else if (attribute.length > 1) {
             const firstAttributeName = attribute[0];
-            // get the relationship attribute
-            const firstAttribute = findAttribute(Table.attributes, firstAttributeName);
-            const relatedTable = firstAttribute.definition?.tableClass || firstAttribute.elements.definition?.tableClass;
+            // get the relationship attribute (may be undefined for plain JSON paths)
+            const firstAttribute = findAttribute(Table?.attributes, firstAttributeName);
+            const relatedTable = firstAttribute?.definition?.tableClass || firstAttribute?.elements?.definition?.tableClass;
+            if (!relatedTable) {
+                // Plain JSON nested path — walk the path on each record. Array
+                // intermediates use `some` semantics (match if any element matches).
+                const restAttribute = attribute.length > 2 ? attribute.slice(1) : attribute[1];
+                const leafFilter = filterByType({ attribute: restAttribute, value, comparator, negated }, NESTED_PATH_TABLE, context, null, false, estimatedIncomingCount);
+                if (!leafFilter)
+                    return;
+                return function nestedRecordFilter(record, entry) {
+                    const subObject = record?.[firstAttributeName];
+                    if (subObject == null)
+                        return leafFilter(undefined, entry);
+                    if (Array.isArray(subObject))
+                        return subObject.some((item) => leafFilter(item, entry));
+                    return leafFilter(subObject, entry);
+                };
+            }
             // TODO: If this is a relationship, we can potentially make this more efficient by using the index
             // and retrieving the set of matching ids first
             const filterMap = filtered?.[firstAttributeName];
@@ -613,6 +731,7 @@ function filterByType(searchCondition, Table, context, filtered, isPrimaryKey, e
                 attribute: attribute.length > 2 ? attribute.slice(1) : attribute[1],
                 value,
                 comparator,
+                negated,
             }, relatedTable, context, filterMap?.[firstAttributeName]?.joined, attribute[1] === relatedTable.primaryKey, estimatedIncomingCount);
             if (!nextFilter)
                 return;
@@ -623,7 +742,7 @@ function filterByType(searchCondition, Table, context, filtered, isPrimaryKey, e
                 return;
             }
             const resolver = Table.propertyResolvers?.[firstAttributeName];
-            if (resolver.to)
+            if (resolver?.to)
                 nextFilter.to = resolver.to;
             let subIdFilter;
             const getSubObject = (record, entry) => {
@@ -688,7 +807,7 @@ function filterByType(searchCondition, Table, context, filtered, isPrimaryKey, e
                     filtered[firstAttributeName] = {
                         fromRecord(record) {
                             // this is called when selecting the fields to include in results
-                            const value = getSubObject(record).subObject;
+                            const value = getSubObject(record, undefined).subObject;
                             if (Array.isArray(value))
                                 return value.filter(nextFilter).map((value) => value[relatedTable.primaryKey]);
                             return value;
@@ -702,22 +821,27 @@ function filterByType(searchCondition, Table, context, filtered, isPrimaryKey, e
     }
     if (value instanceof Date)
         value = value.getTime();
+    let baseFilter;
     switch (ALTERNATE_COMPARATOR_NAMES[comparator] || comparator) {
-        case terms_js_1.SEARCH_TYPES.EQUALS:
+        case terms_ts_1.SEARCH_TYPES.EQUALS:
         case undefined:
-            return attributeComparator(attribute, (recordValue) => recordValue === value, true);
+            baseFilter = attributeComparator(attribute, (recordValue) => recordValue === value, true);
+            break;
         case 'contains':
-            return attributeComparator(attribute, (recordValue) => recordValue?.toString().includes(value));
+            baseFilter = attributeComparator(attribute, (recordValue) => recordValue?.toString().includes(value));
+            break;
         case 'ends_with':
-            return attributeComparator(attribute, (recordValue) => recordValue?.toString().endsWith(value));
+            baseFilter = attributeComparator(attribute, (recordValue) => recordValue?.toString().endsWith(value));
+            break;
         case 'starts_with':
-            return attributeComparator(attribute, (recordValue) => typeof recordValue === 'string' && recordValue.startsWith(value), true);
+            baseFilter = attributeComparator(attribute, (recordValue) => typeof recordValue === 'string' && recordValue.startsWith(value), true);
+            break;
         case 'prefix':
             if (!Array.isArray(value))
                 value = [value];
             else if (value[value.length - 1] == null)
                 value = value.slice(0, -1);
-            return attributeComparator(attribute, (recordValue) => {
+            baseFilter = attributeComparator(attribute, (recordValue) => {
                 if (!Array.isArray(recordValue))
                     return false;
                 for (let i = 0, l = value.length; i < l; i++) {
@@ -726,29 +850,65 @@ function filterByType(searchCondition, Table, context, filtered, isPrimaryKey, e
                 }
                 return true;
             }, true);
+            break;
         case 'between':
+        case 'gele':
+        case 'gtlt':
+        case 'gtle':
+        case 'gelt': {
             if (value[0] instanceof Date)
                 value[0] = value[0].getTime();
             if (value[1] instanceof Date)
                 value[1] = value[1].getTime();
-            return attributeComparator(attribute, (recordValue) => {
-                return (0, ordered_binary_1.compareKeys)(recordValue, value[0]) >= 0 && (0, ordered_binary_1.compareKeys)(recordValue, value[1]) <= 0;
+            const resolvedComparator = ALTERNATE_COMPARATOR_NAMES[comparator] || comparator;
+            const startInclusive = resolvedComparator === 'between' || resolvedComparator === 'gele' || resolvedComparator === 'gelt';
+            const endInclusive = resolvedComparator === 'between' || resolvedComparator === 'gele' || resolvedComparator === 'gtle';
+            baseFilter = attributeComparator(attribute, (recordValue) => {
+                const cmpStart = (0, ordered_binary_1.compareKeys)(recordValue, value[0]);
+                const cmpEnd = (0, ordered_binary_1.compareKeys)(recordValue, value[1]);
+                return (startInclusive ? cmpStart >= 0 : cmpStart > 0) && (endInclusive ? cmpEnd <= 0 : cmpEnd < 0);
             }, true);
+            break;
+        }
+        case 'in': {
+            if (!Array.isArray(value))
+                throw new hdbError_ts_1.ClientError(`"in" comparator requires an array value`);
+            // Cache the Set on the condition so multi-row evaluation reuses it.
+            let valueSet = searchCondition.cachedSet;
+            if (!valueSet) {
+                valueSet = new Set(value.map((v) => (v instanceof Date ? v.getTime() : v)));
+                searchCondition.cachedSet = valueSet;
+            }
+            baseFilter = attributeComparator(attribute, (recordValue) => valueSet.has(recordValue), true);
+            break;
+        }
         case 'gt':
-            return attributeComparator(attribute, (recordValue) => (0, ordered_binary_1.compareKeys)(recordValue, value) > 0);
+            baseFilter = attributeComparator(attribute, (recordValue) => (0, ordered_binary_1.compareKeys)(recordValue, value) > 0);
+            break;
         case 'ge':
-            return attributeComparator(attribute, (recordValue) => (0, ordered_binary_1.compareKeys)(recordValue, value) >= 0);
+            baseFilter = attributeComparator(attribute, (recordValue) => (0, ordered_binary_1.compareKeys)(recordValue, value) >= 0);
+            break;
         case 'lt':
-            return attributeComparator(attribute, (recordValue) => (0, ordered_binary_1.compareKeys)(recordValue, value) < 0);
+            baseFilter = attributeComparator(attribute, (recordValue) => (0, ordered_binary_1.compareKeys)(recordValue, value) < 0);
+            break;
         case 'le':
-            return attributeComparator(attribute, (recordValue) => (0, ordered_binary_1.compareKeys)(recordValue, value) <= 0);
+            baseFilter = attributeComparator(attribute, (recordValue) => (0, ordered_binary_1.compareKeys)(recordValue, value) <= 0);
+            break;
         case 'ne':
-            return attributeComparator(attribute, (recordValue) => (0, ordered_binary_1.compareKeys)(recordValue, value) !== 0, false, true);
+            baseFilter = attributeComparator(attribute, (recordValue) => (0, ordered_binary_1.compareKeys)(recordValue, value) !== 0, false, true);
+            break;
         case 'sort':
             return () => true;
         default:
-            throw new hdbError_js_1.ClientError(`Unknown query comparator "${comparator}"`);
+            throw new hdbError_ts_1.ClientError(`Unknown query comparator "${comparator}"`);
     }
+    if (negated && baseFilter) {
+        // Wrap with negation. Internal index-optimization state on `baseFilter`
+        // (idFilter, to, etc.) is intentionally not propagated through the
+        // negation wrapper — negated conditions are filter-only in Phase 1.
+        return (record, entry) => !baseFilter(record, entry);
+    }
+    return baseFilter;
     /** Create a comparison function that can take the record and check the attribute's value with the filter function */
     function attributeComparator(attribute, filter, canUseIndex, allowObjectMatching) {
         let thresholdRemainingMisses;
@@ -768,7 +928,9 @@ function filterByType(searchCondition, Table, context, filtered, isPrimaryKey, e
         let misses = 0;
         let filteredSoFar = 3; // what we use to calculate miss rate; we give some buffer so we don't jump to indexed retrieval too quickly
         function recordFilter(record) {
-            const value = record[attribute];
+            // `record` may be null/undefined when called via a nested-path filter
+            // where an intermediate property is missing.
+            const value = record == null ? undefined : record[attribute];
             let matches;
             if (typeof value !== 'object' || !value || allowObjectMatching)
                 matches = filter(value);
@@ -840,26 +1002,32 @@ function estimateCondition(table) {
             // skip if it is cached
             let searchType = condition.comparator || condition.search_type;
             searchType = ALTERNATE_COMPARATOR_NAMES[searchType] || searchType;
-            if (searchType === terms_js_1.SEARCH_TYPES.EQUALS || !searchType) {
+            if (searchType === terms_ts_1.SEARCH_TYPES.EQUALS || !searchType) {
                 const attribute_name = condition[0] ?? condition.attribute;
                 if (attribute_name == null || attribute_name === table.primaryKey)
                     condition.estimated_count = 1;
                 else if (Array.isArray(attribute_name) && attribute_name.length > 1) {
                     const attribute = findAttribute(table.attributes, attribute_name[0]);
-                    const relatedTable = attribute.definition?.tableClass || attribute.elements.definition?.tableClass;
-                    const estimate = estimateCondition(relatedTable)({
-                        value: condition.value,
-                        attribute: attribute_name.length > 2 ? attribute_name.slice(1) : attribute_name[1],
-                        comparator: 'equals',
-                    });
-                    const fromIndex = table.indices[attribute.relationship.from];
-                    // the estimated count is sum of the estimate of the related table and the estimate of the index
-                    condition.estimated_count =
-                        estimate +
-                            (fromIndex
-                                ? (estimate * estimatedEntryCount(table.indices[attribute.relationship.from])) /
-                                    (estimatedEntryCount(relatedTable.primaryStore) || 1)
-                                : estimate);
+                    const relatedTable = attribute?.definition?.tableClass || attribute?.elements?.definition?.tableClass;
+                    if (!relatedTable) {
+                        // Plain JSON nested path — no index, can't estimate cheaply.
+                        condition.estimated_count = Infinity;
+                    }
+                    else {
+                        const estimate = estimateCondition(relatedTable)({
+                            value: condition.value,
+                            attribute: attribute_name.length > 2 ? attribute_name.slice(1) : attribute_name[1],
+                            comparator: 'equals',
+                        });
+                        const fromIndex = table.indices[attribute.relationship?.from];
+                        // the estimated count is sum of the estimate of the related table and the estimate of the index
+                        condition.estimated_count =
+                            estimate +
+                                (fromIndex
+                                    ? (estimate * estimatedEntryCount(table.indices[attribute.relationship.from])) /
+                                        (estimatedEntryCount(relatedTable.primaryStore) || 1)
+                                    : estimate);
+                    }
                 }
                 else {
                     // we only attempt to estimate count on equals operator because that's really all that LMDB supports (some other key-value stores like libmdbx could be considered if we need to do estimated counts of ranges at some point)
@@ -876,6 +1044,23 @@ function estimateCondition(table) {
                 }
                 else
                     condition.estimated_count = Infinity;
+            }
+            else if (searchType === 'in') {
+                const attribute_name = condition[0] ?? condition.attribute;
+                const index = table.indices[attribute_name];
+                if (Array.isArray(condition.value) && index) {
+                    // Sum of per-value matches (over-counts duplicates but is a fine ceiling)
+                    let estimate = 0;
+                    for (const item of condition.value) {
+                        estimate += index.getValuesCount(item);
+                    }
+                    condition.estimated_count = estimate;
+                }
+                else if (Array.isArray(condition.value)) {
+                    condition.estimated_count = Infinity;
+                }
+                else
+                    condition.estimated_count = Infinity;
                 // for range queries (betweens, startsWith, greater, etc.), just arbitrarily guess
             }
             else if (searchType === 'starts_with' || searchType === 'prefix')
@@ -909,11 +1094,12 @@ function estimateCondition(table) {
     }
     return estimateConditionForTable;
 }
-class SyntaxViolation extends hdbError_js_1.Violation {
+class SyntaxViolation extends hdbError_ts_1.Violation {
 }
 const NEEDS_PARSER = /[()[\]|!<>.]|(=\w*=)/;
 const QUERY_PARSER = /([^?&|=<>!([{}\]),]*)([([{}\])|,&]|[=<>!]*)/g;
 const VALUE_PARSER = /([^&|=[\]{}]+)([[\]{}]|[&|=]*)/g;
+const FIQL_OPERATOR_NAME = /^[a-zA-Z_][a-zA-Z_0-9]*$/;
 let lastIndex;
 let currentQuery;
 let queryString;
@@ -986,8 +1172,10 @@ function parseBlock(query, expectedEnd) {
         switch (operator) {
             case '=':
                 if (attribute != undefined) {
-                    // a FIQL operator like =gt= (and don't allow just any string)
-                    if (value.length <= 2)
+                    // FIQL operator like =gt= or =starts_with= — accept any identifier
+                    // (letters, digits, underscores). Unknown comparator names are
+                    // caught at execution time with a clearer error.
+                    if (FIQL_OPERATOR_NAME.test(value))
                         comparator = value;
                     else
                         recordError(`invalid FIQL operator ${value}`);
@@ -1034,13 +1222,7 @@ function parseBlock(query, expectedEnd) {
                 else {
                     if (!query.conditions)
                         recordError('conditions/comparisons are not allowed in a property list');
-                    const condition = {
-                        comparator,
-                        attribute: attribute || null,
-                        value: valueDecoder(value),
-                    };
-                    if (comparator === 'eq')
-                        wildcardDecoding(condition, value);
+                    const condition = buildCondition(attribute, comparator, value, valueDecoder);
                     if (attribute === '') {
                         // this is a nested condition
                         const lastCondition = query.conditions[query.conditions.length - 1];
@@ -1186,13 +1368,7 @@ function parseBlock(query, expectedEnd) {
                     if (query.conditions) {
                         // finish condition
                         if (attribute) {
-                            const condition = {
-                                comparator: comparator || 'equals',
-                                attribute,
-                                value: valueDecoder(value),
-                            };
-                            if (comparator === 'eq')
-                                wildcardDecoding(condition, value);
+                            const condition = buildCondition(attribute, comparator || 'equals', value, valueDecoder);
                             assignOperator(query, lastBinaryOperator);
                             query.conditions.push(condition);
                         }
@@ -1256,7 +1432,7 @@ function typedDecoding(value) {
         else if (type === 'string')
             return decodeURIComponent(valueToCoerce);
         else
-            throw new hdbError_js_1.ClientError(`Unknown type ${type}`);
+            throw new hdbError_ts_1.ClientError(`Unknown type ${type}`);
     }
     return decodeURIComponent(value);
 }
@@ -1272,10 +1448,43 @@ function wildcardDecoding(condition, value) {
             condition.value = decodeURIComponent(value.slice(0, -1));
         }
         else {
-            throw new hdbError_js_1.ClientError('wildcard can only be used at the end of a string');
+            throw new hdbError_ts_1.ClientError('wildcard can only be used at the end of a string');
         }
     }
 }
+/**
+ * Build a condition from a parsed attribute, raw comparator name, raw value
+ * string, and the value decoder for the operator. Centralizes:
+ *  - alias resolution and `not_` prefix handling (for `negated`)
+ *  - `(v1,v2,...)` list-value syntax for list-taking comparators
+ *  - wildcard detection on `eq` (typed-equality)
+ */
+function buildCondition(attribute, rawComparator, rawValue, valueDecoder) {
+    const { comparator: resolvedComparator, negated } = resolveComparator(rawComparator);
+    let value;
+    if (LIST_VALUE_COMPARATORS.has(resolvedComparator) &&
+        rawValue.length >= 2 &&
+        rawValue.charCodeAt(0) === 0x28 /* ( */ &&
+        rawValue.charCodeAt(rawValue.length - 1) === 0x29 /* ) */) {
+        // `(v1,v2,...)` list-value syntax. Each element is decoded individually.
+        const inner = rawValue.slice(1, -1);
+        value = inner.length === 0 ? [] : inner.split(',').map(valueDecoder);
+    }
+    else {
+        value = valueDecoder(rawValue);
+    }
+    const condition = {
+        comparator: resolvedComparator,
+        attribute: attribute || null,
+        value,
+    };
+    if (negated)
+        condition.negated = true;
+    // preserve existing wildcard behavior on coercive equality
+    if (rawComparator === 'eq')
+        wildcardDecoding(condition, rawValue);
+    return condition;
+}
 function toSortObject(sort) {
     const sortObject = toSortEntry(sort[0]);
     if (sort.length > 1) {