@harperfast/harper 5.2.13 → 5.3.0-beta.1

package/dist/resources/search.js

@@ -18,8 +18,10 @@ const ordered_binary_1 = require("ordered-binary");
 const extended_iterable_1 = require("@harperfast/extended-iterable");
 const Table_ts_1 = require("./Table.js");
 const RecordEncoder_ts_1 = require("./RecordEncoder.js");
+const DatabaseTransaction_ts_1 = require("./DatabaseTransaction.js");
 const write_1 = require("./analytics/write");
 const rocksdb_js_1 = require("@harperfast/rocksdb-js");
+const Headers_ts_1 = require("../server/serverHelpers/Headers.js");
 // these are ratios/percentages of overall table size
 const OPEN_RANGE_ESTIMATE = 0.3;
 const BETWEEN_ESTIMATE = 0.1;
@@ -130,7 +132,12 @@ function executeConditions(conditions, operator, table, txn, request, context, t
             return executeConditions(condition.conditions, condition.operator, table, txn, request, context, transformToEntries, filtered
             // recordAccess intentionally omitted: guards run once, at the top level (see above).
             );
-        return searchByIndex(condition, txn, condition.descending || request.reverse === true, table, request.allowFullScan, filtered, context, request.limit !== undefined ? (request.offset || 0) + request.limit : undefined);
+        return searchByIndex(condition, txn, condition.descending || request.reverse === true, table, {
+            allowFullScan: request.allowFullScan,
+            filtered,
+            context,
+            minResults: request.limit === 0 ? 0 : request.limit !== undefined ? (request.offset || 0) + request.limit : undefined,
+        });
     }
     function mapConditionsToFilters(conditions, intersection, estimatedIncomingCount) {
         return conditions
@@ -213,19 +220,56 @@ function composeRecordFilter(recordFilters, table, context) {
     };
 }
 /**
- * Search for records or keys, based on the search condition, using an index if available
+ * A record's index entries are not adjacent — the composite `[indexedValue, primaryKey]` key sorts
+ * on the indexed value first — so comparing neighbours cannot collapse them and the scan has to
+ * remember what it already yielded (#2434).
+ *
+ * Heap cost is one key per distinct record in the scanned range, held for the life of the scan, and
+ * a page window does not cap it: sibling-condition and row filters run downstream, so a selective
+ * filter keeps the scan running and the set growing while the page fills. On a multi-million-row
+ * range that is hundreds of MB where main streamed at O(1). Accepted for #2434 over the bounded
+ * alternative, which costs a primary read per index entry.
+ */
+function distinctRecords(entries) {
+    const distinct = new extended_iterable_1.ExtendedIterable();
+    distinct.iterate = (options) => {
+        // A non-scalar key decodes to a fresh instance per entry, so identity never matches it;
+        // `writeKeyId` is the store's own key equality. Its bytes get their own set because a scalar
+        // string id can legitimately carry exactly those bytes, and one set would fold the two —
+        // the collision `flattenKey` has, and the reason Table.ts prefixes its key ids.
+        const yieldedKeys = new Set();
+        const yieldedEncodedKeys = new Set();
+        // map rather than filter: filter clears `continueOnRecoverableError` for everything below it
+        return entries
+            .map((primaryKey) => {
+            const isEncoded = typeof primaryKey === 'object' && primaryKey !== null;
+            const yielded = isEncoded ? yieldedEncodedKeys : yieldedKeys;
+            const identity = isEncoded ? (0, DatabaseTransaction_ts_1.writeKeyId)(primaryKey) : primaryKey;
+            if (yielded.has(identity))
+                return extended_iterable_1.SKIP;
+            yielded.add(identity);
+            return primaryKey;
+        })
+            .iterate(options);
+    };
+    return distinct;
+}
+/**
+ * Search for records or keys, based on the search condition, using an index if available. The four
+ * leading parameters are required at every call site; everything optional is named, so a new
+ * capability can be added without shifting positions at callers that don't use it.
  * @param searchCondition
  * @param transaction
  * @param reverse
  * @param Table
- * @param allowFullScan
- * @param filtered
+ * @param options
  */
-function searchByIndex(searchCondition, transaction, reverse, Table, allowFullScan, filtered, context, 
-// How many rows the query will ultimately consume (offset + limit), when it is bounded. An
-// approximate index returns a fixed-size candidate list, so without this a query asking for more
-// rows than that list holds silently gets a short result set. Only custom indexes read it.
-minResults) {
+function searchByIndex(searchCondition, transaction, reverse, Table, options = {}) {
+    // A stale positional caller passes `allowFullScan` here. Type checking only covers .ts callers, so
+    // fail loud rather than silently reading every option as undefined.
+    if (typeof options !== 'object' || options === null)
+        throw new TypeError('searchByIndex: the 5th argument is an options object (#2165), not a positional value');
+    const { allowFullScan, filtered, context, minResults } = options;
     let attribute_name = searchCondition[0] ?? searchCondition.attribute;
     let value = searchCondition[1] ?? searchCondition.value;
     const comparator = searchCondition.comparator;
@@ -233,6 +277,9 @@ minResults) {
         throw new hdbError_ts_1.ClientError(`Search condition for ${attribute_name} must have a value`);
     }
     let needFullScan;
+    // `[indexedValue, primaryKey]` is unique, so a scan that stays inside one indexed value cannot
+    // reach a record twice and skips the collapse below
+    let scansOneIndexedValue;
     if (Array.isArray(attribute_name)) {
         const firstAttributeName = attribute_name[0];
         // get the potential relationship attribute
@@ -249,7 +296,7 @@ minResults) {
                 value,
                 comparator,
                 negated: searchCondition.negated,
-            }, transaction, reverse, relatedTable, allowFullScan, joined);
+            }, transaction, reverse, relatedTable, { allowFullScan, filtered: joined });
             if (attribute.relationship.to) {
                 // this is one-to-many or many-to-many, so we need to track the filtering of related entries that match
                 filtered[attribute_name[0]] = joined;
@@ -261,7 +308,7 @@ minResults) {
                 const searchEntry = (relatedEntry) => {
                     if (relatedEntry?.key !== undefined)
                         relatedEntry = relatedEntry.key;
-                    return searchByIndex({ attribute: attribute.relationship.from, value: relatedEntry }, transaction, reverse, Table, allowFullScan, joined);
+                    return searchByIndex({ attribute: attribute.relationship.from, value: relatedEntry }, transaction, reverse, Table, { allowFullScan, filtered: joined });
                 };
                 if (attribute.elements) {
                     filtered[attribute_name[0]] = joined;
@@ -356,6 +403,7 @@ minResults) {
                 start = value;
                 end = value;
                 inclusiveEnd = true;
+                scansOneIndexedValue = true;
                 break;
             case 'in':
                 // Phase 1: route through filter — index-merge optimization is a Phase 2 follow-up.
@@ -473,31 +521,117 @@ minResults) {
             // exploring until it has enough MATCHING results, rather than post-filtering an under-filled
             // candidate set. Only indexes that opt in (filteredSearch) receive it; others post-filter as before.
             const recordFilter = index.customIndex.filteredSearch ? searchCondition.recordFilter : undefined;
-            const loaded = index.customIndex.search(searchCondition, context, recordFilter, minResults).map((entry) => {
-                // if the custom index returns an entry with metadata, merge it with the loaded entry
-                if (typeof entry === 'object' && entry) {
-                    const { key, ...otherProps } = entry;
-                    if (key == null)
-                        return extended_iterable_1.SKIP; // primaryKey missing from HNSW node — skip rather than crash
-                    const loadedEntry = Table.primaryStore.getEntry(key, {
-                        transaction: context && Table._readTxnForContext(context),
-                    });
-                    if (!loadedEntry)
-                        return extended_iterable_1.SKIP; // record was deleted/expired or not yet visible
-                    (0, Table_ts_1.freezeRecord)(loadedEntry?.value);
-                    recordRead(loadedEntry);
-                    return { ...otherProps, ...loadedEntry };
-                }
-                return entry;
+            const waiting = index.customIndex.filePrimary && searchCondition.waitForIndexMilliseconds > 0;
+            const start = waiting ? Promise.withResolvers() : undefined;
+            const controller = waiting ? new AbortController() : undefined;
+            const checkActive = waiting ? (0, DatabaseTransaction_ts_1.getReadTransactionGuard)(transaction) : undefined;
+            const signal = waiting
+                ? context.signal
+                    ? AbortSignal.any([context.signal, controller.signal])
+                    : controller.signal
+                : undefined;
+            const searchContext = waiting
+                ? Object.create(context, {
+                    signal: { value: signal },
+                    indexSearchStart: { value: start.promise.then(() => checkActive?.()) },
+                })
+                : context;
+            const searched = index.customIndex.search(searchCondition, searchContext, {
+                filter: waiting && recordFilter
+                    ? (id) => {
+                        if (signal.aborted)
+                            return false;
+                        checkActive?.();
+                        return recordFilter(id);
+                    }
+                    : recordFilter,
+                minResults,
             });
-            if (index.customIndex.rescoreResults) {
-                const rescored = index.customIndex.rescoreResults(loaded, searchCondition, comparator, attribute_name);
-                if (rescored != null)
-                    return rescored;
+            const coverage = searched.indexCoverage;
+            if (!waiting && coverage && context?.responseHeaders) {
+                (0, Headers_ts_1.appendHeader)(context.responseHeaders, 'Harper-Index-Coverage', `${coverage.state}; lag=${coverage.lagUpperBoundMilliseconds}; tolerance=${coverage.maxLagMilliseconds}`, true);
+                (0, Headers_ts_1.appendHeader)(context.responseHeaders, 'Access-Control-Expose-Headers', 'Harper-Index-Coverage', true);
+            }
+            const processEntries = (entries) => {
+                const loaded = entries
+                    .map((entry) => {
+                    // if the custom index returns an entry with metadata, merge it with the loaded entry
+                    if (typeof entry === 'object' && entry) {
+                        const { key, ...otherProps } = entry;
+                        if (key == null)
+                            return extended_iterable_1.SKIP; // primaryKey missing from HNSW node — skip rather than crash
+                        const loadedEntry = Table.primaryStore.getEntry(key, {
+                            transaction: waiting ? transaction : context && Table._readTxnForContext(context),
+                        });
+                        if (!loadedEntry)
+                            return extended_iterable_1.SKIP; // record was deleted/expired or not yet visible
+                        (0, Table_ts_1.freezeRecord)(loadedEntry?.value);
+                        recordRead(loadedEntry);
+                        return { ...otherProps, ...loadedEntry };
+                    }
+                    return entry;
+                })
+                    .filter((entry) => entry !== extended_iterable_1.SKIP);
+                if (index.customIndex.rescoreResults) {
+                    const rescored = index.customIndex.rescoreResults(loaded, searchCondition, comparator, attribute_name);
+                    if (rescored != null)
+                        return rescored;
+                }
+                return loaded;
+            };
+            if (typeof searched?.then === 'function') {
+                let settled = false;
+                const pending = waiting
+                    ? searched
+                        .then((entries) => {
+                        if (controller.signal.aborted)
+                            return [];
+                        if (signal.aborted)
+                            throw signal.reason ?? new Error('Index search aborted', { cause: signal.reason });
+                        checkActive?.();
+                        return processEntries(entries);
+                    })
+                        .finally(() => {
+                        settled = true;
+                    })
+                    : searched.then(processEntries);
+                // A consumer may abandon this lazy iterable without calling next().
+                pending.catch(() => { });
+                const results = new extended_iterable_1.ExtendedIterable();
+                results.iterate = (options) => {
+                    if (!options?.async) {
+                        throw new Error('This index resolves search results asynchronously; the results must be consumed with async iteration');
+                    }
+                    // Overlapping next() calls share one cursor and cannot duplicate its first entry.
+                    const iteratorPromise = pending.then((entries) => entries[Symbol.iterator]());
+                    iteratorPromise.catch(() => { });
+                    let closed = false;
+                    return {
+                        next() {
+                            if (closed)
+                                return Promise.resolve({ done: true, value: undefined });
+                            start?.resolve();
+                            return iteratorPromise.then((inner) => {
+                                if (closed)
+                                    return { done: true, value: undefined };
+                                checkActive?.();
+                                return inner.next();
+                            });
+                        },
+                        return(value) {
+                            closed = true;
+                            if (!settled)
+                                controller?.abort();
+                            iteratorPromise.then((inner) => inner.return?.(value), () => { });
+                            return Promise.resolve({ done: true, value });
+                        },
+                    };
+                };
+                return results;
             }
-            return loaded;
+            return processEntries(searched);
         }
-        return index.getRange(rangeOptions).map(filter
+        const scanned = index.getRange(rangeOptions).map(filter
             ? function ({ key, value }) {
                 let recordMatcher;
                 if (typeof key === 'string' && key.length > terms_ts_1.MAX_SEARCH_KEY_LENGTH) {
@@ -520,6 +654,14 @@ minResults) {
                 }));
             }
             : ({ value }) => value);
+        // Collapsing AFTER the map is required, not incidental: the condition's own filter tests one
+        // element at a time, so a record whose first entry fails it and whose second passes must
+        // still be reached. Collapsing first would settle that record on the failing entry and drop
+        // it. Scoped to attributes the schema declares multi-valued; a runtime array under an
+        // undeclared attribute repeats the same way but stays outside this boundary.
+        return scansOneIndexedValue || !findAttribute(Table.attributes, attribute_name)?.elements
+            ? scanned
+            : distinctRecords(scanned);
     }
     else {
         return Table.primaryStore
@@ -1077,6 +1219,96 @@ function filterByType(searchCondition, Table, context, filtered, isPrimaryKey, e
         return recordFilter;
     }
 }
+/**
+ * Estimates a range comparator's entry count from the store's statistical estimate, blended with
+ * the table-fraction heuristic by the estimate's confidence so low-confidence estimates degrade
+ * to the old behavior. Returns undefined (caller falls back to the heuristic) when the store
+ * cannot estimate, the shape is unexpected, or the executed range wouldn't match this one.
+ */
+function estimateRangeCondition(table, condition, searchType, fraction) {
+    const attributeName = condition[0] ?? condition.attribute;
+    const isPrimaryKey = attributeName === table.primaryKey;
+    const store = isPrimaryKey ? table.primaryStore : usableIndex(table, attributeName);
+    // LMDB-backed and custom index stores do not implement estimateCount
+    if (typeof store?.estimateCount !== 'function')
+        return undefined;
+    let value = condition[1] ?? condition.value;
+    if (value instanceof Date)
+        value = value.getTime();
+    let range;
+    switch (searchType) {
+        case 'lt':
+            // `start: true` mirrors searchByIndex: `true` sorts above `null`, so an indexNulls
+            // index's `[null, primaryKey]` entries are outside the executed range
+            range = { start: true, end: value };
+            break;
+        case 'le':
+            range = { start: true, end: value, inclusiveEnd: true };
+            break;
+        case 'gt':
+            range = { start: value, exclusiveStart: true };
+            break;
+        case 'ge':
+            range = { start: value };
+            break;
+        case 'between':
+        case 'gele':
+        case 'gelt':
+        case 'gtlt':
+        case 'gtle': {
+            if (!Array.isArray(value))
+                return undefined;
+            let [start, end] = value;
+            if (start instanceof Date)
+                start = start.getTime();
+            if (end instanceof Date)
+                end = end.getTime();
+            range = {
+                start,
+                end,
+                inclusiveEnd: searchType === 'between' || searchType === 'gele' || searchType === 'gtle',
+                exclusiveStart: searchType === 'gtlt' || searchType === 'gtle',
+            };
+            break;
+        }
+        case 'starts_with': {
+            const prefix = value?.toString();
+            if (!prefix)
+                return undefined;
+            range = { start: prefix, end: getStringPrefixUpperBound(prefix) };
+            break;
+        }
+        case 'prefix': {
+            let start = Array.isArray(value) ? value : [value, null];
+            if (start[start.length - 1] != null)
+                start = start.concat(null);
+            const end = start.slice(0);
+            end[end.length - 1] = ordered_binary_1.MAXIMUM_KEY;
+            range = { start, end };
+            break;
+        }
+        default:
+            return undefined;
+    }
+    // Long string bounds get truncated + filtered at execution (searchByIndex), so the
+    // executed range is wider than this one; don't estimate what won't be iterated.
+    if ((typeof range.start === 'string' && range.start.length > terms_ts_1.MAX_SEARCH_KEY_LENGTH) ||
+        (typeof range.end === 'string' && range.end.length > terms_ts_1.MAX_SEARCH_KEY_LENGTH)) {
+        return undefined;
+    }
+    let count, confidence;
+    try {
+        ({ count, confidence } = store.estimateCount(range) ?? {});
+    }
+    catch {
+        // a concurrently closing/dropped store must degrade the plan, not fail the query
+        return undefined;
+    }
+    if (!Number.isFinite(count) || count < 0 || !Number.isFinite(confidence) || confidence < 0 || confidence > 1)
+        return undefined;
+    const heuristic = fraction * estimatedEntryCount(table.primaryStore) + 1;
+    return Math.max(1, Math.round(confidence * count + (1 - confidence) * heuristic));
+}
 /** The index a condition can be driven by; searchByIndex refuses a rebuilding one, so it reads as absent. */
 function usableIndex(table, attributeName) {
     const index = attributeName == null ? undefined : table.indices[attributeName];
@@ -1133,6 +1365,14 @@ function estimateCondition(table) {
             if (drivesRebuildingIndex(table, condition[0] ?? condition.attribute)) {
                 condition.estimated_count = Infinity;
             }
+            else if (condition.negated) {
+                // a negated condition always executes as a full scan (searchByIndex forces
+                // needFullScan), so follow the filter-only convention used by contains/ends_with:
+                // estimate Infinity so its positive-range estimate can never win the driving-condition
+                // ordering. Short-circuit here rather than computing (and discarding) a positive-range
+                // estimate that would cross the native FFI boundary for nothing.
+                condition.estimated_count = Infinity;
+            }
             else if (searchType === terms_ts_1.SEARCH_TYPES.EQUALS || !searchType) {
                 const attribute_name = condition[0] ?? condition.attribute;
                 if (attribute_name == null || attribute_name === table.primaryKey)
@@ -1191,12 +1431,15 @@ function estimateCondition(table) {
                 }
                 else
                     condition.estimated_count = Infinity;
-                // for range queries (betweens, startsWith, greater, etc.), just arbitrarily guess
             }
             else if (searchType === 'starts_with' || searchType === 'prefix')
-                condition.estimated_count = STARTS_WITH_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
+                condition.estimated_count =
+                    estimateRangeCondition(table, condition, searchType, STARTS_WITH_ESTIMATE) ??
+                        STARTS_WITH_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
             else if (searchType === 'between')
-                condition.estimated_count = BETWEEN_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
+                condition.estimated_count =
+                    estimateRangeCondition(table, condition, searchType, BETWEEN_ESTIMATE) ??
+                        BETWEEN_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
             else if (searchType === 'sort') {
                 const attribute_name = condition[0] ?? condition.attribute;
                 const index = usableIndex(table, attribute_name);
@@ -1214,7 +1457,9 @@ function estimateCondition(table) {
                     // allow custom index to define its own estimation of counts
                     condition.estimated_count = index.customIndex.estimateCount(condition.value);
                 else
-                    condition.estimated_count = OPEN_RANGE_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
+                    condition.estimated_count =
+                        estimateRangeCondition(table, condition, searchType, OPEN_RANGE_ESTIMATE) ??
+                            OPEN_RANGE_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1;
             }
             // we give a condition significantly more weight/preference if we will be ordering by it
             if (typeof condition.descending === 'boolean')
@@ -1427,6 +1672,7 @@ function parseBlock(query, expectedEnd) {
                         break;
                     case 'group-by':
                         recordError('group by is not implemented yet');
+                        break;
                     case 'sort':
                         query.sort = toSortObject(args);
                         break;