@devisfuture/mega-collection 2.4.6 → 2.4.7

package/dist/search/nested.d.ts

@@ -0,0 +1,41 @@
+import { CollectionItem } from "../types";
+import type { SearchNestedCollectionStorage } from "./types";
+export declare class SearchNestedCollection<T extends CollectionItem> {
+    private readonly storage;
+    private readonly registeredFields;
+    private readonly fieldDescriptors;
+    constructor(storage?: SearchNestedCollectionStorage);
+    registerFields(fieldPaths?: readonly string[]): void;
+    hasRegisteredFields(): boolean;
+    hasField(fieldPath: string): boolean;
+    hasIndexes(): boolean;
+    clearIndexes(): void;
+    buildIndexes(data: T[]): void;
+    addItems(items: T[], startIndex: number): void;
+    updateItem(item: T, previousItem: T, itemIndex: number): void;
+    removeItem(item: T, itemIndex: number): void;
+    moveItem(item: T, fromIndex: number, toIndex: number): void;
+    /**
+     * Returns dataset indices of items matching the query across all registered
+     * nested fields. Deduplicates indices that appear in multiple nested fields.
+     */
+    searchAllIndexedFieldIndices(lowerQuery: string, uniqueQueryGrams: ReadonlySet<string>, restrictionLookup?: Uint8Array | null, candidateIndices?: readonly number[] | null): number[];
+    searchIndexedField(data: T[], fieldPath: string, lowerQuery: string, uniqueQueryGrams: ReadonlySet<string>, restrictionLookup?: Uint8Array | null, candidateIndices?: readonly number[] | null, take?: number): T[];
+    /**
+     * Core indexed search for a nested field: returns dataset indices of items
+     * whose nested values match the query.
+     */
+    searchIndexedFieldIndices(fieldPath: string, lowerQuery: string, uniqueQueryGrams: ReadonlySet<string>, restrictionLookup?: Uint8Array | null, candidateIndices?: readonly number[] | null, take?: number): number[];
+    searchFieldLinear(data: T[], fieldPath: string, lowerQuery: string): T[];
+    searchFieldLinearIndices(data: T[], fieldPath: string, lowerQuery: string, sourceIndices?: readonly number[]): number[];
+    matchesAnyField(item: T, lowerQuery: string): boolean;
+    private registerField;
+    private getNormalizedValues;
+    private buildIndex;
+    private addItemsToField;
+    private updateItemInField;
+    private removeItemFromField;
+    private moveItemForField;
+    private getNormalizedItemValue;
+}
+//# sourceMappingURL=nested.d.ts.map

package/dist/search/nested.mjs

@@ -0,0 +1,210 @@
+import { createNestedFieldDescriptor as e } from "../internal.mjs";
+import { indexLowerValue as t, intersectPostingLists as n, intersectPostingListsInCandidates as r, removeLowerValue as i } from "./ngram.mjs";
+//#region src/search/nested.ts
+var a = class {
+	constructor(e = {
+		ngramIndexes: /* @__PURE__ */ new Map(),
+		normalizedFieldValues: /* @__PURE__ */ new Map()
+	}) {
+		this.storage = e, this.registeredFields = /* @__PURE__ */ new Set(), this.fieldDescriptors = /* @__PURE__ */ new Map();
+	}
+	registerFields(e) {
+		if (e?.length) for (let t of e) this.registerField(t);
+	}
+	hasRegisteredFields() {
+		return this.registeredFields.size > 0;
+	}
+	hasField(e) {
+		return this.registeredFields.has(e);
+	}
+	hasIndexes() {
+		return this.storage.ngramIndexes.size > 0;
+	}
+	clearIndexes() {
+		this.storage.ngramIndexes.clear(), this.storage.normalizedFieldValues.clear();
+	}
+	buildIndexes(e) {
+		this.clearIndexes();
+		for (let t of this.registeredFields) this.buildIndex(e, t);
+	}
+	addItems(e, t) {
+		if (!(e.length === 0 || this.storage.ngramIndexes.size === 0)) for (let n of this.storage.ngramIndexes.keys()) this.addItemsToField(n, e, t);
+	}
+	updateItem(e, t, n) {
+		if (this.storage.ngramIndexes.size !== 0) for (let r of this.storage.ngramIndexes.keys()) this.updateItemInField(r, e, t, n);
+	}
+	removeItem(e, t) {
+		if (this.storage.ngramIndexes.size !== 0) for (let n of this.storage.ngramIndexes.keys()) this.removeItemFromField(n, e, t);
+	}
+	moveItem(e, t, n) {
+		if (!(this.storage.ngramIndexes.size === 0 || t === n)) for (let r of this.storage.ngramIndexes.keys()) this.moveItemForField(r, e, t, n);
+	}
+	searchAllIndexedFieldIndices(e, t, n, r) {
+		let i = /* @__PURE__ */ new Set(), a = [];
+		for (let o of this.storage.ngramIndexes.keys()) for (let s of this.searchIndexedFieldIndices(o, e, t, n, r)) i.has(s) || (i.add(s), a.push(s));
+		return a;
+	}
+	searchIndexedField(e, t, n, r, i, a, o = Infinity) {
+		let s = this.searchIndexedFieldIndices(t, n, r, i, a, o), c = [];
+		for (let t = 0; t < s.length; t++) {
+			let n = e[s[t]];
+			n && c.push(n);
+		}
+		return c;
+	}
+	searchIndexedFieldIndices(e, t, i, a, o, s = Infinity) {
+		let c = this.storage.ngramIndexes.get(e);
+		if (!c) return [];
+		let l = this.storage.normalizedFieldValues.get(e) ?? [];
+		return o == null ? n(c, i, l, t, {
+			restrictionLookup: a,
+			take: s
+		}) : r(c, i, l, t, {
+			candidateIndices: o,
+			restrictionLookup: a,
+			take: s
+		});
+	}
+	searchFieldLinear(e, t, n) {
+		let r = this.searchFieldLinearIndices(e, t, n), i = [];
+		for (let t = 0; t < r.length; t++) {
+			let n = e[r[t]];
+			n && i.push(n);
+		}
+		return i;
+	}
+	searchFieldLinearIndices(e, t, n, r) {
+		let i = this.fieldDescriptors.get(t);
+		if (!i) return [];
+		let { collectionKey: a, nestedKey: o } = i, s = [];
+		if (r) {
+			for (let t = 0; t < r.length; t++) {
+				let i = r[t], c = e[i][a];
+				if (!Array.isArray(c)) continue;
+				let l = !1;
+				for (let e = 0; e < c.length; e++) {
+					let t = c[e][o];
+					if (typeof t == "string" && t.toLowerCase().includes(n)) {
+						l = !0;
+						break;
+					}
+				}
+				l && s.push(i);
+			}
+			return s;
+		}
+		for (let t = 0; t < e.length; t++) {
+			let r = e[t][a];
+			if (!Array.isArray(r)) continue;
+			let i = !1;
+			for (let e = 0; e < r.length; e++) {
+				let t = r[e][o];
+				if (typeof t == "string" && t.toLowerCase().includes(n)) {
+					i = !0;
+					break;
+				}
+			}
+			i && s.push(t);
+		}
+		return s;
+	}
+	matchesAnyField(e, t) {
+		for (let n of this.registeredFields) {
+			let r = this.fieldDescriptors.get(n);
+			if (!r) continue;
+			let { collectionKey: i, nestedKey: a } = r, o = e[i];
+			if (Array.isArray(o)) for (let e = 0; e < o.length; e++) {
+				let n = o[e][a];
+				if (typeof n == "string" && n.toLowerCase().includes(t)) return !0;
+			}
+		}
+		return !1;
+	}
+	registerField(t) {
+		let n = e(t);
+		n && (this.registeredFields.add(t), this.fieldDescriptors.set(t, n));
+	}
+	getNormalizedValues(e) {
+		let t = this.storage.normalizedFieldValues.get(e);
+		if (t) return t;
+		let n = [];
+		return this.storage.normalizedFieldValues.set(e, n), n;
+	}
+	buildIndex(e, n) {
+		let r = this.fieldDescriptors.get(n);
+		if (!r) return;
+		let { collectionKey: i, nestedKey: a } = r, o = /* @__PURE__ */ new Map(), s = Array(e.length);
+		for (let n = 0, r = e.length; n < r; n++) {
+			let r = e[n][i];
+			if (!Array.isArray(r)) continue;
+			let c = [];
+			for (let e = 0; e < r.length; e++) {
+				let t = r[e][a];
+				typeof t == "string" && c.push(t.toLowerCase());
+			}
+			if (c.length === 0) continue;
+			let l = c.join("\n");
+			s[n] = l, t(o, l, n);
+		}
+		this.storage.ngramIndexes.set(n, o), this.storage.normalizedFieldValues.set(n, s);
+	}
+	addItemsToField(e, n, r) {
+		let i = this.fieldDescriptors.get(e), a = this.storage.ngramIndexes.get(e);
+		if (!i || !a) return;
+		let o = this.getNormalizedValues(e), { collectionKey: s, nestedKey: c } = i;
+		for (let e = 0; e < n.length; e++) {
+			let i = n[e][s];
+			if (!Array.isArray(i)) continue;
+			let l = [], u = r + e;
+			for (let e = 0; e < i.length; e++) {
+				let t = i[e][c];
+				typeof t == "string" && l.push(t.toLowerCase());
+			}
+			if (l.length === 0) continue;
+			let d = l.join("\n");
+			o[u] = d, t(a, d, u);
+		}
+	}
+	updateItemInField(e, n, r, a) {
+		let o = this.storage.ngramIndexes.get(e);
+		if (!o) return;
+		let s = this.getNormalizedValues(e), c = this.getNormalizedItemValue(e, r), l = this.getNormalizedItemValue(e, n);
+		if (c !== l) {
+			if (c && i(o, c, a), !l) {
+				delete s[a];
+				return;
+			}
+			s[a] = l, t(o, l, a);
+		}
+	}
+	removeItemFromField(e, t, n) {
+		let r = this.storage.ngramIndexes.get(e);
+		if (!r) return;
+		let a = this.getNormalizedValues(e), o = a[n] ?? this.getNormalizedItemValue(e, t);
+		o && i(r, o, n), delete a[n];
+	}
+	moveItemForField(e, n, r, a) {
+		let o = this.storage.ngramIndexes.get(e);
+		if (!o) return;
+		let s = this.getNormalizedValues(e), c = s[r] ?? this.getNormalizedItemValue(e, n);
+		if (!c) {
+			delete s[r];
+			return;
+		}
+		i(o, c, r), t(o, c, a), s[a] = c, delete s[r];
+	}
+	getNormalizedItemValue(e, t) {
+		let n = this.fieldDescriptors.get(e);
+		if (!n) return null;
+		let { collectionKey: r, nestedKey: i } = n, a = t[r];
+		if (!Array.isArray(a)) return null;
+		let o = [];
+		for (let e = 0; e < a.length; e++) {
+			let t = a[e][i];
+			typeof t == "string" && o.push(t.toLowerCase());
+		}
+		return o.length === 0 ? null : o.join("\n");
+	}
+};
+//#endregion
+export { a as SearchNestedCollection };

package/dist/search/ngram.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Minimum query length required to use the n-gram index directly.
+ * Queries shorter than this fall back to a linear scan.
+ */
+import type { IntersectPostingListsInCandidatesOptions, IntersectPostingListsOptions } from "./types";
+export type IntersectionPlan = {
+    smallestPostingList: ReadonlySet<number>;
+    matches: (candidateIndex: number) => boolean;
+};
+export declare function createIntersectionPlan(ngramMap: Map<string, Set<number>>, uniqueQueryGrams: ReadonlySet<string>, normalizedValues: (string | undefined)[], lowerQuery: string): IntersectionPlan | null;
+export declare const MINIMUM_SHORT_QUERY_INDEX_LENGTH = 2;
+export declare function buildIntersectionQueryGrams(lowerQuery: string): ReadonlySet<string>;
+export declare function indexLowerValue(ngramMap: Map<string, Set<number>>, lowerValue: string, itemIndex: number): void;
+export declare function estimateIntersectionCandidateCount(ngramMap: Map<string, Set<number>>, uniqueQueryGrams: ReadonlySet<string>): number;
+/**
+ * Intersects posting lists for the given query grams and confirms each
+ * candidate against the full normalizedValues string. Returns dataset indices
+ * of confirmed matches.
+ */
+export declare function intersectPostingLists(ngramMap: Map<string, Set<number>>, uniqueQueryGrams: ReadonlySet<string>, normalizedValues: (string | undefined)[], lowerQuery: string, options?: IntersectPostingListsOptions): number[];
+export declare function intersectPostingListsInCandidates(ngramMap: Map<string, Set<number>>, uniqueQueryGrams: ReadonlySet<string>, normalizedValues: (string | undefined)[], lowerQuery: string, options: IntersectPostingListsInCandidatesOptions): number[];
+export declare function removeLowerValue(ngramMap: Map<string, Set<number>>, lowerValue: string, itemIndex: number): void;
+//# sourceMappingURL=ngram.d.ts.map

package/dist/search/ngram.mjs

@@ -0,0 +1,96 @@
+//#region src/search/ngram.ts
+function e(e) {
+	for (let t = 0; t < e.length - 1; t++) {
+		let n = t;
+		for (let r = t + 1; r < e.length; r++) e[r].size < e[n].size && (n = r);
+		if (n !== t) {
+			let r = e[t];
+			e[t] = e[n], e[n] = r;
+		}
+	}
+}
+function t(t, n) {
+	let r = [];
+	for (let e of n) {
+		let n = t.get(e);
+		if (!n) return null;
+		r.push(n);
+	}
+	return e(r), r;
+}
+function n(e, t) {
+	for (let n = 0; n < t.length; n++) if (!t[n].has(e)) return !1;
+	return !0;
+}
+function r(e, r, i, a) {
+	let o = t(e, r);
+	if (o === null) return null;
+	let s = o[0];
+	return {
+		smallestPostingList: s,
+		matches: (e) => s.has(e) && n(e, o) && !!i[e]?.includes(a)
+	};
+}
+function i(e) {
+	let t = Math.min(3, Math.max(2, e.length)), n = e.length - t + 1, r = Array(n);
+	for (let i = 0; i < n; i++) r[i] = e.substring(i, i + t);
+	return r;
+}
+function a(e, t) {
+	let n = e.get(t);
+	if (n) return n;
+	let r = /* @__PURE__ */ new Set();
+	return e.set(t, r), r;
+}
+function o(e) {
+	let t = i(e);
+	if (t.length <= 12) return new Set(t);
+	let n = /* @__PURE__ */ new Set(), r = t.length - 1;
+	for (let e = 0; e <= 11; e++) {
+		let i = Math.round(e * r / 11);
+		n.add(t[i]);
+	}
+	return n;
+}
+function s(e, t, n) {
+	let r = Math.min(3, t.length);
+	for (let i = 2; i <= r; i++) {
+		let r = t.length - i;
+		for (let o = 0; o <= r; o++) a(e, t.substring(o, o + i)).add(n);
+	}
+}
+function c(e, t, n, i, a = {}) {
+	let { restrictionLookup: o = null, take: s = Infinity } = a, c = r(e, t, n, i);
+	if (c === null) return [];
+	let l = [];
+	for (let e of c.smallestPostingList) if (!(o !== null && !o[e]) && c.matches(e) && (l.push(e), l.length >= s)) break;
+	return l;
+}
+function l(e, t, n, i, a) {
+	let { candidateIndices: o, restrictionLookup: s = null, take: c = Infinity } = a;
+	if (o.length === 0) return [];
+	let l = r(e, t, n, i);
+	if (l === null) return [];
+	let u = [];
+	if (s === null || o.length <= l.smallestPostingList.size) {
+		for (let e = 0; e < o.length; e++) {
+			let t = o[e];
+			if (l.matches(t) && (u.push(t), u.length >= c)) break;
+		}
+		return u;
+	}
+	for (let e of l.smallestPostingList) if (s[e] && l.matches(e) && (u.push(e), u.length >= c)) break;
+	return u;
+}
+function u(e, t, n) {
+	let r = Math.min(3, t.length);
+	for (let i = 2; i <= r; i++) {
+		let r = t.length - i;
+		for (let a = 0; a <= r; a++) {
+			let r = t.substring(a, a + i), o = e.get(r);
+			o && (o.delete(n), o.size === 0 && e.delete(r));
+		}
+	}
+}
+//#endregion
+export { o as buildIntersectionQueryGrams, r as createIntersectionPlan, s as indexLowerValue, c as intersectPostingLists, l as intersectPostingListsInCandidates, u as removeLowerValue };

package/dist/search/text-search.d.ts

@@ -0,0 +1,146 @@
+/**
+ * TextSearchEngine class for performing fast substring search on string fields
+ * using n-gram indexing and intersection.
+ */
+import { State } from "../State";
+import { CollectionItem, type UpdateDescriptor } from "../types";
+import type { SearchQueryOptions, TextSearchEngineStats, TextSearchEngineOptions } from "./types";
+export declare class TextSearchEngine<T extends CollectionItem> {
+    private readonly state;
+    private readonly namespace;
+    private readonly nestedCollection;
+    private readonly minQueryLength;
+    private readonly silent;
+    private cachedIndexedFieldsList;
+    private cachedLinearSearchFieldsList;
+    private readonly emittedWarningKeys;
+    private readonly warnings;
+    /**
+     * Lightweight normalizedValues-only cache for fields without n-gram index.
+     * Built lazily on first full-dataset linear scan for non-indexed fields.
+     */
+    private normalizedValuesCache;
+    private combinedNormalizedValuesCache;
+    /**
+     * Creates a new TextSearchEngine with optional data and fields to index.
+     */
+    constructor(options?: TextSearchEngineOptions<T> & {
+        state?: State<T>;
+    });
+    private get dataset();
+    private get runtime();
+    private get flatIndexes();
+    private get indexedFields();
+    private shouldDeferMutationIndexUpdates;
+    private markDeferredMutationState;
+    private ensureConfiguredIndexesReady;
+    private rebuildConfiguredIndexes;
+    /**
+     * Builds an n-gram index for the given field from arbitrary data.
+     */
+    private buildIndexFromData;
+    search(query: string, options?: SearchQueryOptions): T[];
+    search(field: (keyof T & string) | (string & {}), query: string, options?: SearchQueryOptions): T[];
+    searchAll(query: string, options?: SearchQueryOptions): T[];
+    private normalizeQuery;
+    private normalizeSearchWindow;
+    private shouldTrackPreviousResult;
+    private hasReachedWindowLimit;
+    private sliceItems;
+    private collectItemsFromIndices;
+    private createLookup;
+    private getRestrictionLookup;
+    /**
+     * Returns the data source for search based on filterByPreviousResult setting.
+     * When the new query narrows the previous one, returns previousResult with indices.
+     * Otherwise resets previous state and returns the full dataset.
+     */
+    private getSearchSource;
+    /**
+     * Saves the search result for potential reuse on subsequent narrowing queries.
+     */
+    private saveSearchResult;
+    private persistSearchResult;
+    /**
+     * Resets the previous search result, forcing the next search to use the full dataset.
+     *
+     * @see {@link TextSearchEngineOptions.filterByPreviousResult}
+     */
+    resetSearchState(): this;
+    getWarnings(): string[];
+    getStats(): TextSearchEngineStats;
+    resetStats(): this;
+    private clearPreviousSearchState;
+    private recordIndexedQuery;
+    private recordFallbackQuery;
+    private shouldEmitFallbackWarning;
+    private warnAboutFallback;
+    private getResolvedFlatIndex;
+    private getQueryGrams;
+    /**
+     * Searches all indexed fields.
+     */
+    private searchAllFields;
+    private searchAllFieldsIndexed;
+    private searchAllFieldsIndexedInCandidates;
+    /**
+     * Searches a specific field.
+     */
+    private searchField;
+    /**
+     * Searches a field using prepared query grams, returning matched items and indices.
+     * Delegates to searchFieldWithPreparedQueryIndices for the core intersection logic.
+     */
+    private searchFieldWithPreparedQuery;
+    /**
+     * Core indexed search: returns dataset indices of matching items.
+     * If the cached index is stale (version mismatch), rebuilds it lazily.
+     */
+    private searchFieldWithPreparedQueryIndices;
+    /**
+     * Returns the cached list of indexed field names, rebuilding if stale.
+     */
+    private getIndexedFieldsList;
+    private materializeIndicesResult;
+    private getLinearSearchFields;
+    /**
+     * Eagerly builds normalizedValues for a field without n-gram overhead.
+     * Stores in a lightweight cache so subsequent linear scans reuse it.
+     */
+    private buildNormalizedValuesOnly;
+    private buildCombinedNormalizedValues;
+    private buildCombinedNormalizedValue;
+    private getCombinedNormalizedValues;
+    /**
+     * Updates only the entry at `index` in every cached normalizedValues array,
+     * avoiding a full O(n) rebuild when a single item is updated.
+     */
+    private invalidateNormalizedValuesCacheEntry;
+    /**
+     * Searches all fields linearly, using pre-normalized values when available.
+     * When `sourceIndices` is provided, iterates only those dataset positions
+     * (previousResult narrowing path) and resolves items via this.dataset[idx].
+     */
+    private searchLinearAllFields;
+    /**
+     * Searches a specific field linearly, using pre-normalized values when available.
+     * When `sourceIndices` is provided, resolves items via this.dataset[idx].
+     */
+    private searchLinearSingleField;
+    clearIndexes(): this;
+    getOriginData(): T[];
+    data(data: T[]): this;
+    add(items: T[]): this;
+    update(descriptor: UpdateDescriptor<T>): this;
+    private applyAddedItems;
+    clearData(): this;
+    private handleStateMutation;
+    private addItemsToField;
+    private applyUpdatedItem;
+    private applyRemovedItem;
+    private updateIndexedField;
+    private removeIndexedFieldValue;
+    private moveIndexedFieldValue;
+    private getNormalizedFieldValue;
+}
+//# sourceMappingURL=text-search.d.ts.map