@delali/narsil 0.3.1-next.79 → 0.3.1-next.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-1lEYL3WK.d.ts → capabilities-CAdkp2P-.d.ts} +2 -2
- package/dist/{chunk-XSZTF4ZQ.mjs → chunk-2453RJYK.mjs} +2 -2
- package/dist/{chunk-DFW3U5UW.mjs → chunk-4CWHWOBI.mjs} +24 -195
- package/dist/{chunk-R6WBLBTC.mjs → chunk-4D6OH4EF.mjs} +10 -46
- package/dist/{chunk-RTR44V2A.mjs → chunk-4VBG6PEH.mjs} +586 -4
- package/dist/{chunk-Y3L5MDRV.mjs → chunk-5XNIRLFC.mjs} +2 -2
- package/dist/{chunk-2E35UM3G.mjs → chunk-B3UOQJD3.mjs} +1 -1
- package/dist/{chunk-PJWG547Z.mjs → chunk-GNYXCLJS.mjs} +5 -5
- package/dist/{chunk-55WEJKWT.mjs → chunk-IH3GKNDQ.mjs} +197 -93
- package/dist/{chunk-AMGDG4ZJ.mjs → chunk-IN5CH4P2.mjs} +3 -3
- package/dist/{chunk-PXDQFYD5.mjs → chunk-KJ7LB3TR.mjs} +24 -140
- package/dist/chunk-LS5IRM23.mjs +140 -0
- package/dist/{chunk-4MQ5F4IL.mjs → chunk-OLXM65YH.mjs} +1 -1
- package/dist/{chunk-IBR6634K.mjs → chunk-R6A773WK.mjs} +2 -69
- package/dist/{chunk-4344I4MR.mjs → chunk-SREPGC6O.mjs} +336 -13
- package/dist/{chunk-I64LXKO3.mjs → chunk-TPJU7NDJ.mjs} +1 -0
- package/dist/chunk-WQBCBKY3.mjs +8 -0
- package/dist/{chunk-P2BEMMOD.mjs → chunk-XYDP7CCJ.mjs} +151 -204
- package/dist/{chunk-Z3BXCZXS.mjs → chunk-XZF4LZ3X.mjs} +5 -3
- package/dist/chunk-YTJCGKM4.mjs +56 -0
- package/dist/chunk-ZCKPRCYI.mjs +30 -0
- package/dist/{client-ZXcbSViH.d.ts → client-CKUtkLz3.d.ts} +2 -2
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +1 -0
- package/dist/{config-DHL2KsfD.d.ts → config-BuIkeM5C.d.ts} +39 -3
- package/dist/{config-DTe5EbWW.d.ts → config-CSDEPraV.d.ts} +1 -1
- package/dist/direct-executor-JVXDR5Y5.mjs +19 -0
- package/dist/distribution/coordinator/etcd.d.ts +1 -1
- package/dist/distribution/coordinator/etcd.mjs +3 -2
- package/dist/distribution/coordinator/in-memory.d.ts +1 -1
- package/dist/distribution/coordinator/in-memory.mjs +2 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +759 -65
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-BjPBzWkA.d.ts → engine-DMNZJAUq.d.ts} +1 -1
- package/dist/{types-ChrVy4lH.d.ts → index-DzONZUKE.d.ts} +79 -37
- package/dist/index.browser.mjs +138 -539
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +17 -15
- package/dist/invalidation/filesystem.mjs +2 -1
- package/dist/{memory-DW3OJ0im.d.ts → memory-BPKNx9Tl.d.ts} +1288 -1287
- package/dist/partial-words.mjs +6 -5
- package/dist/pattern.mjs +5 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +7 -7
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/server.d.ts +9 -9
- package/dist/server.mjs +21 -13
- package/dist/{tasks-BTbwBqar.d.ts → tasks-DGyz586z.d.ts} +1 -1
- package/dist/{thread-XBDVXKUE.mjs → thread-33M5AOEI.mjs} +13 -10
- package/dist/vector/search-worker.mjs +5 -4
- package/dist/workers/entry.mjs +5 -5
- package/package.json +13 -7
- package/dist/chunk-IOZPBNSG.mjs +0 -55
- package/dist/chunk-WFVOBZS4.mjs +0 -485
- package/dist/direct-executor-WB2HKU4N.mjs +0 -17
package/README.md
CHANGED
|
@@ -100,7 +100,7 @@ Every hit holds the document, its id, and its BM25 score. Beside the hits, `resu
|
|
|
100
100
|
|
|
101
101
|
## Features
|
|
102
102
|
|
|
103
|
-
**Search.** For [full-text search](https://github.com/assetcorp/narsil/blob/main/docs/full-text-search.md), the engine scores with BM25 and supports field boosting, fuzzy matching through bounded Levenshtein distance, search as you type through last-word prefix matching, and term-coverage and score thresholds. You can combine a query with [filters, facets, sorting, grouping, cursor pagination, and pinned results](https://github.com/assetcorp/narsil/blob/main/docs/filters-facets-and-pagination.md).
|
|
103
|
+
**Search.** For [full-text search](https://github.com/assetcorp/narsil/blob/main/docs/full-text-search.md), the engine scores with BM25 and supports field boosting, fuzzy matching through bounded Levenshtein distance, search as you type through last-word prefix matching, and term-coverage and score thresholds. You can combine a query with [filters, facets, sorting, grouping, cursor pagination, and pinned results](https://github.com/assetcorp/narsil/blob/main/docs/filters-facets-and-pagination.md). On a field that you declare for [pattern search](https://github.com/assetcorp/narsil/blob/main/docs/filters-facets-and-pagination.md#pattern-tests), a filter can find text anywhere inside a value with `contains`, test the whole value against a wildcard pattern, look for a match of a regular expression, and accept a typo or two in an `eq` test. With [partial-word search](https://github.com/assetcorp/narsil/blob/main/docs/full-text-search.md#matching-inside-words), a query word also matches inside longer words, so a search for `proof` finds `waterproof`.
|
|
104
104
|
|
|
105
105
|
**Vector and hybrid retrieval.** For [vector search](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md), the engine compares vectors by cosine similarity, dot product, or Euclidean distance. It scans a field exactly until the field holds 1,024 vectors, a count that you can change, after which it builds an HNSW graph with optimised scalar quantization on by default. Outside a browser, on an arm64 or x64 machine under macOS, Linux, or Windows, the engine searches that graph through a [native core in C](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md#native-search-core), which returns the same results as its WebAssembly search. For [hybrid search](https://github.com/assetcorp/narsil/blob/main/docs/hybrid-search.md), the engine fuses BM25 and vector rankings through reciprocal rank fusion or linear blending. [Embedding adapters](https://github.com/assetcorp/narsil/blob/main/docs/embedding-adapters.md) turn text into vectors on insert and query, through OpenAI, local Transformers.js models, or an adapter of your own.
|
|
106
106
|
|
|
@@ -110,7 +110,7 @@ Every hit holds the document, its id, and its BM25 score. Beside the hits, `resu
|
|
|
110
110
|
|
|
111
111
|
**Scale.** The engine routes each document to a [partition](https://github.com/assetcorp/narsil/blob/main/docs/partitions-and-workers.md) by a deterministic hash. When you call `rebalance()`, the engine reshapes the partitions online and buffers incoming writes in a write-ahead queue. Once an index holds 1,000 documents, the engine serves keyword queries from worker copies on half its worker threads, while the HTTP server receives requests on the threads that hold those copies.
|
|
112
112
|
|
|
113
|
-
**Operations.** The [HTTP server](https://github.com/assetcorp/narsil/blob/main/docs/http-server.md) subpath wraps an engine in a REST API with health probes, bulk NDJSON import, snapshot and restore endpoints, and task-based long operations. The [client](https://github.com/assetcorp/narsil/blob/main/docs/client.md) subpath calls every one of those routes from a browser or from Node under the engine's own method names. The [React](https://github.com/assetcorp/narsil/blob/main/docs/react.md) subpath
|
|
113
|
+
**Operations.** The [HTTP server](https://github.com/assetcorp/narsil/blob/main/docs/http-server.md) subpath wraps an engine in a REST API with health probes, bulk NDJSON import, snapshot and restore endpoints, and task-based long operations. The [client](https://github.com/assetcorp/narsil/blob/main/docs/client.md) subpath calls every one of those routes from a browser or from Node under the engine's own method names. The [React](https://github.com/assetcorp/narsil/blob/main/docs/react.md) subpath turns the read methods into hooks that call either a client or an engine in the same page. For observability, you have [events, plugins, and memory reporting](https://github.com/assetcorp/narsil/blob/main/docs/observability.md). [Language modules](https://github.com/assetcorp/narsil/blob/main/docs/language-support.md) cover 107 languages as separate entry points, including 20 African languages.
|
|
114
114
|
|
|
115
115
|
## Documentation
|
|
116
116
|
|
|
@@ -130,7 +130,7 @@ Every hit holds the document, its id, and its BM25 score. Beside the hits, `resu
|
|
|
130
130
|
| [HTTP server](https://github.com/assetcorp/narsil/blob/main/docs/http-server.md) | Wrapping an engine in a REST API, every route it serves, and long-running tasks |
|
|
131
131
|
| [Cluster mode](https://github.com/assetcorp/narsil/blob/main/docs/cluster.md) | Multi-node indexes: nodes and roles, replication, routed writes, distributed searches and reads, and the calls that fail in a cluster |
|
|
132
132
|
| [Client](https://github.com/assetcorp/narsil/blob/main/docs/client.md) | Reaching a server from a browser or Node, following a task, and the codes it raises |
|
|
133
|
-
| [React](https://github.com/assetcorp/narsil/blob/main/docs/react.md) | The hooks over the
|
|
133
|
+
| [React](https://github.com/assetcorp/narsil/blob/main/docs/react.md) | The hooks over a client or an engine in the page, one shared request per key, and loading a corpus from a component |
|
|
134
134
|
| [Observability](https://github.com/assetcorp/narsil/blob/main/docs/observability.md) | Plugin hooks, engine events, and memory reporting |
|
|
135
135
|
| [Errors](https://github.com/assetcorp/narsil/blob/main/docs/errors.md) | The codes you handle most often, and the three sets that `NarsilError.code` covers |
|
|
136
136
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
3
|
-
import { h as TaskStore } from './tasks-
|
|
2
|
+
import { C as ClusterNamespace } from './config-BuIkeM5C.js';
|
|
3
|
+
import { h as TaskStore } from './tasks-DGyz586z.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Context handed to {@link OnRequestHook} for every inbound request, captured
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-
|
|
2
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
1
|
+
import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-KJ7LB3TR.mjs';
|
|
2
|
+
import { ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
|
|
3
3
|
|
|
4
4
|
// src/narsil/internals.ts
|
|
5
5
|
var engineCores = /* @__PURE__ */ new WeakMap();
|
|
@@ -1,16 +1,18 @@
|
|
|
1
|
-
import { requirePatternSearchFor, patternIndexOf, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, computeFacets, cloneProjected, projectionKeepsField, setNestedValue, fnv1a, requirePatternSearch, PATTERN_HOST, validatedFilterOf, everyValueFacetConfig, mergeFacets, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, undeclaredFilterField, coveringRanges } from './chunk-
|
|
2
|
-
import { getLanguage, detectRuntime } from './chunk-
|
|
3
|
-
import { MAX_DOC_ID_LENGTH
|
|
4
|
-
import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, isTombstoned, addToOrdinalFilter, ordinalFilterValues, nativeScoresOf, toScore, removeFromOrdinalFilter, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, ordinalFilterHas, STORE_CALIBRATION_GENERATION, entryForOrd, toDistance, sharedMemoryAvailable, requireWholeNeighbourLists, topLayerInParts, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-
|
|
5
|
-
import { noteFallback } from './chunk-
|
|
6
|
-
import { GrowableUint32, bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto,
|
|
7
|
-
import {
|
|
8
|
-
import { tokenizeOptions, getNestedValue, visitTokens, textFieldsChanged, getFlatSchema, tokenize, getOrCreateFieldNameIndex, getFieldValueByInternalId } from './chunk-
|
|
9
|
-
import {
|
|
1
|
+
import { createGrowingPartialWords, requirePatternSearchFor, requirePartialWordSearchFor, buildPartialWordTableFrom, patternIndexOf, encodePartialSpellingsOf, countPartialSpelling, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, expandPartialWords, computeFacets, cloneProjected, projectionKeepsField, setNestedValue, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, PATTERN_HOST, validatedFilterOf, everyValueFacetConfig, mergeFacets, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, undeclaredFilterField, coveringRanges } from './chunk-XYDP7CCJ.mjs';
|
|
2
|
+
import { getLanguage, detectRuntime } from './chunk-OLXM65YH.mjs';
|
|
3
|
+
import { MAX_DOC_ID_LENGTH } from './chunk-ZCKPRCYI.mjs';
|
|
4
|
+
import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, isTombstoned, addToOrdinalFilter, ordinalFilterValues, nativeScoresOf, toScore, removeFromOrdinalFilter, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, ordinalFilterHas, STORE_CALIBRATION_GENERATION, entryForOrd, toDistance, sharedMemoryAvailable, requireWholeNeighbourLists, topLayerInParts, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-IN5CH4P2.mjs';
|
|
5
|
+
import { noteFallback } from './chunk-B3UOQJD3.mjs';
|
|
6
|
+
import { GrowableUint32, bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet, bitsetIsEmpty } from './chunk-SREPGC6O.mjs';
|
|
7
|
+
import { GEO_UNIT_METRES } from './chunk-WQBCBKY3.mjs';
|
|
8
|
+
import { tokenizeOptions, getNestedValue, visitTokens, expandTermPrefix, suggestDisplayTerms, textFieldsChanged, sameFields, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, getFlatSchema, tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, getOrCreateFieldNameIndex, getFieldValueByInternalId, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, partitionDocumentFrequency } from './chunk-4VBG6PEH.mjs';
|
|
9
|
+
import { schemaFieldsOf, partialFieldsOf, validateDocument, validateDocumentStrict, encodeDocument, extractVectorFieldsFromSchema, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, VINCENTY_MAX_ITERATIONS, DEG_TO_RAD } from './chunk-KJ7LB3TR.mjs';
|
|
10
10
|
import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-KEHK2O6P.mjs';
|
|
11
11
|
import { fuzzyPrefixOf, boundedLevenshtein } from './chunk-GES7U2PM.mjs';
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
12
|
+
import { isWordIndexedBase, parsedTypeOf, isListBase } from './chunk-LS5IRM23.mjs';
|
|
13
|
+
import { toReducedSortValue, readSortField, defaultSortMode, compareComparableValues, truncateSortString, SORT_VALUE_MAX_CODE_POINTS } from './chunk-R6A773WK.mjs';
|
|
14
|
+
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
15
|
+
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
14
16
|
import { totalmem } from 'os';
|
|
15
17
|
import { getHeapStatistics } from 'v8';
|
|
16
18
|
import { decode } from '@msgpack/msgpack';
|
|
@@ -1644,7 +1646,7 @@ function countSurface(counts, surface, token) {
|
|
|
1644
1646
|
}
|
|
1645
1647
|
function countOccurrence(counts, spellings, surface, token) {
|
|
1646
1648
|
countSurface(counts, surface, token);
|
|
1647
|
-
if (spellings !== null)
|
|
1649
|
+
if (spellings !== null) countPartialSpelling(spellings, token, surface ?? token, 1);
|
|
1648
1650
|
}
|
|
1649
1651
|
function addFieldOccurrences(occurrencesByToken, fieldTokens) {
|
|
1650
1652
|
const fieldCounts = /* @__PURE__ */ new Map();
|
|
@@ -1966,36 +1968,6 @@ function updateFieldIndexOnly(state, docId, oldFields, newDoc, fields) {
|
|
|
1966
1968
|
}
|
|
1967
1969
|
}
|
|
1968
1970
|
|
|
1969
|
-
// src/filters/pattern-tests.ts
|
|
1970
|
-
function excluding(context, fieldPath, excluded) {
|
|
1971
|
-
const universe = bitsetAnd(context.presentBitset(fieldPath), context.allDocIdsBitset);
|
|
1972
|
-
return bitsetAnd(universe, bitsetNot(excluded, context.capacity));
|
|
1973
|
-
}
|
|
1974
|
-
function textsOf(operand) {
|
|
1975
|
-
const texts = /* @__PURE__ */ new Set();
|
|
1976
|
-
if (!Array.isArray(operand)) return texts;
|
|
1977
|
-
for (const text of operand) if (typeof text === "string") texts.add(text);
|
|
1978
|
-
return texts;
|
|
1979
|
-
}
|
|
1980
|
-
function evaluatePatternTests(fieldPath, filter, context, bitsets) {
|
|
1981
|
-
const caseFold = filter.caseFold === true;
|
|
1982
|
-
const matching = (operator, texts, tolerance = 0) => {
|
|
1983
|
-
const into = createBitSet(context.capacity);
|
|
1984
|
-
for (const text of texts) context.matchPattern(fieldPath, operator, text, caseFold, tolerance, into);
|
|
1985
|
-
return into;
|
|
1986
|
-
};
|
|
1987
|
-
const eqTolerance = isPatternTolerance(filter.tolerance) ? filter.tolerance : 0;
|
|
1988
|
-
if (typeof filter.eq === "string") bitsets.push(matching("eq", [filter.eq], eqTolerance));
|
|
1989
|
-
if (typeof filter.ne === "string") bitsets.push(excluding(context, fieldPath, matching("eq", [filter.ne])));
|
|
1990
|
-
if (filter.in !== void 0) bitsets.push(matching("eq", textsOf(filter.in)));
|
|
1991
|
-
if (filter.nin !== void 0) bitsets.push(excluding(context, fieldPath, matching("eq", textsOf(filter.nin))));
|
|
1992
|
-
if (typeof filter.startsWith === "string") bitsets.push(matching("startsWith", [filter.startsWith]));
|
|
1993
|
-
if (typeof filter.endsWith === "string") bitsets.push(matching("endsWith", [filter.endsWith]));
|
|
1994
|
-
if (typeof filter.contains === "string") bitsets.push(matching("contains", [filter.contains]));
|
|
1995
|
-
if (typeof filter.wildcard === "string") bitsets.push(matching("wildcard", [filter.wildcard]));
|
|
1996
|
-
if (typeof filter.regex === "string") bitsets.push(matching("regex", [filter.regex]));
|
|
1997
|
-
}
|
|
1998
|
-
|
|
1999
1971
|
// src/filters/field-tests.ts
|
|
2000
1972
|
var COMPARISON_OPERATORS = ["eq", "ne", "gt", "gte", "lt", "lte", "between"];
|
|
2001
1973
|
var TEXT_RANGE_OPERATORS = ["gt", "gte", "lt", "lte"];
|
|
@@ -2175,7 +2147,7 @@ function narrowByFieldTests(fieldPath, f, parsed, context, narrow) {
|
|
|
2175
2147
|
narrowByTextRange(fieldPath, f, context, narrow);
|
|
2176
2148
|
if (parsed.pattern) {
|
|
2177
2149
|
const patternBits = [];
|
|
2178
|
-
|
|
2150
|
+
requirePatternSearch(fieldPath).evaluateTests(fieldPath, f, context, patternBits);
|
|
2179
2151
|
for (const matched of patternBits) narrow(matched);
|
|
2180
2152
|
} else {
|
|
2181
2153
|
narrowByValue(fieldPath, f, parsed, context, bits, narrow);
|
|
@@ -3422,68 +3394,6 @@ function mergeSegmentState(target, source) {
|
|
|
3422
3394
|
target.scoreBuffer = null;
|
|
3423
3395
|
}
|
|
3424
3396
|
|
|
3425
|
-
// src/core/partition/partial-words/platform.ts
|
|
3426
|
-
function platformPartialWordSearch() {
|
|
3427
|
-
return partialWordSearchModule;
|
|
3428
|
-
}
|
|
3429
|
-
|
|
3430
|
-
// src/core/partition/partial-words/registry.ts
|
|
3431
|
-
var registered = null;
|
|
3432
|
-
function isPartialWordSearchModule(value) {
|
|
3433
|
-
return value !== null && typeof value === "object" && value.name === "partialWords" && "createGrowingSlot" in value && typeof value.createGrowingSlot === "function" && "createFrozenSlot" in value && typeof value.createFrozenSlot === "function" && "buildTable" in value && typeof value.buildTable === "function" && "expand" in value && typeof value.expand === "function";
|
|
3434
|
-
}
|
|
3435
|
-
function registerPartialWordSearch(module) {
|
|
3436
|
-
if (!isPartialWordSearchModule(module)) {
|
|
3437
|
-
throw new NarsilError(
|
|
3438
|
-
ErrorCodes.CONFIG_INVALID,
|
|
3439
|
-
'Pass registerPartialWordSearch the partialWordSearch object that "@delali/narsil/partial-words" exports'
|
|
3440
|
-
);
|
|
3441
|
-
}
|
|
3442
|
-
registered = module;
|
|
3443
|
-
}
|
|
3444
|
-
function availablePartialWordSearch() {
|
|
3445
|
-
return registered ?? platformPartialWordSearch();
|
|
3446
|
-
}
|
|
3447
|
-
function requirePartialWordSearchFor(schema) {
|
|
3448
|
-
const fields = partialFieldsOf(schema);
|
|
3449
|
-
if (fields.length === 0 || availablePartialWordSearch() !== null) return;
|
|
3450
|
-
throw new NarsilError(
|
|
3451
|
-
ErrorCodes.CONFIG_INVALID,
|
|
3452
|
-
`The type of field "${fields[0]}" includes "partial", but this bundle contains no partial-word search code. Add import { partialWordSearch } from '@delali/narsil/partial-words', then call registerPartialWordSearch(partialWordSearch) before you create or load the index`,
|
|
3453
|
-
{ field: fields[0] }
|
|
3454
|
-
);
|
|
3455
|
-
}
|
|
3456
|
-
var EMPTY_GROWING_SLOT = {
|
|
3457
|
-
readerFor: () => null,
|
|
3458
|
-
spellings: () => null,
|
|
3459
|
-
bytes: () => 0,
|
|
3460
|
-
afterDocument: () => void 0,
|
|
3461
|
-
afterMerge: () => void 0,
|
|
3462
|
-
forgetSpellings: () => void 0,
|
|
3463
|
-
clear: () => void 0
|
|
3464
|
-
};
|
|
3465
|
-
var EMPTY_FROZEN_SLOT = {
|
|
3466
|
-
readerFor: () => null,
|
|
3467
|
-
spellings: () => null,
|
|
3468
|
-
bytes: () => 0,
|
|
3469
|
-
afterRemoval: () => void 0,
|
|
3470
|
-
tableOfEveryDocument: () => null,
|
|
3471
|
-
countKeptSpellings: () => void 0
|
|
3472
|
-
};
|
|
3473
|
-
function createGrowingPartialWords(view) {
|
|
3474
|
-
return availablePartialWordSearch()?.createGrowingSlot(view) ?? EMPTY_GROWING_SLOT;
|
|
3475
|
-
}
|
|
3476
|
-
function createFrozenPartialWords(initial, tokenTable, documents) {
|
|
3477
|
-
return availablePartialWordSearch()?.createFrozenSlot(initial, tokenTable, documents) ?? EMPTY_FROZEN_SLOT;
|
|
3478
|
-
}
|
|
3479
|
-
function buildPartialWordTableFrom(source, tokenTable) {
|
|
3480
|
-
return availablePartialWordSearch()?.buildTable(source, tokenTable) ?? null;
|
|
3481
|
-
}
|
|
3482
|
-
function expandPartialWords(subs, analysis, requests) {
|
|
3483
|
-
const module = availablePartialWordSearch();
|
|
3484
|
-
return module === null ? requests.map(() => []) : module.expand(subs, analysis, requests);
|
|
3485
|
-
}
|
|
3486
|
-
|
|
3487
3397
|
// src/core/partition/rebuild.ts
|
|
3488
3398
|
var EMPTY_STATS = {
|
|
3489
3399
|
totalDocuments: 0,
|
|
@@ -4585,99 +4495,18 @@ function mergeSegmentPayload(target, payload, documents) {
|
|
|
4585
4495
|
}
|
|
4586
4496
|
|
|
4587
4497
|
// src/core/partition/pattern-snapshot.ts
|
|
4588
|
-
function loadFailed(fieldPath, problem) {
|
|
4589
|
-
return new NarsilError(
|
|
4590
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
4591
|
-
`The pattern index of field "${fieldPath}" in this partition is corrupt: ${problem}`,
|
|
4592
|
-
{ field: fieldPath }
|
|
4593
|
-
);
|
|
4594
|
-
}
|
|
4595
|
-
function serializePatternIndex(state, fieldPath, index) {
|
|
4596
|
-
const resolver = state.docStore.resolver();
|
|
4597
|
-
const arrays = index.toArrays(fieldPath, null);
|
|
4598
|
-
const positionOf = /* @__PURE__ */ new Map();
|
|
4599
|
-
const docIds = [];
|
|
4600
|
-
for (const ordinal of arrays.docIds) {
|
|
4601
|
-
const externalId = resolver.toExternal(ordinal);
|
|
4602
|
-
if (externalId === void 0) continue;
|
|
4603
|
-
positionOf.set(ordinal, docIds.length);
|
|
4604
|
-
docIds.push(externalId);
|
|
4605
|
-
}
|
|
4606
|
-
const runs = /* @__PURE__ */ Object.create(null);
|
|
4607
|
-
for (let run = 0; run + 1 < arrays.offsets.length; run++) {
|
|
4608
|
-
const positions = [];
|
|
4609
|
-
for (let at = arrays.offsets[run]; at < arrays.offsets[run + 1]; at++) {
|
|
4610
|
-
const position = positionOf.get(arrays.postings[at]);
|
|
4611
|
-
if (position !== void 0) positions.push(position);
|
|
4612
|
-
}
|
|
4613
|
-
if (positions.length === 0) continue;
|
|
4614
|
-
const base = run * PATTERN_RUN_CODE_POINTS;
|
|
4615
|
-
runs[String.fromCodePoint(arrays.runs[base], arrays.runs[base + 1], arrays.runs[base + 2])] = positions;
|
|
4616
|
-
}
|
|
4617
|
-
return { docIds, runs };
|
|
4618
|
-
}
|
|
4619
4498
|
function serializePatternIndexes(state, schema) {
|
|
4620
4499
|
const patternFields = schemaFieldsOf(schema).filter((field) => field.pattern);
|
|
4621
4500
|
if (patternFields.length === 0) return void 0;
|
|
4501
|
+
const resolver = state.docStore.resolver();
|
|
4502
|
+
const toExternal = (ordinal) => resolver.toExternal(ordinal);
|
|
4622
4503
|
const serialized = /* @__PURE__ */ Object.create(null);
|
|
4623
4504
|
for (const { path: fieldPath } of patternFields) {
|
|
4624
4505
|
const index = state.patternIndexes.get(fieldPath);
|
|
4625
|
-
serialized[fieldPath] = index === void 0 ? { docIds: [], runs: {} } :
|
|
4506
|
+
serialized[fieldPath] = index === void 0 ? { docIds: [], runs: {} } : requirePatternSearch(fieldPath).serializeIndex(fieldPath, index, toExternal);
|
|
4626
4507
|
}
|
|
4627
4508
|
return serialized;
|
|
4628
4509
|
}
|
|
4629
|
-
function appendRunCodePoints(fieldPath, key, out) {
|
|
4630
|
-
const start = out.length;
|
|
4631
|
-
for (const character of key) out.push(character.codePointAt(0) ?? 0);
|
|
4632
|
-
const count = out.length - start;
|
|
4633
|
-
if (count !== PATTERN_RUN_CODE_POINTS) {
|
|
4634
|
-
throw loadFailed(
|
|
4635
|
-
fieldPath,
|
|
4636
|
-
`the key "${key}" holds ${count} code points, while each key must hold ${PATTERN_RUN_CODE_POINTS}`
|
|
4637
|
-
);
|
|
4638
|
-
}
|
|
4639
|
-
}
|
|
4640
|
-
function requirePositions(fieldPath, key, list, documentCount) {
|
|
4641
|
-
if (!Array.isArray(list)) throw loadFailed(fieldPath, `the run "${key}" must map to a list of positions`);
|
|
4642
|
-
let previous = -1;
|
|
4643
|
-
for (const position of list) {
|
|
4644
|
-
if (!Number.isInteger(position) || position < 0 || position >= documentCount) {
|
|
4645
|
-
throw loadFailed(fieldPath, `the run "${key}" names position ${String(position)} outside its document list`);
|
|
4646
|
-
}
|
|
4647
|
-
if (position <= previous) {
|
|
4648
|
-
throw loadFailed(fieldPath, `the positions of the run "${key}" are out of ascending order`);
|
|
4649
|
-
}
|
|
4650
|
-
previous = position;
|
|
4651
|
-
}
|
|
4652
|
-
return list;
|
|
4653
|
-
}
|
|
4654
|
-
function requireDistinctDocuments(fieldPath, docIds) {
|
|
4655
|
-
const seen = /* @__PURE__ */ new Set();
|
|
4656
|
-
for (const docId of docIds) {
|
|
4657
|
-
if (seen.has(docId)) throw loadFailed(fieldPath, `the document list names "${docId}" twice`);
|
|
4658
|
-
seen.add(docId);
|
|
4659
|
-
}
|
|
4660
|
-
}
|
|
4661
|
-
function decodePatternEntry(fieldPath, entry) {
|
|
4662
|
-
requireDistinctDocuments(fieldPath, entry.docIds);
|
|
4663
|
-
const runs = [];
|
|
4664
|
-
const offsets = [0];
|
|
4665
|
-
const postings = [];
|
|
4666
|
-
for (const [key, list] of Object.entries(entry.runs)) {
|
|
4667
|
-
appendRunCodePoints(fieldPath, key, runs);
|
|
4668
|
-
for (const position of requirePositions(fieldPath, key, list, entry.docIds.length)) postings.push(position);
|
|
4669
|
-
offsets.push(postings.length);
|
|
4670
|
-
}
|
|
4671
|
-
const positions = new Uint32Array(entry.docIds.length);
|
|
4672
|
-
for (let position = 0; position < positions.length; position++) positions[position] = position;
|
|
4673
|
-
return {
|
|
4674
|
-
fieldPath,
|
|
4675
|
-
docIds: positions,
|
|
4676
|
-
runs: Uint32Array.from(runs),
|
|
4677
|
-
offsets: Uint32Array.from(offsets),
|
|
4678
|
-
postings: Uint32Array.from(postings)
|
|
4679
|
-
};
|
|
4680
|
-
}
|
|
4681
4510
|
function rebuildPatternIndex(state, fieldPath) {
|
|
4682
4511
|
for (const [docId, stored] of state.docStore.all()) {
|
|
4683
4512
|
const value = getNestedValue(stored.fields, fieldPath);
|
|
@@ -4695,7 +4524,7 @@ function loadPatternIndexes(state, serialized, schema) {
|
|
|
4695
4524
|
rebuildPatternIndex(state, fieldPath);
|
|
4696
4525
|
continue;
|
|
4697
4526
|
}
|
|
4698
|
-
const decoded =
|
|
4527
|
+
const decoded = requirePatternSearch(fieldPath).decodeIndex(fieldPath, entry);
|
|
4699
4528
|
const ordinals = entry.docIds.map((docId) => resolver.toInternal(docId) ?? -1);
|
|
4700
4529
|
patternIndexOf(state.patternIndexes, fieldPath).addArrays(decoded, (position) => ordinals[position]);
|
|
4701
4530
|
}
|
|
@@ -9352,7 +9181,7 @@ function sharePartialWordTable(table) {
|
|
|
9352
9181
|
function sharedPartialWordsOf(tokenData, spellings, documentData, analysis) {
|
|
9353
9182
|
let source = spellings;
|
|
9354
9183
|
if (source === null && analysis !== null && analysis.fields.length > 0) {
|
|
9355
|
-
source =
|
|
9184
|
+
source = partialSpellingsOfDocuments(storedDocumentsOf(documentData), analysis);
|
|
9356
9185
|
}
|
|
9357
9186
|
if (source === null || source.fields.length === 0) return null;
|
|
9358
9187
|
const table = buildPartialWordTableFrom(source, wrapFrozenTokenTable(tokenData));
|
|
@@ -10219,7 +10048,7 @@ function mergePartialWords(inputs, mergedTokenCount, analysis) {
|
|
|
10219
10048
|
}
|
|
10220
10049
|
const counts = /* @__PURE__ */ new Map();
|
|
10221
10050
|
for (const input of inputs) input.segment.partialWords.countKeptSpellings(counts, analysis, input.remap);
|
|
10222
|
-
return { spellings:
|
|
10051
|
+
return { spellings: encodePartialSpellingsOf(analysis.fields, counts), reused: null };
|
|
10223
10052
|
}
|
|
10224
10053
|
function mergeFrozenSegments(segments, analysis) {
|
|
10225
10054
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
@@ -10535,7 +10364,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
10535
10364
|
const addToken = (token, position, surface) => {
|
|
10536
10365
|
postings.addToken(token, positionOffset + position);
|
|
10537
10366
|
surfaces.count(surface, token);
|
|
10538
|
-
if (fieldSpellings !== null)
|
|
10367
|
+
if (fieldSpellings !== null) countPartialSpelling(fieldSpellings, token, surface ?? token, 1);
|
|
10539
10368
|
};
|
|
10540
10369
|
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
10541
10370
|
const document = documents[ordinal].document;
|
|
@@ -10569,7 +10398,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
10569
10398
|
fieldLengthColumns: lengths.columns,
|
|
10570
10399
|
totalFieldLengths: lengths.totalsByName(),
|
|
10571
10400
|
surfaceForms: surfaces.encode(),
|
|
10572
|
-
partialSpellings: partialFields.length === 0 ? null :
|
|
10401
|
+
partialSpellings: partialFields.length === 0 ? null : encodePartialSpellingsOf(partialFields, spellings),
|
|
10573
10402
|
...encodeFieldIndexes(filters, identity)
|
|
10574
10403
|
};
|
|
10575
10404
|
}
|
|
@@ -11113,4 +10942,4 @@ function createDirectExecutor(options) {
|
|
|
11113
10942
|
};
|
|
11114
10943
|
}
|
|
11115
10944
|
|
|
11116
|
-
export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES, NO_LIVE_VECTOR_AT_POSITION, SHUTDOWN_TIMEOUT_MS, VALID_HNSW_METRICS, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, assembleGraphParts, buildSegmentPayload, bytesToVectors, countDead, createDirectExecutor, createFrozenSegment, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, deadBitsWhere, deadBytesFor, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, encodeSegmentState, freezeSegmentShared, generateId, getStopWords, getTokenizer, graphPartsOfEarlierPayload, hasStopWords, hasTokenizer, isCompositePartition, isDead, mergeFrozenSegments, mergeSegmentContents, missingWorkerEntry, nodeThreadId, partitionStateOf, readHeapStatistics,
|
|
10945
|
+
export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES, NO_LIVE_VECTOR_AT_POSITION, SHUTDOWN_TIMEOUT_MS, VALID_HNSW_METRICS, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, assembleGraphParts, buildSegmentPayload, bytesToVectors, countDead, createDirectExecutor, createFrozenSegment, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, deadBitsWhere, deadBytesFor, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, encodeSegmentState, freezeSegmentShared, generateId, getStopWords, getTokenizer, graphPartsOfEarlierPayload, hasStopWords, hasTokenizer, isCompositePartition, isDead, mergeFrozenSegments, mergeSegmentContents, missingWorkerEntry, nodeThreadId, partitionStateOf, readHeapStatistics, registerStopWords, registerTokenizer, resolveIndexAnalysis, resolvePartitionInsertOptions, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, segmentArraysOf, selectionKeeps, shareSegmentArrays, spawnNodeWorker, splitWorkerBudget, survivorDocumentsOf, vectorsPerFile, vectorsToBytes, workerResourceLimits };
|
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, fnv1aBytes, validatedFilterOf, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, undeclaredFieldOnStrictIndex } from './chunk-
|
|
2
|
-
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT,
|
|
1
|
+
import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, fnv1aBytes, validatedFilterOf, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, partialWordsToHighlight, partialWordHighlighter, undeclaredFieldOnStrictIndex } from './chunk-XYDP7CCJ.mjs';
|
|
2
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-ZCKPRCYI.mjs';
|
|
3
3
|
import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
|
|
4
|
-
import { tokenize } from './chunk-
|
|
5
|
-
import { partialFieldsOf,
|
|
4
|
+
import { tokenize } from './chunk-4VBG6PEH.mjs';
|
|
5
|
+
import { partialFieldsOf, DEFAULT_MAX_PER_GROUP, flattenSchema, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-KJ7LB3TR.mjs';
|
|
6
6
|
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
7
|
+
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isWordIndexedFieldType, isGeopointOrVectorType } from './chunk-LS5IRM23.mjs';
|
|
8
|
+
import { compareComparableKeys } from './chunk-R6A773WK.mjs';
|
|
9
|
+
import { holdsUnpairedSurrogate, compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
10
|
+
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
9
11
|
|
|
10
12
|
// src/search/cursor-binding.ts
|
|
11
13
|
var ABSENT_TAG = 0;
|
|
@@ -855,31 +857,6 @@ function applySnippetTruncation(taggedText, originalText, positions, maxLen, pre
|
|
|
855
857
|
if (needsTrailingEllipsis) snippet = `${snippet}...`;
|
|
856
858
|
return { snippet, visiblePositions };
|
|
857
859
|
}
|
|
858
|
-
function containsPartialWord(spelling, words) {
|
|
859
|
-
for (const word of words) {
|
|
860
|
-
if (spelling.length > word.length && spelling.includes(word)) return true;
|
|
861
|
-
}
|
|
862
|
-
return false;
|
|
863
|
-
}
|
|
864
|
-
function partialWordMatcher(text, language, partial) {
|
|
865
|
-
const indexed = tokenize(text, language, {
|
|
866
|
-
stem: true,
|
|
867
|
-
removeStopWords: true,
|
|
868
|
-
collectSurfaces: true,
|
|
869
|
-
customTokenizer: partial.customTokenizer,
|
|
870
|
-
stopWordOverride: partial.stopWordOverride
|
|
871
|
-
});
|
|
872
|
-
const positions = /* @__PURE__ */ new Set();
|
|
873
|
-
const spellings = /* @__PURE__ */ new Set();
|
|
874
|
-
for (let at = 0; at < indexed.tokens.length; at++) {
|
|
875
|
-
const spelling = indexed.surfaces?.[at] ?? indexed.tokens[at].token;
|
|
876
|
-
if (!containsPartialWord(spelling, partial.words)) continue;
|
|
877
|
-
positions.add(indexed.tokens[at].position);
|
|
878
|
-
spellings.add(spelling);
|
|
879
|
-
}
|
|
880
|
-
if (partial.customTokenizer !== void 0) return (_position, fieldToken) => spellings.has(fieldToken);
|
|
881
|
-
return (position) => positions.has(position);
|
|
882
|
-
}
|
|
883
860
|
function highlightField(text, queryTokens, language, options) {
|
|
884
861
|
const preTag = options?.preTag ?? "<mark>";
|
|
885
862
|
const postTag = options?.postTag ?? "</mark>";
|
|
@@ -916,7 +893,7 @@ function highlightField(text, queryTokens, language, options) {
|
|
|
916
893
|
matchByFieldToken.set(fieldToken, matches);
|
|
917
894
|
return matches;
|
|
918
895
|
};
|
|
919
|
-
const matchesInside = options?.partialWords === void 0 ? null :
|
|
896
|
+
const matchesInside = options?.partialWords === void 0 ? null : partialWordHighlighter(text, language, options.partialWords);
|
|
920
897
|
for (let i = 0; i < fieldResult.tokens.length; i++) {
|
|
921
898
|
if (i >= charOffsets.length) break;
|
|
922
899
|
const fieldToken = fieldResult.tokens[i].token;
|
|
@@ -966,20 +943,7 @@ function partialWordTagging(params, schema, queryTokens, written, analysis) {
|
|
|
966
943
|
if (params.partialWords !== true || params.exact === true) return null;
|
|
967
944
|
const fields = partialFieldsOf(schema).filter((field) => params.fields === void 0 || params.fields.includes(field));
|
|
968
945
|
if (fields.length === 0) return null;
|
|
969
|
-
const
|
|
970
|
-
for (const word of written) {
|
|
971
|
-
if (!writtenAt.has(word.position)) writtenAt.set(word.position, word.token);
|
|
972
|
-
}
|
|
973
|
-
const seenTokens = /* @__PURE__ */ new Set();
|
|
974
|
-
const words = [];
|
|
975
|
-
for (const queryToken of queryTokens) {
|
|
976
|
-
if (seenTokens.has(queryToken.token)) continue;
|
|
977
|
-
seenTokens.add(queryToken.token);
|
|
978
|
-
const word = writtenAt.get(queryToken.position);
|
|
979
|
-
if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
|
|
980
|
-
words.push(word);
|
|
981
|
-
if (words.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
|
|
982
|
-
}
|
|
946
|
+
const words = partialWordsToHighlight(queryTokens, written);
|
|
983
947
|
if (words.length === 0) return null;
|
|
984
948
|
return {
|
|
985
949
|
fields: new Set(fields),
|