@delali/narsil 0.3.0 → 0.3.1-next.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
- package/dist/browser/chunk-BDOSMEEP.mjs +493 -0
- package/dist/browser/chunk-EJDUFWSO.mjs +602 -0
- package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
- package/dist/browser/chunk-JOVZ6HMT.mjs +4358 -0
- package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
- package/dist/browser/chunk-KPJMDPHG.mjs +24558 -0
- package/dist/browser/chunk-PGIM3HIM.mjs +2743 -0
- package/dist/browser/chunk-RHG4O77A.mjs +206 -0
- package/dist/browser/chunk-RIJSLY2I.mjs +360 -0
- package/dist/browser/durability-wiring-SRUYT3VV.mjs +3875 -0
- package/dist/browser/module-CBB2ETYP.mjs +3111 -0
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-EEZQO6RF.mjs +35 -0
- package/dist/browser/runtime-7ZAHCHWR.mjs +557 -0
- package/dist/browser/snapshot-OAAA5UJ7.mjs +1210 -0
- package/dist/browser/write-7D6VIK6O.mjs +208 -0
- package/dist/{capabilities-DSsOGz6F.d.ts → capabilities-DiGlB6Sr.d.ts} +68 -7
- package/dist/chunk-2RIRGOUL.mjs +113 -0
- package/dist/chunk-37CD3OKS.mjs +87 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-46D6JQIG.mjs +426 -0
- package/dist/{chunk-VMRR3DQU.mjs → chunk-5JNA5SKS.mjs} +546 -35
- package/dist/chunk-747OHOUA.mjs +11 -0
- package/dist/chunk-75XIEF6H.mjs +126 -0
- package/dist/chunk-7X5JST5L.mjs +114 -0
- package/dist/chunk-BOFBLJFZ.mjs +209 -0
- package/dist/{chunk-USKFU7J5.mjs → chunk-BWJVNZPR.mjs} +8 -1
- package/dist/chunk-C7SLA6CM.mjs +19 -0
- package/dist/{chunk-POWOTUKJ.mjs → chunk-CQUMNIIG.mjs} +3 -6
- package/dist/chunk-CVXLLEQ3.mjs +1011 -0
- package/dist/chunk-DJHHF2L5.mjs +17 -0
- package/dist/chunk-E74YLAIG.mjs +24 -0
- package/dist/{chunk-UXNWQZST.mjs → chunk-EMRMGYRE.mjs} +236 -18
- package/dist/chunk-EWZLIYHV.mjs +57 -0
- package/dist/chunk-GBEEG5AN.mjs +35 -0
- package/dist/chunk-GES7U2PM.mjs +68 -0
- package/dist/chunk-HJSN7UYG.mjs +10 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-HMFNFVP7.mjs +14 -0
- package/dist/chunk-J5QP52HA.mjs +1751 -0
- package/dist/{chunk-YNVIUGF5.mjs → chunk-JZVARCRZ.mjs} +3 -28
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-B2TFROFX.mjs → chunk-KWTW6VTG.mjs} +465 -250
- package/dist/chunk-KZ3NX2P7.mjs +21 -0
- package/dist/chunk-L5ORED7V.mjs +25 -0
- package/dist/{chunk-THYRHHO2.mjs → chunk-LGAMKHA2.mjs} +2009 -2351
- package/dist/chunk-LKOB5COA.mjs +117 -0
- package/dist/chunk-LPQOEU5M.mjs +96 -0
- package/dist/{chunk-3LHGNRYE.mjs → chunk-LXWDQLTW.mjs} +7 -2
- package/dist/{chunk-L6TNPLAR.mjs → chunk-OEYI3KRB.mjs} +270 -108
- package/dist/chunk-OM362BRO.mjs +948 -0
- package/dist/chunk-OZDFA4WC.mjs +67 -0
- package/dist/chunk-P36MUFEJ.mjs +10 -0
- package/dist/chunk-PC2G76VW.mjs +2540 -0
- package/dist/{chunk-GCM3JFVA.mjs → chunk-PNEYIHA4.mjs} +11 -2
- package/dist/chunk-PNFKUCG3.mjs +1130 -0
- package/dist/chunk-Q6XWVUZX.mjs +44 -0
- package/dist/chunk-R27EMH5M.mjs +617 -0
- package/dist/{chunk-YMZCMH5M.mjs → chunk-R6A773WK.mjs} +19 -24
- package/dist/chunk-RF7PLG43.mjs +106 -0
- package/dist/chunk-RFBLQLLZ.mjs +16 -0
- package/dist/{chunk-Y3L5MDRV.mjs → chunk-RFVUDM5E.mjs} +17 -2
- package/dist/chunk-S7FL6UAH.mjs +521 -0
- package/dist/chunk-S7GNDRPS.mjs +651 -0
- package/dist/chunk-SP3NM3ZG.mjs +3537 -0
- package/dist/chunk-U476IXKQ.mjs +10 -0
- package/dist/chunk-UA3AUGNV.mjs +462 -0
- package/dist/chunk-UTG5YNXV.mjs +507 -0
- package/dist/chunk-VI4R6W25.mjs +8484 -0
- package/dist/chunk-WQBCBKY3.mjs +8 -0
- package/dist/{chunk-24RXHMEL.mjs → chunk-WVCOCKBU.mjs} +3794 -7570
- package/dist/chunk-X5CV2E2B.mjs +1 -0
- package/dist/chunk-XADZPPZI.mjs +614 -0
- package/dist/chunk-YEEQHT4V.mjs +212 -0
- package/dist/chunk-YTJCGKM4.mjs +56 -0
- package/dist/chunk-YYQIODMF.mjs +275 -0
- package/dist/chunk-ZI34TVOA.mjs +144 -0
- package/dist/{chunk-OOSIDHA3.mjs → chunk-ZKWYWW4Y.mjs} +1 -1
- package/dist/chunk-ZM7KU4YW.mjs +35 -0
- package/dist/{client-BQK6Tf6A.d.ts → client-BOStAkc5.d.ts} +160 -37
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +209 -45
- package/dist/{config-DmXULPa-.d.ts → config-Cjvzbs_j.d.ts} +51 -3
- package/dist/config-DffQlmRq.d.ts +421 -0
- package/dist/direct-executor-ZD64PPEZ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.d.ts +79 -18
- package/dist/distribution/coordinator/etcd.mjs +553 -109
- package/dist/distribution/coordinator/in-memory.d.ts +4 -3
- package/dist/distribution/coordinator/in-memory.mjs +22 -3
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +79 -14
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +12 -7
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +50 -28
- package/dist/distribution.d.ts +144 -23
- package/dist/distribution.mjs +14479 -8173
- package/dist/durability-wiring-FEUBR5BT.mjs +3330 -0
- package/dist/embeddings/openai.d.ts +2 -2
- package/dist/embeddings/openai.mjs +22 -133
- package/dist/{engine-CIOyFozK.d.ts → engine-rC0vjjY-.d.ts} +165 -141
- package/dist/{memory-yFaVn2xZ.d.ts → events-DqOIg5xa.d.ts} +645 -136
- package/dist/{types-DAK7mnUZ.d.ts → index-Bkdt8nDw.d.ts} +182 -45
- package/dist/index.browser.mjs +4 -32180
- package/dist/index.d.ts +91 -20
- package/dist/index.mjs +33 -16
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +2 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/module-3S65MVCM.mjs +2151 -0
- package/dist/partial-words-DTIHO8G4.d.ts +19 -0
- package/dist/partial-words.d.ts +22 -0
- package/dist/partial-words.mjs +17 -0
- package/dist/pattern-Cledccbl.d.ts +18 -0
- package/dist/pattern.d.ts +22 -0
- package/dist/pattern.mjs +12 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +119 -72
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -18
- package/dist/react.d.ts +279 -183
- package/dist/react.mjs +201 -48
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/rerank-K7C4WZFG.mjs +60 -0
- package/dist/rerankers/http.d.ts +50 -0
- package/dist/rerankers/http.mjs +104 -0
- package/dist/restore-queue-KQHA7ZV2.mjs +35 -0
- package/dist/runtime-6Y4ZYXOG.mjs +177 -0
- package/dist/{schema-d5Afu03A.d.ts → schema-4W_7tk1f.d.ts} +81 -11
- package/dist/server.d.ts +13 -12
- package/dist/server.mjs +611 -80
- package/dist/snapshot-QGZPODM5.mjs +799 -0
- package/dist/synonyms-CpOAHPa-.d.ts +52 -0
- package/dist/synonyms.d.ts +22 -0
- package/dist/synonyms.mjs +12 -0
- package/dist/{tasks-CH2Dod8w.d.ts → tasks-CwPTz-tP.d.ts} +10 -3
- package/dist/thread-N3IUGUS3.mjs +697 -0
- package/dist/{types-BQDlIvfS.d.ts → types-2qoitTXg.d.ts} +1 -1
- package/dist/{types-D5OwhtsQ.d.ts → types-Cz5okLml.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +46 -4
- package/dist/vector/search-worker.mjs +21 -11
- package/dist/workers/entry.mjs +9 -6
- package/dist/write-U5RI7MAX.mjs +241 -0
- package/package.json +55 -9
- package/dist/chunk-3L4U6EJQ.mjs +0 -49
- package/dist/chunk-IFBKLBGB.mjs +0 -7
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-U7KYW4BA.mjs +0 -135
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/chunk-ZNZBXMP5.mjs +0 -2117
- package/dist/chunk-ZZT3EAUC.mjs +0 -11168
- package/dist/config-BhldZgaK.d.ts +0 -194
- package/dist/direct-executor-ALAVE7FC.mjs +0 -12
- package/dist/serialization/crc32-worker.d.ts +0 -20
- package/dist/serialization/crc32-worker.mjs +0 -62
- package/dist/simd-DW5AtJ6B.d.ts +0 -41
- package/dist/thread-DYHEQRVY.mjs +0 -402
|
@@ -0,0 +1,948 @@
|
|
|
1
|
+
import { flattenSchema } from './chunk-OEYI3KRB.mjs';
|
|
2
|
+
import { PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, DEFAULT_FUZZY_PREFIX_LENGTH } from './chunk-E74YLAIG.mjs';
|
|
3
|
+
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
4
|
+
import { isWordIndexedBase } from './chunk-LKOB5COA.mjs';
|
|
5
|
+
import { encodeStringBlob, tokenize, resolveStopWords, transformToken, visitTokens, visitSurfaceSpellings } from './chunk-UTG5YNXV.mjs';
|
|
6
|
+
import { compareCodePoints, codePointCount } from './chunk-YTJCGKM4.mjs';
|
|
7
|
+
|
|
8
|
+
// src/core/partition/suggestions.ts
|
|
9
|
+
function verbatimOccurrences(state, token) {
|
|
10
|
+
const postingList = state.invertedIdx.lookup(token);
|
|
11
|
+
if (!postingList) return 0;
|
|
12
|
+
const derived = postingList.totalTermFrequency - state.surfaceRegistry.stemChangedTotalFor(token);
|
|
13
|
+
return derived > 0 ? derived : 0;
|
|
14
|
+
}
|
|
15
|
+
function suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit) {
|
|
16
|
+
if (limit <= 0 || surfacePrefix.length === 0) return [];
|
|
17
|
+
const groups = /* @__PURE__ */ new Map();
|
|
18
|
+
for (const candidate of state.surfaceRegistry.candidatesForPrefix(surfacePrefix)) {
|
|
19
|
+
const postingList = state.invertedIdx.lookup(candidate.token);
|
|
20
|
+
if (!postingList || postingList.docIdSet.size === 0) continue;
|
|
21
|
+
let group = groups.get(candidate.token);
|
|
22
|
+
if (!group) {
|
|
23
|
+
group = { documentFrequency: postingList.docIdSet.size, surfaces: [] };
|
|
24
|
+
groups.set(candidate.token, group);
|
|
25
|
+
}
|
|
26
|
+
group.surfaces.push({ surface: candidate.surface, occurrences: candidate.occurrences });
|
|
27
|
+
}
|
|
28
|
+
const termPrefixes = state.surfaceRegistry.size() === 0 && stemmedPrefix !== surfacePrefix && stemmedPrefix.length > 0 ? [surfacePrefix, stemmedPrefix] : [surfacePrefix];
|
|
29
|
+
for (const prefix of termPrefixes) {
|
|
30
|
+
for (const suggestion of state.invertedIdx.prefixSearch(prefix, Number.MAX_SAFE_INTEGER)) {
|
|
31
|
+
const occurrences = verbatimOccurrences(state, suggestion.term);
|
|
32
|
+
if (occurrences === 0) continue;
|
|
33
|
+
let group = groups.get(suggestion.term);
|
|
34
|
+
if (!group) {
|
|
35
|
+
group = { documentFrequency: suggestion.documentFrequency, surfaces: [] };
|
|
36
|
+
groups.set(suggestion.term, group);
|
|
37
|
+
} else if (group.surfaces.some((s) => s.surface === suggestion.term)) {
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
group.surfaces.push({ surface: suggestion.term, occurrences });
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
const results = [];
|
|
44
|
+
for (const [token, group] of groups) {
|
|
45
|
+
results.push({ token, documentFrequency: group.documentFrequency, surfaces: group.surfaces });
|
|
46
|
+
}
|
|
47
|
+
results.sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.token, b.token));
|
|
48
|
+
if (results.length > limit) results.length = limit;
|
|
49
|
+
return results;
|
|
50
|
+
}
|
|
51
|
+
function partitionDocumentFrequency(subs, token) {
|
|
52
|
+
let documents = 0;
|
|
53
|
+
for (const sub of subs) documents += sub.invertedIdx.documentFrequency(token);
|
|
54
|
+
return documents;
|
|
55
|
+
}
|
|
56
|
+
function expandTermPrefix(subs, surfacePrefix, stemmedToken, maxExpansions) {
|
|
57
|
+
if (maxExpansions <= 0 || surfacePrefix.length === 0) return [];
|
|
58
|
+
const candidates = /* @__PURE__ */ new Set();
|
|
59
|
+
const holdsSurfaceForms = subs.some((sub) => sub.surfaceRegistry.size() > 0);
|
|
60
|
+
const termPrefixes = !holdsSurfaceForms && stemmedToken !== surfacePrefix && stemmedToken.length > 0 ? [surfacePrefix, stemmedToken] : [surfacePrefix];
|
|
61
|
+
for (const sub of subs) {
|
|
62
|
+
for (const candidate of sub.surfaceRegistry.candidatesForPrefix(surfacePrefix)) candidates.add(candidate.token);
|
|
63
|
+
for (const prefix of termPrefixes) {
|
|
64
|
+
for (const suggestion of sub.invertedIdx.prefixSearch(prefix, Number.MAX_SAFE_INTEGER)) {
|
|
65
|
+
if (candidates.has(suggestion.term)) continue;
|
|
66
|
+
if (verbatimOccurrences(sub, suggestion.term) > 0) candidates.add(suggestion.term);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
const ranked = [];
|
|
71
|
+
for (const token of candidates) {
|
|
72
|
+
const documents = partitionDocumentFrequency(subs, token);
|
|
73
|
+
if (documents > 0) ranked.push({ token, documents });
|
|
74
|
+
}
|
|
75
|
+
ranked.sort((a, b) => b.documents - a.documents || compareCodePoints(a.token, b.token));
|
|
76
|
+
if (ranked.length > maxExpansions) ranked.length = maxExpansions;
|
|
77
|
+
return ranked.map((entry) => entry.token);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// src/core/partition/partial-words/text.ts
|
|
81
|
+
var encoder = new TextEncoder();
|
|
82
|
+
var LAST_BASIC_PLANE_CODE_POINT = 65535;
|
|
83
|
+
var BASIC_PLANE_SIZE = 65536;
|
|
84
|
+
var UTF8_CONTINUATION_MASK = 192;
|
|
85
|
+
var UTF8_CONTINUATION_BITS = 128;
|
|
86
|
+
function codePointsOf(text) {
|
|
87
|
+
const points = [];
|
|
88
|
+
let index = 0;
|
|
89
|
+
while (index < text.length) {
|
|
90
|
+
const point = text.codePointAt(index) ?? text.charCodeAt(index);
|
|
91
|
+
points.push(point);
|
|
92
|
+
index += point > LAST_BASIC_PLANE_CODE_POINT ? 2 : 1;
|
|
93
|
+
}
|
|
94
|
+
return points;
|
|
95
|
+
}
|
|
96
|
+
function runKeyAt(points, at) {
|
|
97
|
+
return String.fromCodePoint(points[at], points[at + 1], points[at + 2]);
|
|
98
|
+
}
|
|
99
|
+
function packedRunKeyAt(points, at) {
|
|
100
|
+
const first = points[at];
|
|
101
|
+
const second = points[at + 1];
|
|
102
|
+
const third = points[at + 2];
|
|
103
|
+
if ((first | second | third) > LAST_BASIC_PLANE_CODE_POINT) return String.fromCodePoint(first, second, third);
|
|
104
|
+
return first * BASIC_PLANE_SIZE * BASIC_PLANE_SIZE + second * BASIC_PLANE_SIZE + third;
|
|
105
|
+
}
|
|
106
|
+
function partialWordProbe(text) {
|
|
107
|
+
return { text, codePoints: codePointsOf(text), utf8: encoder.encode(text) };
|
|
108
|
+
}
|
|
109
|
+
function utf8Contains(haystack, start, end, needle) {
|
|
110
|
+
const last = end - needle.length;
|
|
111
|
+
for (let from = start; from <= last; from++) {
|
|
112
|
+
let at = 0;
|
|
113
|
+
while (at < needle.length && haystack[from + at] === needle[at]) at++;
|
|
114
|
+
if (at === needle.length) return true;
|
|
115
|
+
}
|
|
116
|
+
return false;
|
|
117
|
+
}
|
|
118
|
+
function utf8CodePointCount(bytes, start, end) {
|
|
119
|
+
let count = 0;
|
|
120
|
+
for (let at = start; at < end; at++) {
|
|
121
|
+
if ((bytes[at] & UTF8_CONTINUATION_MASK) !== UTF8_CONTINUATION_BITS) count++;
|
|
122
|
+
}
|
|
123
|
+
return count;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// src/core/partition/partial-words/expansion.ts
|
|
127
|
+
function excludedFromRequest(request, token) {
|
|
128
|
+
if (token === request.ownToken) return true;
|
|
129
|
+
return request.tolerance > 0 && fuzzyTermMatches(request.ownToken, token, request.tolerance, request.prefixLength);
|
|
130
|
+
}
|
|
131
|
+
function addedWordsFor(subs, readers, request) {
|
|
132
|
+
if (request.share <= 0) return [];
|
|
133
|
+
const probe = partialWordProbe(request.written);
|
|
134
|
+
const shortestSpelling = /* @__PURE__ */ new Map();
|
|
135
|
+
for (const reader of readers) {
|
|
136
|
+
reader.forEachContaining(probe, (token, spellingLength) => {
|
|
137
|
+
const known = shortestSpelling.get(token);
|
|
138
|
+
if (known === void 0 || spellingLength < known) shortestSpelling.set(token, spellingLength);
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
const ranked = [];
|
|
142
|
+
for (const [token, spellingLength] of shortestSpelling) {
|
|
143
|
+
if (excludedFromRequest(request, token)) continue;
|
|
144
|
+
const documents = partitionDocumentFrequency(subs, token);
|
|
145
|
+
if (documents > 0) ranked.push({ token, documents, spellingLength });
|
|
146
|
+
}
|
|
147
|
+
ranked.sort((a, b) => b.documents - a.documents || compareCodePoints(a.token, b.token));
|
|
148
|
+
if (ranked.length > request.share) ranked.length = request.share;
|
|
149
|
+
const writtenLength = probe.codePoints.length;
|
|
150
|
+
return ranked.map((word) => ({ token: word.token, factor: writtenLength / word.spellingLength }));
|
|
151
|
+
}
|
|
152
|
+
function expandPartialWords(subs, analysis, requests) {
|
|
153
|
+
const readers = [];
|
|
154
|
+
for (const sub of subs) {
|
|
155
|
+
const reader = sub.partialWords.readerFor(analysis);
|
|
156
|
+
if (reader !== null) readers.push(reader);
|
|
157
|
+
}
|
|
158
|
+
return requests.map((request) => readers.length === 0 ? [] : addedWordsFor(subs, readers, request));
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// src/core/partition/partial-words/constants.ts
|
|
162
|
+
var PARTIAL_WORD_RUN_CODE_POINTS = 3;
|
|
163
|
+
var GROWING_PARTIAL_ENTRY_BYTES = 104;
|
|
164
|
+
var GROWING_PARTIAL_PENDING_ENTRY_BYTES = 8;
|
|
165
|
+
var GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY = 2;
|
|
166
|
+
var GROWING_PARTIAL_RUN_BYTES = 120;
|
|
167
|
+
var GROWING_PARTIAL_RUN_MEMBER_BYTES = 24;
|
|
168
|
+
var FROZEN_PARTIAL_REMOVED_ENTRY_BYTES = 48;
|
|
169
|
+
|
|
170
|
+
// src/core/partition/utils.ts
|
|
171
|
+
function getOrCreateFieldNameIndex(table, fieldName) {
|
|
172
|
+
const existing = table.indexMap.get(fieldName);
|
|
173
|
+
if (existing !== void 0) return existing;
|
|
174
|
+
const idx = table.names.length;
|
|
175
|
+
table.names.push(fieldName);
|
|
176
|
+
table.indexMap.set(fieldName, idx);
|
|
177
|
+
return idx;
|
|
178
|
+
}
|
|
179
|
+
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
180
|
+
function getNestedValue(obj, path) {
|
|
181
|
+
const parts = path.split(".");
|
|
182
|
+
let current = obj;
|
|
183
|
+
for (let i = 0; i < parts.length; i++) {
|
|
184
|
+
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
185
|
+
if (!Object.hasOwn(current, parts[i])) return void 0;
|
|
186
|
+
current = current[parts[i]];
|
|
187
|
+
}
|
|
188
|
+
return current;
|
|
189
|
+
}
|
|
190
|
+
function stringArraysEqual(a, b) {
|
|
191
|
+
if (a === b) return true;
|
|
192
|
+
if (!a || !b) return false;
|
|
193
|
+
if (a.length !== b.length) return false;
|
|
194
|
+
for (let i = 0; i < a.length; i++) {
|
|
195
|
+
if (a[i] !== b[i]) return false;
|
|
196
|
+
}
|
|
197
|
+
return true;
|
|
198
|
+
}
|
|
199
|
+
function textFieldsChanged(oldDoc, newDoc, fields) {
|
|
200
|
+
for (const { path, base } of fields) {
|
|
201
|
+
if (!isWordIndexedBase(base)) continue;
|
|
202
|
+
const oldVal = getNestedValue(oldDoc, path);
|
|
203
|
+
const newVal = getNestedValue(newDoc, path);
|
|
204
|
+
if (base === "string") {
|
|
205
|
+
if (oldVal !== newVal) return true;
|
|
206
|
+
} else {
|
|
207
|
+
if (!stringArraysEqual(oldVal, newVal)) return true;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
return false;
|
|
211
|
+
}
|
|
212
|
+
function parseVectorDimension(fieldType) {
|
|
213
|
+
const match = VECTOR_PATTERN.exec(fieldType);
|
|
214
|
+
return match ? Number.parseInt(match[1], 10) : null;
|
|
215
|
+
}
|
|
216
|
+
function tokenizeOptions(options) {
|
|
217
|
+
return {
|
|
218
|
+
stem: true,
|
|
219
|
+
removeStopWords: true,
|
|
220
|
+
collectSurfaces: options?.collectSurfaces === true,
|
|
221
|
+
stopWordOverride: options?.stopWordOverride,
|
|
222
|
+
customTokenizer: options?.customTokenizer
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
function getFlatSchema(state, schema) {
|
|
226
|
+
if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
|
|
227
|
+
state.flatSchemaCache = flattenSchema(schema);
|
|
228
|
+
state.lastSchemaRef = schema;
|
|
229
|
+
return state.flatSchemaCache;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// src/core/partition/partial-words/spellings.ts
|
|
233
|
+
var LAST_BASIC_PLANE_CODE_POINT2 = 65535;
|
|
234
|
+
function codePointCountOf(text) {
|
|
235
|
+
let count = 0;
|
|
236
|
+
for (let index = 0; index < text.length; count++) {
|
|
237
|
+
index += (text.codePointAt(index) ?? 0) > LAST_BASIC_PLANE_CODE_POINT2 ? 2 : 1;
|
|
238
|
+
}
|
|
239
|
+
return count;
|
|
240
|
+
}
|
|
241
|
+
function holdsRun(spelling) {
|
|
242
|
+
if (spelling.length > 2 * PARTIAL_WORD_RUN_CODE_POINTS) return true;
|
|
243
|
+
if (spelling.length <= PARTIAL_WORD_RUN_CODE_POINTS) return false;
|
|
244
|
+
return codePointCountOf(spelling) > PARTIAL_WORD_RUN_CODE_POINTS;
|
|
245
|
+
}
|
|
246
|
+
function countSpelling(counts, token, spelling, change) {
|
|
247
|
+
if (!holdsRun(spelling)) return;
|
|
248
|
+
let byToken = counts.get(token);
|
|
249
|
+
if (byToken === void 0) {
|
|
250
|
+
if (change <= 0) return;
|
|
251
|
+
byToken = /* @__PURE__ */ new Map();
|
|
252
|
+
counts.set(token, byToken);
|
|
253
|
+
}
|
|
254
|
+
const next = (byToken.get(spelling) ?? 0) + change;
|
|
255
|
+
if (next > 0) {
|
|
256
|
+
byToken.set(spelling, next);
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
byToken.delete(spelling);
|
|
260
|
+
if (byToken.size === 0) counts.delete(token);
|
|
261
|
+
}
|
|
262
|
+
function textsOf(value) {
|
|
263
|
+
if (typeof value === "string") return [value];
|
|
264
|
+
if (!Array.isArray(value)) return [];
|
|
265
|
+
return value.filter((element) => typeof element === "string");
|
|
266
|
+
}
|
|
267
|
+
function visitFieldTexts(document, fields, visit) {
|
|
268
|
+
for (const field of fields) {
|
|
269
|
+
for (const text of textsOf(getNestedValue(document, field))) visit(text);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
function visitDocumentSpellings(document, analysis, visit) {
|
|
273
|
+
const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
|
|
274
|
+
const visitOccurrence = (token, _position, surface) => {
|
|
275
|
+
visit(token, surface ?? token);
|
|
276
|
+
};
|
|
277
|
+
visitFieldTexts(document, analysis.fields, (text) => visitTokens(text, analysis.language, tokenizing, visitOccurrence));
|
|
278
|
+
}
|
|
279
|
+
function visitDocumentSurfaceSpellings(document, analysis, visit) {
|
|
280
|
+
if (analysis.options?.customTokenizer !== void 0) return;
|
|
281
|
+
visitFieldTexts(document, analysis.fields, (text) => visitSurfaceSpellings(text, analysis.language, visit));
|
|
282
|
+
}
|
|
283
|
+
function countDocumentSpellings(counts, document, analysis, direction) {
|
|
284
|
+
visitDocumentSpellings(document, analysis, (token, spelling) => countSpelling(counts, token, spelling, direction));
|
|
285
|
+
}
|
|
286
|
+
function spellingCountsOf(documents, analysis) {
|
|
287
|
+
const counts = /* @__PURE__ */ new Map();
|
|
288
|
+
for (const document of documents) countDocumentSpellings(counts, document, analysis, 1);
|
|
289
|
+
return counts;
|
|
290
|
+
}
|
|
291
|
+
function encodePartialSpellings(fields, counts) {
|
|
292
|
+
const encoded = { fields: [...fields], tokens: [], spellings: [], counts: [] };
|
|
293
|
+
for (const [token, byToken] of counts) {
|
|
294
|
+
for (const [spelling, count] of byToken) {
|
|
295
|
+
encoded.tokens.push(token);
|
|
296
|
+
encoded.spellings.push(spelling);
|
|
297
|
+
encoded.counts.push(count);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
return encoded;
|
|
301
|
+
}
|
|
302
|
+
function sameFields(left, right) {
|
|
303
|
+
if (left.length !== right.length) return false;
|
|
304
|
+
for (let at = 0; at < left.length; at++) {
|
|
305
|
+
if (left[at] !== right[at]) return false;
|
|
306
|
+
}
|
|
307
|
+
return true;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
// src/core/partition/partial-words/frozen-table.ts
|
|
311
|
+
var decoder = new TextDecoder();
|
|
312
|
+
function compareEntries(left, right) {
|
|
313
|
+
return compareCodePoints(left.spelling, right.spelling) || left.tokenIndex - right.tokenIndex;
|
|
314
|
+
}
|
|
315
|
+
function orderedEntriesOf(source, tokenTable) {
|
|
316
|
+
const entries = [];
|
|
317
|
+
for (let at = 0; at < source.tokens.length; at++) {
|
|
318
|
+
if (source.counts[at] <= 0) continue;
|
|
319
|
+
const tokenIndex = tokenTable.find(source.tokens[at]);
|
|
320
|
+
if (tokenIndex >= 0) entries.push({ spelling: source.spellings[at], tokenIndex, count: source.counts[at] });
|
|
321
|
+
}
|
|
322
|
+
entries.sort(compareEntries);
|
|
323
|
+
const distinct = [];
|
|
324
|
+
for (const entry of entries) {
|
|
325
|
+
const last = distinct[distinct.length - 1];
|
|
326
|
+
if (last !== void 0 && compareEntries(last, entry) === 0) last.count += entry.count;
|
|
327
|
+
else distinct.push(entry);
|
|
328
|
+
}
|
|
329
|
+
return distinct;
|
|
330
|
+
}
|
|
331
|
+
function buildPartialWordTable(source, tokenTable) {
|
|
332
|
+
const spellings = [];
|
|
333
|
+
const spellingTokens = [];
|
|
334
|
+
const spellingCounts = [];
|
|
335
|
+
const runLists = /* @__PURE__ */ new Map();
|
|
336
|
+
const addSpelling = (spelling, tokenIndex, count) => {
|
|
337
|
+
const points = codePointsOf(spelling);
|
|
338
|
+
if (points.length <= PARTIAL_WORD_RUN_CODE_POINTS) return;
|
|
339
|
+
const entry = spellings.length;
|
|
340
|
+
spellings.push(spelling);
|
|
341
|
+
spellingTokens.push(tokenIndex);
|
|
342
|
+
spellingCounts.push(count);
|
|
343
|
+
for (let at = 0; at + PARTIAL_WORD_RUN_CODE_POINTS <= points.length; at++) {
|
|
344
|
+
const key = runKeyAt(points, at);
|
|
345
|
+
const list = runLists.get(key);
|
|
346
|
+
if (list === void 0) runLists.set(key, [entry]);
|
|
347
|
+
else if (list[list.length - 1] !== entry) list.push(entry);
|
|
348
|
+
}
|
|
349
|
+
};
|
|
350
|
+
for (const entry of orderedEntriesOf(source, tokenTable)) addSpelling(entry.spelling, entry.tokenIndex, entry.count);
|
|
351
|
+
const runKeys = [...runLists.keys()].sort(compareCodePoints);
|
|
352
|
+
const runs = new Uint32Array(runKeys.length * PARTIAL_WORD_RUN_CODE_POINTS);
|
|
353
|
+
const runOffsets = new Uint32Array(runKeys.length + 1);
|
|
354
|
+
let memberCount = 0;
|
|
355
|
+
for (const key of runKeys) memberCount += runLists.get(key)?.length ?? 0;
|
|
356
|
+
const runEntries = new Uint32Array(memberCount);
|
|
357
|
+
let cursor = 0;
|
|
358
|
+
for (let run = 0; run < runKeys.length; run++) {
|
|
359
|
+
const points = codePointsOf(runKeys[run]);
|
|
360
|
+
runs.set(points, run * PARTIAL_WORD_RUN_CODE_POINTS);
|
|
361
|
+
for (const entry of runLists.get(runKeys[run]) ?? []) runEntries[cursor++] = entry;
|
|
362
|
+
runOffsets[run + 1] = cursor;
|
|
363
|
+
}
|
|
364
|
+
const spellingBlob = encodeStringBlob(spellings);
|
|
365
|
+
return {
|
|
366
|
+
fields: [...source.fields],
|
|
367
|
+
spellings: spellingBlob.blob,
|
|
368
|
+
spellingOffsets: spellingBlob.offsets,
|
|
369
|
+
spellingTokens: Uint32Array.from(spellingTokens),
|
|
370
|
+
spellingCounts: Uint32Array.from(spellingCounts),
|
|
371
|
+
runs,
|
|
372
|
+
runOffsets,
|
|
373
|
+
runEntries
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
function countTableSpellings(counts, table, tokenTable) {
|
|
377
|
+
const { spellings, spellingOffsets, spellingTokens, spellingCounts } = table;
|
|
378
|
+
for (let entry = 0; entry < spellingTokens.length; entry++) {
|
|
379
|
+
const spelling = decoder.decode(spellings.subarray(spellingOffsets[entry], spellingOffsets[entry + 1]));
|
|
380
|
+
countSpelling(counts, tokenTable.tokenAt(spellingTokens[entry]), spelling, spellingCounts[entry]);
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
function findRun(runs, first, second, third) {
|
|
384
|
+
let low = 0;
|
|
385
|
+
let high = runs.length / PARTIAL_WORD_RUN_CODE_POINTS;
|
|
386
|
+
while (low < high) {
|
|
387
|
+
const middle = low + high >>> 1;
|
|
388
|
+
const at = middle * PARTIAL_WORD_RUN_CODE_POINTS;
|
|
389
|
+
const difference = runs[at] - first || runs[at + 1] - second || runs[at + 2] - third;
|
|
390
|
+
if (difference === 0) return middle;
|
|
391
|
+
if (difference < 0) low = middle + 1;
|
|
392
|
+
else high = middle;
|
|
393
|
+
}
|
|
394
|
+
return -1;
|
|
395
|
+
}
|
|
396
|
+
function partialWordTableBytes(table) {
|
|
397
|
+
return table.spellings.byteLength + table.spellingOffsets.byteLength + table.spellingTokens.byteLength + table.spellingCounts.byteLength + table.runs.byteLength + table.runOffsets.byteLength + table.runEntries.byteLength;
|
|
398
|
+
}
|
|
399
|
+
function readPartialWordTable(table, tokenTable, removedCounts) {
|
|
400
|
+
const { spellings, spellingOffsets, spellingTokens, spellingCounts, runs, runOffsets, runEntries } = table;
|
|
401
|
+
function narrowestRun(points) {
|
|
402
|
+
let narrowest = -1;
|
|
403
|
+
let narrowestLength = Number.POSITIVE_INFINITY;
|
|
404
|
+
for (let at = 0; at + PARTIAL_WORD_RUN_CODE_POINTS <= points.length; at++) {
|
|
405
|
+
const run = findRun(runs, points[at], points[at + 1], points[at + 2]);
|
|
406
|
+
if (run < 0) return -1;
|
|
407
|
+
const length = runOffsets[run + 1] - runOffsets[run];
|
|
408
|
+
if (length < narrowestLength) {
|
|
409
|
+
narrowest = run;
|
|
410
|
+
narrowestLength = length;
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
return narrowest;
|
|
414
|
+
}
|
|
415
|
+
return {
|
|
416
|
+
forEachContaining(probe, visit) {
|
|
417
|
+
if (probe.codePoints.length < PARTIAL_WORD_RUN_CODE_POINTS) return;
|
|
418
|
+
const run = narrowestRun(probe.codePoints);
|
|
419
|
+
if (run < 0) return;
|
|
420
|
+
for (let member = runOffsets[run]; member < runOffsets[run + 1]; member++) {
|
|
421
|
+
const entry = runEntries[member];
|
|
422
|
+
const start = spellingOffsets[entry];
|
|
423
|
+
const end = spellingOffsets[entry + 1];
|
|
424
|
+
if (end - start <= probe.utf8.length) continue;
|
|
425
|
+
if ((removedCounts.get(entry) ?? 0) >= spellingCounts[entry]) continue;
|
|
426
|
+
if (!utf8Contains(spellings, start, end, probe.utf8)) continue;
|
|
427
|
+
const length = utf8CodePointCount(spellings, start, end);
|
|
428
|
+
if (length <= probe.codePoints.length) continue;
|
|
429
|
+
visit(tokenTable.tokenAt(spellingTokens[entry]), length);
|
|
430
|
+
}
|
|
431
|
+
},
|
|
432
|
+
entryOf(spelling) {
|
|
433
|
+
const probe = partialWordProbe(spelling);
|
|
434
|
+
if (probe.codePoints.length <= PARTIAL_WORD_RUN_CODE_POINTS) return -1;
|
|
435
|
+
const run = narrowestRun(probe.codePoints);
|
|
436
|
+
if (run < 0) return -1;
|
|
437
|
+
let exhausted = -1;
|
|
438
|
+
for (let member = runOffsets[run]; member < runOffsets[run + 1]; member++) {
|
|
439
|
+
const entry = runEntries[member];
|
|
440
|
+
const start = spellingOffsets[entry];
|
|
441
|
+
const end = spellingOffsets[entry + 1];
|
|
442
|
+
if (end - start !== probe.utf8.length || !utf8Contains(spellings, start, end, probe.utf8)) continue;
|
|
443
|
+
if ((removedCounts.get(entry) ?? 0) < spellingCounts[entry]) return entry;
|
|
444
|
+
if (exhausted < 0) exhausted = entry;
|
|
445
|
+
}
|
|
446
|
+
return exhausted;
|
|
447
|
+
},
|
|
448
|
+
bytes() {
|
|
449
|
+
return partialWordTableBytes(table);
|
|
450
|
+
}
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// src/core/partition/partial-words/frozen-slot.ts
|
|
455
|
+
function tombstonedOrdinalsOf(documents) {
|
|
456
|
+
let marks = null;
|
|
457
|
+
for (let ordinal = 0; ordinal < documents.count; ordinal++) {
|
|
458
|
+
if (!documents.isTombstoned(ordinal)) continue;
|
|
459
|
+
marks ??= new Uint8Array(documents.count);
|
|
460
|
+
marks[ordinal] = 1;
|
|
461
|
+
}
|
|
462
|
+
return marks;
|
|
463
|
+
}
|
|
464
|
+
function addCounts(target, source) {
|
|
465
|
+
for (const [token, bySpelling] of source) {
|
|
466
|
+
for (const [spelling, count] of bySpelling) countSpelling(target, token, spelling, count);
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
function countRemovedSpellings(removed, document, analysis) {
|
|
470
|
+
const count = (spelling) => {
|
|
471
|
+
removed.set(spelling, (removed.get(spelling) ?? 0) + 1);
|
|
472
|
+
};
|
|
473
|
+
if (analysis.options?.customTokenizer !== void 0) {
|
|
474
|
+
visitDocumentSpellings(document, analysis, (_token, spelling) => count(spelling));
|
|
475
|
+
return;
|
|
476
|
+
}
|
|
477
|
+
visitDocumentSurfaceSpellings(document, analysis, count);
|
|
478
|
+
}
|
|
479
|
+
function unstemmedSpellingsOf(analysis) {
|
|
480
|
+
const { language } = analysis;
|
|
481
|
+
if (language.normalizer === void 0 || language.tokenizer?.normalizeDiacritics === true) return () => null;
|
|
482
|
+
const stopWords = resolveStopWords(language, analysis.options?.stopWordOverride);
|
|
483
|
+
return (spelling) => {
|
|
484
|
+
if (stopWords.has(spelling)) return null;
|
|
485
|
+
const normalized = transformToken(spelling, language, false, false);
|
|
486
|
+
return normalized === spelling ? null : normalized;
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
function subtractRemovedSpellings(counts, removed, analysis) {
|
|
490
|
+
const tokenOf = /* @__PURE__ */ new Map();
|
|
491
|
+
for (const [token, bySpelling] of counts) {
|
|
492
|
+
for (const spelling of bySpelling.keys()) tokenOf.set(spelling, token);
|
|
493
|
+
}
|
|
494
|
+
const unstemmedSpelling = unstemmedSpellingsOf(analysis);
|
|
495
|
+
for (const [spelling, count] of removed) {
|
|
496
|
+
const stored = tokenOf.has(spelling) ? spelling : unstemmedSpelling(spelling) ?? spelling;
|
|
497
|
+
const token = tokenOf.get(stored);
|
|
498
|
+
if (token !== void 0) countSpelling(counts, token, stored, -count);
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
function createFrozenPartialWordSlot(initial, tokenTable, documents) {
|
|
502
|
+
let table = initial;
|
|
503
|
+
let uncounted = null;
|
|
504
|
+
let removedCounts = /* @__PURE__ */ new Map();
|
|
505
|
+
let reader = initial === null ? null : readPartialWordTable(initial, tokenTable, removedCounts);
|
|
506
|
+
let pendingRemovals = [];
|
|
507
|
+
function isCounted(ordinal) {
|
|
508
|
+
return uncounted === null || uncounted[ordinal] === 0;
|
|
509
|
+
}
|
|
510
|
+
function subtractPendingRemovals(current, analysis) {
|
|
511
|
+
if (pendingRemovals.length === 0) return;
|
|
512
|
+
const removed = /* @__PURE__ */ new Map();
|
|
513
|
+
for (const ordinal of pendingRemovals) countRemovedSpellings(removed, documents.documentAt(ordinal), analysis);
|
|
514
|
+
pendingRemovals = [];
|
|
515
|
+
const unstemmedSpelling = unstemmedSpellingsOf(analysis);
|
|
516
|
+
for (const [spelling, count] of removed) {
|
|
517
|
+
let entry = current.entryOf(spelling);
|
|
518
|
+
if (entry < 0) {
|
|
519
|
+
const fallback = unstemmedSpelling(spelling);
|
|
520
|
+
if (fallback !== null) entry = current.entryOf(fallback);
|
|
521
|
+
}
|
|
522
|
+
if (entry >= 0) removedCounts.set(entry, (removedCounts.get(entry) ?? 0) + count);
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
function rebuild(analysis) {
|
|
526
|
+
uncounted = tombstonedOrdinalsOf(documents);
|
|
527
|
+
const counts = /* @__PURE__ */ new Map();
|
|
528
|
+
for (let ordinal = 0; ordinal < documents.count; ordinal++) {
|
|
529
|
+
if (isCounted(ordinal)) countDocumentSpellings(counts, documents.documentAt(ordinal), analysis, 1);
|
|
530
|
+
}
|
|
531
|
+
const rebuilt = buildPartialWordTable(encodePartialSpellings(analysis.fields, counts), tokenTable);
|
|
532
|
+
table = rebuilt;
|
|
533
|
+
removedCounts = /* @__PURE__ */ new Map();
|
|
534
|
+
pendingRemovals = [];
|
|
535
|
+
const rebuiltReader = readPartialWordTable(rebuilt, tokenTable, removedCounts);
|
|
536
|
+
reader = rebuiltReader;
|
|
537
|
+
return rebuiltReader;
|
|
538
|
+
}
|
|
539
|
+
function correctionsAgainstTable(mergedOrdinals) {
|
|
540
|
+
let corrections = 0;
|
|
541
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
542
|
+
if (mergedOrdinals[ordinal] >= 0 !== isCounted(ordinal)) corrections += 1;
|
|
543
|
+
}
|
|
544
|
+
return corrections;
|
|
545
|
+
}
|
|
546
|
+
function keptCount(mergedOrdinals) {
|
|
547
|
+
let kept = 0;
|
|
548
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
549
|
+
if (mergedOrdinals[ordinal] >= 0) kept += 1;
|
|
550
|
+
}
|
|
551
|
+
return kept;
|
|
552
|
+
}
|
|
553
|
+
return {
|
|
554
|
+
readerFor(analysis) {
|
|
555
|
+
if (analysis.fields.length === 0) return null;
|
|
556
|
+
if (table === null || reader === null || !sameFields(table.fields, analysis.fields)) return rebuild(analysis);
|
|
557
|
+
subtractPendingRemovals(reader, analysis);
|
|
558
|
+
return reader;
|
|
559
|
+
},
|
|
560
|
+
afterRemoval(ordinal) {
|
|
561
|
+
if (table !== null) pendingRemovals.push(ordinal);
|
|
562
|
+
},
|
|
563
|
+
tableOfEveryDocument(analysis) {
|
|
564
|
+
if (table === null || uncounted !== null || !sameFields(table.fields, analysis.fields)) return null;
|
|
565
|
+
return table;
|
|
566
|
+
},
|
|
567
|
+
countKeptSpellings(counts, analysis, mergedOrdinals) {
|
|
568
|
+
if (table === null || !sameFields(table.fields, analysis.fields) || correctionsAgainstTable(mergedOrdinals) > keptCount(mergedOrdinals)) {
|
|
569
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
570
|
+
if (mergedOrdinals[ordinal] >= 0) countDocumentSpellings(counts, documents.documentAt(ordinal), analysis, 1);
|
|
571
|
+
}
|
|
572
|
+
return;
|
|
573
|
+
}
|
|
574
|
+
const corrected = /* @__PURE__ */ new Map();
|
|
575
|
+
countTableSpellings(corrected, table, tokenTable);
|
|
576
|
+
const removed = /* @__PURE__ */ new Map();
|
|
577
|
+
const returned = [];
|
|
578
|
+
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
579
|
+
const kept = mergedOrdinals[ordinal] >= 0;
|
|
580
|
+
if (kept === isCounted(ordinal)) continue;
|
|
581
|
+
if (kept) returned.push(ordinal);
|
|
582
|
+
else countRemovedSpellings(removed, documents.documentAt(ordinal), analysis);
|
|
583
|
+
}
|
|
584
|
+
subtractRemovedSpellings(corrected, removed, analysis);
|
|
585
|
+
for (const ordinal of returned) countDocumentSpellings(corrected, documents.documentAt(ordinal), analysis, 1);
|
|
586
|
+
addCounts(counts, corrected);
|
|
587
|
+
},
|
|
588
|
+
spellings() {
|
|
589
|
+
return null;
|
|
590
|
+
},
|
|
591
|
+
bytes() {
|
|
592
|
+
if (table === null) return 0;
|
|
593
|
+
return partialWordTableBytes(table) + removedCounts.size * FROZEN_PARTIAL_REMOVED_ENTRY_BYTES;
|
|
594
|
+
}
|
|
595
|
+
};
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
// src/core/partition/partial-words/growing-slot.ts
|
|
599
|
+
function createGrowingPartialWordSlot(view) {
|
|
600
|
+
let fields = [];
|
|
601
|
+
let everyDocumentPending = false;
|
|
602
|
+
const pending = /* @__PURE__ */ new Set();
|
|
603
|
+
const entriesByToken = /* @__PURE__ */ new Map();
|
|
604
|
+
const entrySpellings = [];
|
|
605
|
+
const entryTokens = [];
|
|
606
|
+
const entryLengths = [];
|
|
607
|
+
const entryCounts = [];
|
|
608
|
+
const entryInRuns = [];
|
|
609
|
+
const freeEntries = [];
|
|
610
|
+
const entriesOutsideRuns = [];
|
|
611
|
+
const runs = /* @__PURE__ */ new Map();
|
|
612
|
+
let liveEntries = 0;
|
|
613
|
+
let runMembers = 0;
|
|
614
|
+
function resetSpellings() {
|
|
615
|
+
pending.clear();
|
|
616
|
+
entriesByToken.clear();
|
|
617
|
+
entrySpellings.length = 0;
|
|
618
|
+
entryTokens.length = 0;
|
|
619
|
+
entryLengths.length = 0;
|
|
620
|
+
entryCounts.length = 0;
|
|
621
|
+
entryInRuns.length = 0;
|
|
622
|
+
freeEntries.length = 0;
|
|
623
|
+
entriesOutsideRuns.length = 0;
|
|
624
|
+
runs.clear();
|
|
625
|
+
liveEntries = 0;
|
|
626
|
+
runMembers = 0;
|
|
627
|
+
}
|
|
628
|
+
function addEntry(byToken, token, spelling, count) {
|
|
629
|
+
const length = codePointCountOf(spelling);
|
|
630
|
+
if (length <= PARTIAL_WORD_RUN_CODE_POINTS) return;
|
|
631
|
+
const entry = freeEntries.pop() ?? entrySpellings.length;
|
|
632
|
+
entrySpellings[entry] = spelling;
|
|
633
|
+
entryTokens[entry] = token;
|
|
634
|
+
entryLengths[entry] = length;
|
|
635
|
+
entryCounts[entry] = count;
|
|
636
|
+
entryInRuns[entry] = false;
|
|
637
|
+
entriesOutsideRuns.push(entry);
|
|
638
|
+
if (entriesOutsideRuns.length > GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY * entrySpellings.length) {
|
|
639
|
+
dropFreedAndRepeatedPendingEntries();
|
|
640
|
+
}
|
|
641
|
+
byToken.set(spelling, entry);
|
|
642
|
+
liveEntries++;
|
|
643
|
+
}
|
|
644
|
+
function dropFreedAndRepeatedPendingEntries() {
|
|
645
|
+
let kept = 0;
|
|
646
|
+
for (const entry of entriesOutsideRuns) {
|
|
647
|
+
if (entryLengths[entry] === 0 || entryInRuns[entry]) continue;
|
|
648
|
+
entryInRuns[entry] = true;
|
|
649
|
+
entriesOutsideRuns[kept++] = entry;
|
|
650
|
+
}
|
|
651
|
+
entriesOutsideRuns.length = kept;
|
|
652
|
+
for (let at = 0; at < kept; at++) entryInRuns[entriesOutsideRuns[at]] = false;
|
|
653
|
+
}
|
|
654
|
+
function addToRuns(entry) {
|
|
655
|
+
const points = codePointsOf(entrySpellings[entry]);
|
|
656
|
+
for (let at = 0; at + PARTIAL_WORD_RUN_CODE_POINTS <= points.length; at++) {
|
|
657
|
+
const key = packedRunKeyAt(points, at);
|
|
658
|
+
let members = runs.get(key);
|
|
659
|
+
if (members === void 0) {
|
|
660
|
+
members = /* @__PURE__ */ new Set();
|
|
661
|
+
runs.set(key, members);
|
|
662
|
+
}
|
|
663
|
+
const size = members.size;
|
|
664
|
+
members.add(entry);
|
|
665
|
+
if (members.size !== size) runMembers++;
|
|
666
|
+
}
|
|
667
|
+
entryInRuns[entry] = true;
|
|
668
|
+
}
|
|
669
|
+
function addEntriesToRuns() {
|
|
670
|
+
for (const entry of entriesOutsideRuns) {
|
|
671
|
+
if (entryLengths[entry] > 0 && !entryInRuns[entry]) addToRuns(entry);
|
|
672
|
+
}
|
|
673
|
+
entriesOutsideRuns.length = 0;
|
|
674
|
+
}
|
|
675
|
+
function removeFromRuns(entry) {
|
|
676
|
+
const points = codePointsOf(entrySpellings[entry]);
|
|
677
|
+
for (let at = 0; at + PARTIAL_WORD_RUN_CODE_POINTS <= points.length; at++) {
|
|
678
|
+
const key = packedRunKeyAt(points, at);
|
|
679
|
+
const members = runs.get(key);
|
|
680
|
+
if (members === void 0 || !members.delete(entry)) continue;
|
|
681
|
+
runMembers--;
|
|
682
|
+
if (members.size === 0) runs.delete(key);
|
|
683
|
+
}
|
|
684
|
+
entryInRuns[entry] = false;
|
|
685
|
+
}
|
|
686
|
+
function removeEntry(byToken, token, spelling, entry) {
|
|
687
|
+
if (entryInRuns[entry]) removeFromRuns(entry);
|
|
688
|
+
byToken.delete(spelling);
|
|
689
|
+
if (byToken.size === 0) entriesByToken.delete(token);
|
|
690
|
+
entrySpellings[entry] = "";
|
|
691
|
+
entryTokens[entry] = "";
|
|
692
|
+
entryLengths[entry] = 0;
|
|
693
|
+
entryCounts[entry] = 0;
|
|
694
|
+
freeEntries.push(entry);
|
|
695
|
+
liveEntries--;
|
|
696
|
+
}
|
|
697
|
+
function change(token, spelling, delta) {
|
|
698
|
+
let byToken = entriesByToken.get(token);
|
|
699
|
+
const entry = byToken?.get(spelling);
|
|
700
|
+
if (byToken !== void 0 && entry !== void 0) {
|
|
701
|
+
const after = entryCounts[entry] + delta;
|
|
702
|
+
if (after > 0) entryCounts[entry] = after;
|
|
703
|
+
else removeEntry(byToken, token, spelling, entry);
|
|
704
|
+
return;
|
|
705
|
+
}
|
|
706
|
+
if (delta <= 0) return;
|
|
707
|
+
if (byToken === void 0) {
|
|
708
|
+
byToken = /* @__PURE__ */ new Map();
|
|
709
|
+
entriesByToken.set(token, byToken);
|
|
710
|
+
}
|
|
711
|
+
addEntry(byToken, token, spelling, delta);
|
|
712
|
+
if (byToken.size === 0) entriesByToken.delete(token);
|
|
713
|
+
}
|
|
714
|
+
function encodedSpellings() {
|
|
715
|
+
const encoded = { fields: [...fields], tokens: [], spellings: [], counts: [] };
|
|
716
|
+
for (const [token, byToken] of entriesByToken) {
|
|
717
|
+
for (const [spelling, entry] of byToken) {
|
|
718
|
+
encoded.tokens.push(token);
|
|
719
|
+
encoded.spellings.push(spelling);
|
|
720
|
+
encoded.counts.push(entryCounts[entry]);
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
return encoded;
|
|
724
|
+
}
|
|
725
|
+
function applyCounts(source, direction) {
|
|
726
|
+
for (const [token, byToken] of source) {
|
|
727
|
+
for (const [spelling, count] of byToken) change(token, spelling, count * direction);
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
function adoptFields(next) {
|
|
731
|
+
if (sameFields(fields, next)) return;
|
|
732
|
+
fields = [...next];
|
|
733
|
+
resetSpellings();
|
|
734
|
+
everyDocumentPending = fields.length > 0;
|
|
735
|
+
}
|
|
736
|
+
function documentAt(internalId) {
|
|
737
|
+
const docId = view.docStore.getExternalId(internalId);
|
|
738
|
+
return docId === void 0 ? void 0 : view.docStore.get(docId)?.fields;
|
|
739
|
+
}
|
|
740
|
+
function resolvePending(analysis) {
|
|
741
|
+
if (everyDocumentPending) {
|
|
742
|
+
resetSpellings();
|
|
743
|
+
const derived2 = /* @__PURE__ */ new Map();
|
|
744
|
+
for (const [, stored] of view.docStore.all()) countDocumentSpellings(derived2, stored.fields, analysis, 1);
|
|
745
|
+
applyCounts(derived2, 1);
|
|
746
|
+
everyDocumentPending = false;
|
|
747
|
+
return;
|
|
748
|
+
}
|
|
749
|
+
if (pending.size === 0) return;
|
|
750
|
+
const derived = /* @__PURE__ */ new Map();
|
|
751
|
+
for (const internalId of pending) {
|
|
752
|
+
const document = documentAt(internalId);
|
|
753
|
+
if (document !== void 0) countDocumentSpellings(derived, document, analysis, 1);
|
|
754
|
+
}
|
|
755
|
+
pending.clear();
|
|
756
|
+
applyCounts(derived, 1);
|
|
757
|
+
}
|
|
758
|
+
const reader = {
|
|
759
|
+
forEachContaining(probe, visit) {
|
|
760
|
+
const points = probe.codePoints;
|
|
761
|
+
if (points.length < PARTIAL_WORD_RUN_CODE_POINTS) return;
|
|
762
|
+
let narrowest = null;
|
|
763
|
+
for (let at = 0; at + PARTIAL_WORD_RUN_CODE_POINTS <= points.length; at++) {
|
|
764
|
+
const members = runs.get(packedRunKeyAt(points, at));
|
|
765
|
+
if (members === void 0) return;
|
|
766
|
+
if (narrowest === null || members.size < narrowest.size) narrowest = members;
|
|
767
|
+
}
|
|
768
|
+
if (narrowest === null) return;
|
|
769
|
+
for (const entry of narrowest) {
|
|
770
|
+
const length = entryLengths[entry];
|
|
771
|
+
if (length <= points.length) continue;
|
|
772
|
+
if (!entrySpellings[entry].includes(probe.text)) continue;
|
|
773
|
+
visit(entryTokens[entry], length);
|
|
774
|
+
}
|
|
775
|
+
},
|
|
776
|
+
bytes() {
|
|
777
|
+
return slot.bytes();
|
|
778
|
+
}
|
|
779
|
+
};
|
|
780
|
+
const slot = {
|
|
781
|
+
readerFor(analysis) {
|
|
782
|
+
if (analysis.fields.length === 0) return null;
|
|
783
|
+
adoptFields(analysis.fields);
|
|
784
|
+
resolvePending(analysis);
|
|
785
|
+
addEntriesToRuns();
|
|
786
|
+
return reader;
|
|
787
|
+
},
|
|
788
|
+
spellings() {
|
|
789
|
+
if (fields.length === 0 || everyDocumentPending || pending.size > 0) return null;
|
|
790
|
+
return encodedSpellings();
|
|
791
|
+
},
|
|
792
|
+
afterDocument(documentFields, internalId, spellings, direction) {
|
|
793
|
+
if (documentFields.length === 0) return;
|
|
794
|
+
if (!sameFields(fields, documentFields)) {
|
|
795
|
+
const holdsOnlyThisDocument = direction === 1 && view.docStore.count() === 1;
|
|
796
|
+
adoptFields(documentFields);
|
|
797
|
+
if (holdsOnlyThisDocument) everyDocumentPending = false;
|
|
798
|
+
}
|
|
799
|
+
if (everyDocumentPending) return;
|
|
800
|
+
if (direction === -1 && pending.delete(internalId)) return;
|
|
801
|
+
if (spellings !== null) {
|
|
802
|
+
applyCounts(spellings, direction);
|
|
803
|
+
} else if (direction === 1) {
|
|
804
|
+
pending.add(internalId);
|
|
805
|
+
} else {
|
|
806
|
+
everyDocumentPending = true;
|
|
807
|
+
}
|
|
808
|
+
},
|
|
809
|
+
afterMerge(spellings, internalIds) {
|
|
810
|
+
if (internalIds.length === 0) return;
|
|
811
|
+
if (fields.length === 0 && spellings !== null && view.docStore.count() === internalIds.length) {
|
|
812
|
+
adoptFields(spellings.fields);
|
|
813
|
+
everyDocumentPending = false;
|
|
814
|
+
}
|
|
815
|
+
if (fields.length === 0 || everyDocumentPending) return;
|
|
816
|
+
if (spellings === null || !sameFields(fields, spellings.fields)) {
|
|
817
|
+
for (const internalId of internalIds) pending.add(internalId);
|
|
818
|
+
return;
|
|
819
|
+
}
|
|
820
|
+
for (let at = 0; at < spellings.tokens.length; at++) {
|
|
821
|
+
change(spellings.tokens[at], spellings.spellings[at], spellings.counts[at]);
|
|
822
|
+
}
|
|
823
|
+
},
|
|
824
|
+
forgetSpellings() {
|
|
825
|
+
resetSpellings();
|
|
826
|
+
everyDocumentPending = fields.length > 0;
|
|
827
|
+
},
|
|
828
|
+
clear() {
|
|
829
|
+
fields = [];
|
|
830
|
+
everyDocumentPending = false;
|
|
831
|
+
resetSpellings();
|
|
832
|
+
},
|
|
833
|
+
bytes() {
|
|
834
|
+
return liveEntries * GROWING_PARTIAL_ENTRY_BYTES + runs.size * GROWING_PARTIAL_RUN_BYTES + runMembers * GROWING_PARTIAL_RUN_MEMBER_BYTES + entriesOutsideRuns.length * GROWING_PARTIAL_PENDING_ENTRY_BYTES;
|
|
835
|
+
}
|
|
836
|
+
};
|
|
837
|
+
return slot;
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
// src/core/partition/partial-words/highlight.ts
|
|
841
|
+
function containsPartialWord(spelling, words) {
|
|
842
|
+
for (const word of words) {
|
|
843
|
+
if (spelling.length > word.length && spelling.includes(word)) return true;
|
|
844
|
+
}
|
|
845
|
+
return false;
|
|
846
|
+
}
|
|
847
|
+
function partialWordMatcher(text, language, partial) {
|
|
848
|
+
const indexed = tokenize(text, language, {
|
|
849
|
+
stem: true,
|
|
850
|
+
removeStopWords: true,
|
|
851
|
+
collectSurfaces: true,
|
|
852
|
+
customTokenizer: partial.customTokenizer,
|
|
853
|
+
stopWordOverride: partial.stopWordOverride
|
|
854
|
+
});
|
|
855
|
+
const positions = /* @__PURE__ */ new Set();
|
|
856
|
+
const spellings = /* @__PURE__ */ new Set();
|
|
857
|
+
for (let at = 0; at < indexed.tokens.length; at++) {
|
|
858
|
+
const spelling = indexed.surfaces?.[at] ?? indexed.tokens[at].token;
|
|
859
|
+
if (!containsPartialWord(spelling, partial.words)) continue;
|
|
860
|
+
positions.add(indexed.tokens[at].position);
|
|
861
|
+
spellings.add(spelling);
|
|
862
|
+
}
|
|
863
|
+
if (partial.customTokenizer !== void 0) return (_position, fieldToken) => spellings.has(fieldToken);
|
|
864
|
+
return (position) => positions.has(position);
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
// src/core/partition/partial-words/query.ts
|
|
868
|
+
function addPartialWords(group, added) {
|
|
869
|
+
const membersByToken = /* @__PURE__ */ new Map();
|
|
870
|
+
for (const member of group.members) membersByToken.set(member.token, member);
|
|
871
|
+
for (const word of added) {
|
|
872
|
+
const member = membersByToken.get(word.token);
|
|
873
|
+
if (member === void 0) {
|
|
874
|
+
group.members.push({ token: word.token, factor: 0, partialFactor: word.factor });
|
|
875
|
+
} else if (word.factor > member.partialFactor) {
|
|
876
|
+
member.partialFactor = word.factor;
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
function partialWordRequests(query) {
|
|
881
|
+
const eligible = [];
|
|
882
|
+
for (const queryToken of query.queryTokens) {
|
|
883
|
+
const word = query.written.get(queryToken.position);
|
|
884
|
+
if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
|
|
885
|
+
eligible.push({ token: queryToken.token, written: word });
|
|
886
|
+
if (eligible.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
|
|
887
|
+
}
|
|
888
|
+
const count = eligible.length;
|
|
889
|
+
if (count === 0) return [];
|
|
890
|
+
const evenShare = Math.floor(PARTIAL_WORDS_ADDED_PER_QUERY / count);
|
|
891
|
+
const remainder = PARTIAL_WORDS_ADDED_PER_QUERY % count;
|
|
892
|
+
return eligible.map((entry, order) => ({
|
|
893
|
+
token: entry.token,
|
|
894
|
+
request: {
|
|
895
|
+
written: entry.written,
|
|
896
|
+
ownToken: entry.token,
|
|
897
|
+
share: evenShare + (order < remainder ? 1 : 0),
|
|
898
|
+
tolerance: entry.token === query.prefixToken ? 0 : query.tolerance,
|
|
899
|
+
prefixLength: query.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH
|
|
900
|
+
}
|
|
901
|
+
}));
|
|
902
|
+
}
|
|
903
|
+
function addPartialWordMembers(query, expand, groupOf) {
|
|
904
|
+
const requests = partialWordRequests(query);
|
|
905
|
+
const added = expand(
|
|
906
|
+
query.analysis,
|
|
907
|
+
requests.map((entry) => entry.request)
|
|
908
|
+
);
|
|
909
|
+
for (let at = 0; at < requests.length; at++) {
|
|
910
|
+
if (added[at].length === 0) continue;
|
|
911
|
+
const token = requests[at].token;
|
|
912
|
+
addPartialWords(groupOf(token, token !== query.prefixToken), added[at]);
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
function partialWordsToHighlight(queryTokens, writtenWords) {
|
|
916
|
+
const writtenAt = /* @__PURE__ */ new Map();
|
|
917
|
+
for (const word of writtenWords) {
|
|
918
|
+
if (!writtenAt.has(word.position)) writtenAt.set(word.position, word.token);
|
|
919
|
+
}
|
|
920
|
+
const seenTokens = /* @__PURE__ */ new Set();
|
|
921
|
+
const words = [];
|
|
922
|
+
for (const queryToken of queryTokens) {
|
|
923
|
+
if (seenTokens.has(queryToken.token)) continue;
|
|
924
|
+
seenTokens.add(queryToken.token);
|
|
925
|
+
const word = writtenAt.get(queryToken.position);
|
|
926
|
+
if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
|
|
927
|
+
words.push(word);
|
|
928
|
+
if (words.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
|
|
929
|
+
}
|
|
930
|
+
return words;
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
// src/core/partition/partial-words/module.ts
|
|
934
|
+
var partialWordSearchModule = {
|
|
935
|
+
name: "partialWords",
|
|
936
|
+
createGrowingSlot: createGrowingPartialWordSlot,
|
|
937
|
+
createFrozenSlot: createFrozenPartialWordSlot,
|
|
938
|
+
buildTable: buildPartialWordTable,
|
|
939
|
+
expand: expandPartialWords,
|
|
940
|
+
countSpelling,
|
|
941
|
+
encodeSpellings: encodePartialSpellings,
|
|
942
|
+
spellingCountsOf,
|
|
943
|
+
highlightMatcher: partialWordMatcher,
|
|
944
|
+
addTermMembers: addPartialWordMembers,
|
|
945
|
+
highlightWords: partialWordsToHighlight
|
|
946
|
+
};
|
|
947
|
+
|
|
948
|
+
export { expandTermPrefix, getFlatSchema, getNestedValue, getOrCreateFieldNameIndex, parseVectorDimension, partialWordSearchModule, partitionDocumentFrequency, sameFields, suggestDisplayTerms, textFieldsChanged, tokenizeOptions, visitDocumentSpellings, visitDocumentSurfaceSpellings };
|