@delali/narsil 0.3.1-next.86 → 0.3.1-next.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/chunk-3GZQKGVQ.mjs +2719 -0
- package/dist/browser/chunk-3ZRMZIH4.mjs +3959 -0
- package/dist/browser/chunk-FHFFWLEK.mjs +669 -0
- package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
- package/dist/browser/chunk-JACCR3BJ.mjs +602 -0
- package/dist/browser/chunk-NQW2YXEX.mjs +346 -0
- package/dist/browser/chunk-RAHUQOJL.mjs +24103 -0
- package/dist/browser/durability-wiring-VSUBLUDK.mjs +3250 -0
- package/dist/browser/module-E25UDADM.mjs +2766 -0
- package/dist/browser/snapshot-5MXBEDXF.mjs +888 -0
- package/dist/browser/write-6SBYTS62.mjs +206 -0
- package/dist/{capabilities-CAdkp2P-.d.ts → capabilities-SJPttcgQ.d.ts} +2 -2
- package/dist/chunk-37CD3OKS.mjs +87 -0
- package/dist/{chunk-4VBG6PEH.mjs → chunk-A3UB2MXL.mjs} +175 -124
- package/dist/{chunk-OLXM65YH.mjs → chunk-AV4IJKD2.mjs} +1 -18
- package/dist/{chunk-LS5IRM23.mjs → chunk-BVOJIYRV.mjs} +2 -25
- package/dist/chunk-C7SLA6CM.mjs +19 -0
- package/dist/chunk-CL4OM7GB.mjs +522 -0
- package/dist/{chunk-KJ7LB3TR.mjs → chunk-CVQWQVFX.mjs} +3 -22
- package/dist/{chunk-4D6OH4EF.mjs → chunk-DCK4P5BI.mjs} +32 -32
- package/dist/chunk-DJHHF2L5.mjs +17 -0
- package/dist/{chunk-XPK3GB2P.mjs → chunk-EMXAYPTQ.mjs} +1776 -3336
- package/dist/chunk-FGMGUDHX.mjs +21 -0
- package/dist/{chunk-2453RJYK.mjs → chunk-J2BPXTD7.mjs} +167 -176
- package/dist/chunk-L5ORED7V.mjs +25 -0
- package/dist/chunk-MWJIAREY.mjs +614 -0
- package/dist/chunk-MXS5HWIH.mjs +515 -0
- package/dist/chunk-P36MUFEJ.mjs +10 -0
- package/dist/chunk-PR4O7CAP.mjs +18 -0
- package/dist/{chunk-ZCKPRCYI.mjs → chunk-Q5DC6WAB.mjs} +1 -1
- package/dist/chunk-QAMUC2BJ.mjs +1730 -0
- package/dist/{chunk-NFF5TDRG.mjs → chunk-QZN3MIJ3.mjs} +1 -16
- package/dist/chunk-R6BMMQXH.mjs +7316 -0
- package/dist/chunk-R7OSDAFK.mjs +44 -0
- package/dist/chunk-RVM7AODE.mjs +73 -0
- package/dist/chunk-S4LQDSUL.mjs +462 -0
- package/dist/{chunk-SREPGC6O.mjs → chunk-T6CNO72J.mjs} +1 -1
- package/dist/chunk-UKSUENHK.mjs +997 -0
- package/dist/{chunk-B3F7LZXM.mjs → chunk-ULRPNYAG.mjs} +1220 -2318
- package/dist/{chunk-GNYXCLJS.mjs → chunk-URGLSNKR.mjs} +4 -3
- package/dist/chunk-V7YUSKKZ.mjs +703 -0
- package/dist/chunk-VY4DXZ5P.mjs +631 -0
- package/dist/{chunk-XYDP7CCJ.mjs → chunk-X4SEXL4U.mjs} +54 -14
- package/dist/{chunk-RM6FRYEC.mjs → chunk-XSZVS4S3.mjs} +3 -2
- package/dist/chunk-Y2F66IQJ.mjs +74 -0
- package/dist/chunk-YCJLVNWR.mjs +262 -0
- package/dist/{client-CKUtkLz3.d.ts → client-BMK53IR2.d.ts} +3 -3
- package/dist/client.d.ts +8 -8
- package/dist/{config-CSDEPraV.d.ts → config-BUR-7eVQ.d.ts} +2 -2
- package/dist/{config-BuIkeM5C.d.ts → config-UJx1dyX1.d.ts} +2 -2
- package/dist/direct-executor-EGESD26S.mjs +18 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +35 -33
- package/dist/durability-wiring-ZLRQU6FG.mjs +2797 -0
- package/dist/{engine-DMNZJAUq.d.ts → engine-9X6W5naj.d.ts} +2 -2
- package/dist/{index-DzONZUKE.d.ts → index-vCMu8OeQ.d.ts} +1 -1
- package/dist/index.browser.mjs +2 -38128
- package/dist/index.d.ts +23 -9
- package/dist/index.mjs +18 -14
- package/dist/{memory-BPKNx9Tl.d.ts → memory-CV9NAxLY.d.ts} +1 -1
- package/dist/module-WIDYIJLY.mjs +1819 -0
- package/dist/partial-words.mjs +5 -3
- package/dist/pattern.mjs +3 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -13
- package/dist/react.d.ts +7 -7
- package/dist/{schema-DgyDijhf.d.ts → schema-DtsgmNfU.d.ts} +1 -1
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +14 -9
- package/dist/snapshot-22CUEYYW.mjs +469 -0
- package/dist/{tasks-DGyz586z.d.ts → tasks-BGaZJyoY.d.ts} +1 -1
- package/dist/{thread-EUUNJNKU.mjs → thread-ZP3UMIZT.mjs} +12 -9
- package/dist/vector/search-worker.d.ts +30 -3
- package/dist/vector/search-worker.mjs +6 -2
- package/dist/workers/entry.mjs +5 -3
- package/dist/write-PXDKPG2R.mjs +233 -0
- package/package.json +17 -8
- package/dist/chunk-IDLP54FH.mjs +0 -15198
- package/dist/direct-executor-SI2QV4YI.mjs +0 -19
- package/dist/simd-DW5AtJ6B.d.ts +0 -41
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-JACCR3BJ.mjs';
|
|
2
|
+
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-RAHUQOJL.mjs';
|
|
3
|
+
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-NQW2YXEX.mjs';
|
|
4
|
+
import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-FHFFWLEK.mjs';
|
|
5
|
+
import { decode } from '@msgpack/msgpack';
|
|
6
|
+
|
|
7
|
+
// src/engine/export/json-line.ts
|
|
8
|
+
function refuse(path, kind, docId) {
|
|
9
|
+
throw new NarsilError(
|
|
10
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
11
|
+
`Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
|
|
12
|
+
{ docId, field: path }
|
|
13
|
+
);
|
|
14
|
+
}
|
|
15
|
+
function encodeNumber(value, path, target) {
|
|
16
|
+
if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
|
|
17
|
+
if (!target.tagged) refuse(path, String(value), target.docId);
|
|
18
|
+
return `{"${NUMBER_TAG}":"${String(value)}"}`;
|
|
19
|
+
}
|
|
20
|
+
function encodeDate(value, path, target) {
|
|
21
|
+
const milliseconds = value.getTime();
|
|
22
|
+
if (!target.tagged || Number.isNaN(milliseconds)) refuse(path, "a date", target.docId);
|
|
23
|
+
const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
|
|
24
|
+
const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
|
|
25
|
+
return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
|
|
26
|
+
}
|
|
27
|
+
function encodeBytes(value, path, target) {
|
|
28
|
+
if (!target.tagged) refuse(path, "bytes", target.docId);
|
|
29
|
+
return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
|
|
30
|
+
}
|
|
31
|
+
function encodeList(value, path, target) {
|
|
32
|
+
target.parts.push("[");
|
|
33
|
+
for (let at = 0; at < value.length; at++) {
|
|
34
|
+
if (at > 0) target.parts.push(",");
|
|
35
|
+
encodeValue(value[at], `${path}[${at}]`, target);
|
|
36
|
+
}
|
|
37
|
+
target.parts.push("]");
|
|
38
|
+
}
|
|
39
|
+
function encodeMap(value, path, target) {
|
|
40
|
+
const proto = Object.getPrototypeOf(value);
|
|
41
|
+
if (proto !== Object.prototype && proto !== null) refuse(path, "a value other than a plain object", target.docId);
|
|
42
|
+
const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
|
|
43
|
+
const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
|
|
44
|
+
target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
|
|
45
|
+
for (let at = 0; at < entries.length; at++) {
|
|
46
|
+
const [key, entry] = entries[at];
|
|
47
|
+
if (at > 0) target.parts.push(",");
|
|
48
|
+
target.parts.push(JSON.stringify(key), ":");
|
|
49
|
+
encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
|
|
50
|
+
}
|
|
51
|
+
target.parts.push(wrapped ? "}}" : "}");
|
|
52
|
+
}
|
|
53
|
+
function encodeValue(value, path, target) {
|
|
54
|
+
if (value === null) {
|
|
55
|
+
target.parts.push("null");
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
switch (typeof value) {
|
|
59
|
+
case "string":
|
|
60
|
+
target.parts.push(JSON.stringify(value));
|
|
61
|
+
return;
|
|
62
|
+
case "boolean":
|
|
63
|
+
target.parts.push(value ? "true" : "false");
|
|
64
|
+
return;
|
|
65
|
+
case "number":
|
|
66
|
+
target.parts.push(encodeNumber(value, path, target));
|
|
67
|
+
return;
|
|
68
|
+
case "object":
|
|
69
|
+
break;
|
|
70
|
+
default:
|
|
71
|
+
refuse(path, `a ${typeof value}`, target.docId);
|
|
72
|
+
}
|
|
73
|
+
if (Array.isArray(value)) encodeList(value, path, target);
|
|
74
|
+
else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
|
|
75
|
+
else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
|
|
76
|
+
else encodeMap(value, path, target);
|
|
77
|
+
}
|
|
78
|
+
function encodeExactJson(value, docId) {
|
|
79
|
+
const target = { docId, tagged: false, parts: [] };
|
|
80
|
+
encodeValue(value, "", target);
|
|
81
|
+
return target.parts.join("");
|
|
82
|
+
}
|
|
83
|
+
function encodeDocumentJson(document, docId) {
|
|
84
|
+
const target = { docId, tagged: true, parts: [] };
|
|
85
|
+
encodeValue(document, "", target);
|
|
86
|
+
return target.parts.join("");
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// src/engine/export/write.ts
|
|
90
|
+
var VECTOR_FILE_ENVELOPE_VERSION = 2;
|
|
91
|
+
var FLOAT32_BYTES = 4;
|
|
92
|
+
async function locateVectors(checkpoint) {
|
|
93
|
+
const located = /* @__PURE__ */ new Map();
|
|
94
|
+
for (const field of checkpoint.manifest.vectors) {
|
|
95
|
+
const byDocId = /* @__PURE__ */ new Map();
|
|
96
|
+
for (const ref of field.files) {
|
|
97
|
+
const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
|
|
98
|
+
envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
|
|
99
|
+
what: `vector file "${ref.key}"`,
|
|
100
|
+
allowsCompression: false
|
|
101
|
+
});
|
|
102
|
+
const decoded = decodeVectorFilePayload(decode(payload));
|
|
103
|
+
for (let position = 0; position < decoded.docIds.length; position++) {
|
|
104
|
+
if (isDead(ref.dead, position)) continue;
|
|
105
|
+
byDocId.set(decoded.docIds[position], { ref, position, dimension: decoded.dimension });
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
located.set(field.fieldPath, byDocId);
|
|
109
|
+
}
|
|
110
|
+
return located;
|
|
111
|
+
}
|
|
112
|
+
async function readVector(checkpoint, location) {
|
|
113
|
+
const width = location.dimension * FLOAT32_BYTES;
|
|
114
|
+
const offset = location.ref.bytes - (location.ref.count - location.position) * width;
|
|
115
|
+
return Array.from(bytesToVectors((await checkpoint.readRange(location.ref, offset, width)).slice()));
|
|
116
|
+
}
|
|
117
|
+
async function readSegment(checkpoint, ref) {
|
|
118
|
+
const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
|
|
119
|
+
envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
|
|
120
|
+
what: `segment "${ref.key}"`,
|
|
121
|
+
allowsCompression: true
|
|
122
|
+
});
|
|
123
|
+
return decodeKeywordSegment(payload, false);
|
|
124
|
+
}
|
|
125
|
+
var decoder = new TextDecoder();
|
|
126
|
+
function idAt(contents, ordinal) {
|
|
127
|
+
const ids = contents.arrays.idTable;
|
|
128
|
+
return decoder.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
|
|
129
|
+
}
|
|
130
|
+
async function winnersOf(checkpoint, partition) {
|
|
131
|
+
const winners = /* @__PURE__ */ new Map();
|
|
132
|
+
for (let at = 0; at < partition.segments.length; at++) {
|
|
133
|
+
const contents = await readSegment(checkpoint, partition.segments[at]);
|
|
134
|
+
for (const docId of contents.tombstones) winners.delete(docId);
|
|
135
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) winners.set(idAt(contents, ordinal), at);
|
|
136
|
+
}
|
|
137
|
+
return winners;
|
|
138
|
+
}
|
|
139
|
+
async function* documentLines(checkpoint) {
|
|
140
|
+
const vectors = await locateVectors(checkpoint);
|
|
141
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
142
|
+
const winners = await winnersOf(checkpoint, partition);
|
|
143
|
+
for (let at = 0; at < partition.segments.length; at++) {
|
|
144
|
+
const contents = await readSegment(checkpoint, partition.segments[at]);
|
|
145
|
+
const table = contents.arrays.documentTable;
|
|
146
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
|
|
147
|
+
const docId = idAt(contents, ordinal);
|
|
148
|
+
if (winners.get(docId) !== at) continue;
|
|
149
|
+
const stored = decodeStoredDocument(
|
|
150
|
+
table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
|
|
151
|
+
table.discardedFields
|
|
152
|
+
);
|
|
153
|
+
const document = { id: docId, ...stored };
|
|
154
|
+
document.id = docId;
|
|
155
|
+
for (const [fieldPath, located] of vectors) {
|
|
156
|
+
const location = located.get(docId);
|
|
157
|
+
if (location !== void 0) setNestedValue(document, fieldPath, await readVector(checkpoint, location));
|
|
158
|
+
}
|
|
159
|
+
yield `${encodeDocumentJson(document, docId)}
|
|
160
|
+
`;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
async function* exportLines(header, checkpoint) {
|
|
166
|
+
yield `${encodeExactJson({ format: DOCUMENT_EXPORT_FORMAT, version: DOCUMENT_EXPORT_VERSION, index: header }, "")}
|
|
167
|
+
`;
|
|
168
|
+
yield* documentLines(checkpoint);
|
|
169
|
+
}
|
|
170
|
+
function documentExportStream(header, checkpoint) {
|
|
171
|
+
const lines = exportLines(header, checkpoint);
|
|
172
|
+
const encoder = new TextEncoder();
|
|
173
|
+
let released = false;
|
|
174
|
+
const release = () => {
|
|
175
|
+
if (released) return;
|
|
176
|
+
released = true;
|
|
177
|
+
checkpoint.release();
|
|
178
|
+
};
|
|
179
|
+
return new ReadableStream({
|
|
180
|
+
async pull(controller) {
|
|
181
|
+
try {
|
|
182
|
+
let text = "";
|
|
183
|
+
while (text.length < EXPORT_CHUNK_BYTES) {
|
|
184
|
+
const step = await lines.next();
|
|
185
|
+
if (step.done) {
|
|
186
|
+
if (text.length > 0) controller.enqueue(encoder.encode(text));
|
|
187
|
+
release();
|
|
188
|
+
controller.close();
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
text += step.value;
|
|
192
|
+
}
|
|
193
|
+
controller.enqueue(encoder.encode(text));
|
|
194
|
+
} catch (error) {
|
|
195
|
+
release();
|
|
196
|
+
controller.error(error);
|
|
197
|
+
}
|
|
198
|
+
},
|
|
199
|
+
async cancel() {
|
|
200
|
+
release();
|
|
201
|
+
await lines.return(void 0);
|
|
202
|
+
}
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
export { documentExportStream };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
3
|
-
import { h as TaskStore } from './tasks-
|
|
2
|
+
import { C as ClusterNamespace } from './config-UJx1dyX1.js';
|
|
3
|
+
import { h as TaskStore } from './tasks-BGaZJyoY.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Context handed to {@link OnRequestHook} for every inbound request, captured
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
// src/vector/osq/record.ts
|
|
2
|
+
var OSQ_RECORD_TRAILER_BYTES = 16;
|
|
3
|
+
var TRAILER_LOWER = 0;
|
|
4
|
+
var TRAILER_UPPER = 4;
|
|
5
|
+
var TRAILER_CORRECTION = 8;
|
|
6
|
+
var TRAILER_SUM = 12;
|
|
7
|
+
var QUERY_PLANES_FOR_NARROW_CODES = 4;
|
|
8
|
+
var LEVELS_IN_A_TWO_BIT_BYTE = 4;
|
|
9
|
+
function osqCodeBytes(dimension, bits) {
|
|
10
|
+
return Math.ceil(dimension * bits / 8);
|
|
11
|
+
}
|
|
12
|
+
function osqRecordBytes(dimension, bits) {
|
|
13
|
+
return osqCodeBytes(dimension, bits) + OSQ_RECORD_TRAILER_BYTES;
|
|
14
|
+
}
|
|
15
|
+
function packLevels(levels, bits, target, offset) {
|
|
16
|
+
const dimension = levels.length;
|
|
17
|
+
if (bits === 8) {
|
|
18
|
+
target.set(levels, offset);
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
const perByte = 8 / bits;
|
|
22
|
+
target.fill(0, offset, offset + osqCodeBytes(dimension, bits));
|
|
23
|
+
for (let i = 0; i < dimension; i++) {
|
|
24
|
+
const value = levels[i];
|
|
25
|
+
if (value === 0) continue;
|
|
26
|
+
target[offset + Math.floor(i / perByte)] |= value << i % perByte * bits;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
function unpackLevels(bytes, offset, dimension, bits) {
|
|
30
|
+
const levels = new Uint8Array(dimension);
|
|
31
|
+
if (bits === 8) {
|
|
32
|
+
levels.set(bytes.subarray(offset, offset + dimension));
|
|
33
|
+
return levels;
|
|
34
|
+
}
|
|
35
|
+
const perByte = 8 / bits;
|
|
36
|
+
const mask = (1 << bits) - 1;
|
|
37
|
+
for (let i = 0; i < dimension; i++) {
|
|
38
|
+
levels[i] = bytes[offset + Math.floor(i / perByte)] >> i % perByte * bits & mask;
|
|
39
|
+
}
|
|
40
|
+
return levels;
|
|
41
|
+
}
|
|
42
|
+
function osqStagedQueryBytes(dimension, documentBits) {
|
|
43
|
+
if (documentBits === 8) return dimension;
|
|
44
|
+
if (documentBits === 4) return osqCodeBytes(dimension, 4);
|
|
45
|
+
return QUERY_PLANES_FOR_NARROW_CODES * osqCodeBytes(dimension, documentBits);
|
|
46
|
+
}
|
|
47
|
+
function stageQueryLevels(levels, documentBits, target) {
|
|
48
|
+
if (documentBits === 8 || documentBits === 4) {
|
|
49
|
+
packLevels(levels, documentBits, target, 0);
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
const planeBytes = osqCodeBytes(levels.length, documentBits);
|
|
53
|
+
target.fill(0, 0, QUERY_PLANES_FOR_NARROW_CODES * planeBytes);
|
|
54
|
+
if (documentBits === 2) {
|
|
55
|
+
for (let i = 0; i < levels.length; i++) {
|
|
56
|
+
target[i % LEVELS_IN_A_TWO_BIT_BYTE * planeBytes + Math.floor(i / LEVELS_IN_A_TWO_BIT_BYTE)] = levels[i];
|
|
57
|
+
}
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
for (let i = 0; i < levels.length; i++) {
|
|
61
|
+
const value = levels[i];
|
|
62
|
+
if (value === 0) continue;
|
|
63
|
+
const byte = i >> 3;
|
|
64
|
+
const bit = 1 << (i & 7);
|
|
65
|
+
for (let plane = 0; plane < QUERY_PLANES_FOR_NARROW_CODES; plane++) {
|
|
66
|
+
if (value >> plane & 1) target[plane * planeBytes + byte] |= bit;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
function writeRecord(target, offset, code, bits, trailer = new DataView(target.buffer, target.byteOffset, target.byteLength)) {
|
|
71
|
+
packLevels(code.levels, bits, target, offset);
|
|
72
|
+
const at = offset + osqCodeBytes(code.levels.length, bits);
|
|
73
|
+
trailer.setFloat32(at + TRAILER_LOWER, code.lower, true);
|
|
74
|
+
trailer.setFloat32(at + TRAILER_UPPER, code.upper, true);
|
|
75
|
+
trailer.setFloat32(at + TRAILER_CORRECTION, code.correction, true);
|
|
76
|
+
trailer.setUint32(at + TRAILER_SUM, code.sum, true);
|
|
77
|
+
}
|
|
78
|
+
function readTrailer(trailer, codeOffset, codeBytes, into) {
|
|
79
|
+
const at = codeOffset + codeBytes;
|
|
80
|
+
into.lower = trailer.getFloat32(at + TRAILER_LOWER, true);
|
|
81
|
+
into.upper = trailer.getFloat32(at + TRAILER_UPPER, true);
|
|
82
|
+
into.correction = trailer.getFloat32(at + TRAILER_CORRECTION, true);
|
|
83
|
+
into.sum = trailer.getUint32(at + TRAILER_SUM, true);
|
|
84
|
+
return into;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export { osqCodeBytes, osqRecordBytes, osqStagedQueryBytes, readTrailer, stageQueryLevels, unpackLevels, writeRecord };
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { flattenSchema
|
|
1
|
+
import { flattenSchema } from './chunk-CVQWQVFX.mjs';
|
|
2
|
+
import { PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, DEFAULT_FUZZY_PREFIX_LENGTH } from './chunk-FGMGUDHX.mjs';
|
|
2
3
|
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
3
|
-
import { isWordIndexedBase } from './chunk-
|
|
4
|
+
import { isWordIndexedBase } from './chunk-BVOJIYRV.mjs';
|
|
4
5
|
import { compareCodePoints, codePointCount } from './chunk-YTJCGKM4.mjs';
|
|
5
6
|
import { createNarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
6
7
|
|
|
@@ -347,97 +348,6 @@ function expandPartialWords(subs, analysis, requests) {
|
|
|
347
348
|
return requests.map((request) => readers.length === 0 ? [] : addedWordsFor(subs, readers, request));
|
|
348
349
|
}
|
|
349
350
|
|
|
350
|
-
// src/core/partition/partial-words/constants.ts
|
|
351
|
-
var PARTIAL_WORD_RUN_CODE_POINTS = 3;
|
|
352
|
-
var GROWING_PARTIAL_ENTRY_BYTES = 104;
|
|
353
|
-
var GROWING_PARTIAL_PENDING_ENTRY_BYTES = 8;
|
|
354
|
-
var GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY = 2;
|
|
355
|
-
var GROWING_PARTIAL_RUN_BYTES = 120;
|
|
356
|
-
var GROWING_PARTIAL_RUN_MEMBER_BYTES = 24;
|
|
357
|
-
var FROZEN_PARTIAL_REMOVED_ENTRY_BYTES = 48;
|
|
358
|
-
|
|
359
|
-
// src/core/partition/constants.ts
|
|
360
|
-
var TARGET_BLOCK_ENTRIES = 128;
|
|
361
|
-
var PRUNING_REJECTION_SLACK = 1e-12;
|
|
362
|
-
var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
|
|
363
|
-
var SEGMENT_TOKEN_TABLE_START = 1 << 12;
|
|
364
|
-
var SEGMENT_POSTING_LOG_START = 1 << 14;
|
|
365
|
-
var SEGMENT_POSITION_POOL_START = 1 << 16;
|
|
366
|
-
var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
|
|
367
|
-
var LOOKUP_KEY_SCRATCH_BYTES = 4096;
|
|
368
|
-
var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
|
|
369
|
-
var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
|
|
370
|
-
|
|
371
|
-
// src/core/partition/frozen/string-blob.ts
|
|
372
|
-
var SURROGATE = /[\ud800-\udfff]/;
|
|
373
|
-
var UTF8_BYTES_PER_UTF16_UNIT = 3;
|
|
374
|
-
var encoder2 = new TextEncoder();
|
|
375
|
-
var lookupKeyBytes = new Uint8Array(LOOKUP_KEY_SCRATCH_BYTES);
|
|
376
|
-
function encodeLookupKey(text) {
|
|
377
|
-
if (text.length * UTF8_BYTES_PER_UTF16_UNIT > LOOKUP_KEY_SCRATCH_BYTES) return encoder2.encode(text);
|
|
378
|
-
const { written } = encoder2.encodeInto(text, lookupKeyBytes);
|
|
379
|
-
return lookupKeyBytes.subarray(0, written);
|
|
380
|
-
}
|
|
381
|
-
function encodeOneByOne(strings) {
|
|
382
|
-
const encoded = new Array(strings.length);
|
|
383
|
-
let blobLength = 0;
|
|
384
|
-
for (let i = 0; i < strings.length; i++) {
|
|
385
|
-
encoded[i] = encoder2.encode(strings[i]);
|
|
386
|
-
blobLength += encoded[i].length;
|
|
387
|
-
}
|
|
388
|
-
const blob = new Uint8Array(blobLength);
|
|
389
|
-
const offsets = new Uint32Array(strings.length + 1);
|
|
390
|
-
let cursor = 0;
|
|
391
|
-
for (let i = 0; i < strings.length; i++) {
|
|
392
|
-
blob.set(encoded[i], cursor);
|
|
393
|
-
cursor += encoded[i].length;
|
|
394
|
-
offsets[i + 1] = cursor;
|
|
395
|
-
}
|
|
396
|
-
return { blob, offsets };
|
|
397
|
-
}
|
|
398
|
-
function basicPlaneUtf8Bytes(text) {
|
|
399
|
-
let bytes = text.length;
|
|
400
|
-
for (let i = 0; i < text.length; i++) {
|
|
401
|
-
const unit = text.charCodeAt(i);
|
|
402
|
-
if (unit >= 128) bytes += unit < 2048 ? 1 : 2;
|
|
403
|
-
}
|
|
404
|
-
return bytes;
|
|
405
|
-
}
|
|
406
|
-
function encodeStringBlob(strings) {
|
|
407
|
-
const joined = strings.join("");
|
|
408
|
-
if (SURROGATE.test(joined)) return encodeOneByOne(strings);
|
|
409
|
-
const blob = encoder2.encode(joined);
|
|
410
|
-
const offsets = new Uint32Array(strings.length + 1);
|
|
411
|
-
let cursor = 0;
|
|
412
|
-
for (let i = 0; i < strings.length; i++) {
|
|
413
|
-
cursor += basicPlaneUtf8Bytes(strings[i]);
|
|
414
|
-
offsets[i + 1] = cursor;
|
|
415
|
-
}
|
|
416
|
-
return cursor === blob.length ? { blob, offsets } : encodeOneByOne(strings);
|
|
417
|
-
}
|
|
418
|
-
function orderByComparator(strings) {
|
|
419
|
-
const order = new Array(strings.length);
|
|
420
|
-
for (let i = 0; i < strings.length; i++) order[i] = i;
|
|
421
|
-
return order.sort((a, b) => compareCodePoints(strings[a], strings[b]));
|
|
422
|
-
}
|
|
423
|
-
function alreadyInCodePointOrder(strings) {
|
|
424
|
-
for (let i = 1; i < strings.length; i++) {
|
|
425
|
-
if (compareCodePoints(strings[i - 1], strings[i]) > 0) return false;
|
|
426
|
-
}
|
|
427
|
-
return true;
|
|
428
|
-
}
|
|
429
|
-
function codePointOrder(strings) {
|
|
430
|
-
if (alreadyInCodePointOrder(strings)) return strings.map((_, i) => i);
|
|
431
|
-
if (SURROGATE.test(strings.join(""))) return orderByComparator(strings);
|
|
432
|
-
const positions = /* @__PURE__ */ new Map();
|
|
433
|
-
for (let i = 0; i < strings.length; i++) positions.set(strings[i], i);
|
|
434
|
-
if (positions.size !== strings.length) return orderByComparator(strings);
|
|
435
|
-
const order = new Array(strings.length);
|
|
436
|
-
const sorted = [...strings].sort();
|
|
437
|
-
for (let i = 0; i < sorted.length; i++) order[i] = positions.get(sorted[i]) ?? 0;
|
|
438
|
-
return order;
|
|
439
|
-
}
|
|
440
|
-
|
|
441
351
|
// src/core/tokenizer/ngrams.ts
|
|
442
352
|
var OTHER_SCRIPT = 0;
|
|
443
353
|
var HAN_SCRIPT = 1;
|
|
@@ -641,6 +551,17 @@ function visitTokens(text, language, options, visit) {
|
|
|
641
551
|
visit(processed, position, surface === processed ? void 0 : surface, part);
|
|
642
552
|
}
|
|
643
553
|
}
|
|
554
|
+
function visitSurfaceSpellings(text, language, visit) {
|
|
555
|
+
const rawParts = splitText(normalizeForSplitting(text), language);
|
|
556
|
+
const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
|
|
557
|
+
const removeDiacritics = language.tokenizer?.normalizeDiacritics ?? false;
|
|
558
|
+
const stripPossessives = language.tokenizer?.stripPossessive ?? false;
|
|
559
|
+
for (const part of rawParts) {
|
|
560
|
+
const candidate = stripPossessives ? stripPossessive(part) : part;
|
|
561
|
+
if (candidate.length < minLength) continue;
|
|
562
|
+
visit(removeDiacritics ? transformToken(candidate, language, false, true) : candidate);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
644
565
|
function tokenize(text, language, options) {
|
|
645
566
|
if (options?.customTokenizer) {
|
|
646
567
|
const customResult = options.customTokenizer.tokenize(text);
|
|
@@ -657,6 +578,98 @@ function tokenize(text, language, options) {
|
|
|
657
578
|
return { tokens, originalTokens, surfaces };
|
|
658
579
|
}
|
|
659
580
|
|
|
581
|
+
// src/core/partition/partial-words/constants.ts
|
|
582
|
+
var PARTIAL_WORD_RUN_CODE_POINTS = 3;
|
|
583
|
+
var GROWING_PARTIAL_ENTRY_BYTES = 104;
|
|
584
|
+
var GROWING_PARTIAL_PENDING_ENTRY_BYTES = 8;
|
|
585
|
+
var GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY = 2;
|
|
586
|
+
var GROWING_PARTIAL_RUN_BYTES = 120;
|
|
587
|
+
var GROWING_PARTIAL_RUN_MEMBER_BYTES = 24;
|
|
588
|
+
var FROZEN_PARTIAL_REMOVED_ENTRY_BYTES = 48;
|
|
589
|
+
|
|
590
|
+
// src/core/partition/constants.ts
|
|
591
|
+
var TARGET_BLOCK_ENTRIES = 128;
|
|
592
|
+
var PRUNING_REJECTION_SLACK = 1e-12;
|
|
593
|
+
var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
|
|
594
|
+
var SEGMENT_TOKEN_TABLE_START = 1 << 12;
|
|
595
|
+
var SEGMENT_POSTING_LOG_START = 1 << 14;
|
|
596
|
+
var SEGMENT_POSITION_POOL_START = 1 << 16;
|
|
597
|
+
var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
|
|
598
|
+
var LOOKUP_KEY_SCRATCH_BYTES = 4096;
|
|
599
|
+
var FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION = 0.9;
|
|
600
|
+
var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
|
|
601
|
+
var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
|
|
602
|
+
|
|
603
|
+
// src/core/partition/frozen/string-blob.ts
|
|
604
|
+
var SURROGATE = /[\ud800-\udfff]/;
|
|
605
|
+
var UTF8_BYTES_PER_UTF16_UNIT = 3;
|
|
606
|
+
var encoder2 = new TextEncoder();
|
|
607
|
+
var lookupKeyBytes = new Uint8Array(LOOKUP_KEY_SCRATCH_BYTES);
|
|
608
|
+
function encodeLookupKey(text) {
|
|
609
|
+
if (text.length * UTF8_BYTES_PER_UTF16_UNIT > LOOKUP_KEY_SCRATCH_BYTES) return encoder2.encode(text);
|
|
610
|
+
const { written } = encoder2.encodeInto(text, lookupKeyBytes);
|
|
611
|
+
return lookupKeyBytes.subarray(0, written);
|
|
612
|
+
}
|
|
613
|
+
function encodeOneByOne(strings) {
|
|
614
|
+
const encoded = new Array(strings.length);
|
|
615
|
+
let blobLength = 0;
|
|
616
|
+
for (let i = 0; i < strings.length; i++) {
|
|
617
|
+
encoded[i] = encoder2.encode(strings[i]);
|
|
618
|
+
blobLength += encoded[i].length;
|
|
619
|
+
}
|
|
620
|
+
const blob = new Uint8Array(blobLength);
|
|
621
|
+
const offsets = new Uint32Array(strings.length + 1);
|
|
622
|
+
let cursor = 0;
|
|
623
|
+
for (let i = 0; i < strings.length; i++) {
|
|
624
|
+
blob.set(encoded[i], cursor);
|
|
625
|
+
cursor += encoded[i].length;
|
|
626
|
+
offsets[i + 1] = cursor;
|
|
627
|
+
}
|
|
628
|
+
return { blob, offsets };
|
|
629
|
+
}
|
|
630
|
+
function basicPlaneUtf8Bytes(text) {
|
|
631
|
+
let bytes = text.length;
|
|
632
|
+
for (let i = 0; i < text.length; i++) {
|
|
633
|
+
const unit = text.charCodeAt(i);
|
|
634
|
+
if (unit >= 128) bytes += unit < 2048 ? 1 : 2;
|
|
635
|
+
}
|
|
636
|
+
return bytes;
|
|
637
|
+
}
|
|
638
|
+
function encodeStringBlob(strings) {
|
|
639
|
+
const joined = strings.join("");
|
|
640
|
+
if (SURROGATE.test(joined)) return encodeOneByOne(strings);
|
|
641
|
+
const blob = encoder2.encode(joined);
|
|
642
|
+
const offsets = new Uint32Array(strings.length + 1);
|
|
643
|
+
let cursor = 0;
|
|
644
|
+
for (let i = 0; i < strings.length; i++) {
|
|
645
|
+
cursor += basicPlaneUtf8Bytes(strings[i]);
|
|
646
|
+
offsets[i + 1] = cursor;
|
|
647
|
+
}
|
|
648
|
+
return cursor === blob.length ? { blob, offsets } : encodeOneByOne(strings);
|
|
649
|
+
}
|
|
650
|
+
function orderByComparator(strings) {
|
|
651
|
+
const order = new Array(strings.length);
|
|
652
|
+
for (let i = 0; i < strings.length; i++) order[i] = i;
|
|
653
|
+
return order.sort((a, b) => compareCodePoints(strings[a], strings[b]));
|
|
654
|
+
}
|
|
655
|
+
function alreadyInCodePointOrder(strings) {
|
|
656
|
+
for (let i = 1; i < strings.length; i++) {
|
|
657
|
+
if (compareCodePoints(strings[i - 1], strings[i]) > 0) return false;
|
|
658
|
+
}
|
|
659
|
+
return true;
|
|
660
|
+
}
|
|
661
|
+
function codePointOrder(strings) {
|
|
662
|
+
if (alreadyInCodePointOrder(strings)) return strings.map((_, i) => i);
|
|
663
|
+
if (SURROGATE.test(strings.join(""))) return orderByComparator(strings);
|
|
664
|
+
const positions = /* @__PURE__ */ new Map();
|
|
665
|
+
for (let i = 0; i < strings.length; i++) positions.set(strings[i], i);
|
|
666
|
+
if (positions.size !== strings.length) return orderByComparator(strings);
|
|
667
|
+
const order = new Array(strings.length);
|
|
668
|
+
const sorted = [...strings].sort();
|
|
669
|
+
for (let i = 0; i < sorted.length; i++) order[i] = positions.get(sorted[i]) ?? 0;
|
|
670
|
+
return order;
|
|
671
|
+
}
|
|
672
|
+
|
|
660
673
|
// src/core/partition/utils.ts
|
|
661
674
|
function getOrCreateFieldNameIndex(table, fieldName) {
|
|
662
675
|
const existing = table.indexMap.get(fieldName);
|
|
@@ -718,16 +731,6 @@ function getFlatSchema(state, schema) {
|
|
|
718
731
|
state.lastSchemaRef = schema;
|
|
719
732
|
return state.flatSchemaCache;
|
|
720
733
|
}
|
|
721
|
-
function getFieldValueForDoc(docStore, docId, fieldPath) {
|
|
722
|
-
const stored = docStore.get(docId);
|
|
723
|
-
if (!stored) return void 0;
|
|
724
|
-
return getNestedValue(stored.fields, fieldPath);
|
|
725
|
-
}
|
|
726
|
-
function getFieldValueByInternalId(docStore, internalId, fieldPath) {
|
|
727
|
-
const externalId = docStore.getExternalId(internalId);
|
|
728
|
-
if (externalId === void 0) return void 0;
|
|
729
|
-
return getFieldValueForDoc(docStore, externalId, fieldPath);
|
|
730
|
-
}
|
|
731
734
|
|
|
732
735
|
// src/core/partition/partial-words/spellings.ts
|
|
733
736
|
var LAST_BASIC_PLANE_CODE_POINT2 = 65535;
|
|
@@ -764,16 +767,24 @@ function textsOf(value) {
|
|
|
764
767
|
if (!Array.isArray(value)) return [];
|
|
765
768
|
return value.filter((element) => typeof element === "string");
|
|
766
769
|
}
|
|
767
|
-
function
|
|
770
|
+
function visitFieldTexts(document, fields, visit) {
|
|
771
|
+
for (const field of fields) {
|
|
772
|
+
for (const text of textsOf(getNestedValue(document, field))) visit(text);
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
function visitDocumentSpellings(document, analysis, visit) {
|
|
768
776
|
const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
|
|
769
|
-
const
|
|
770
|
-
|
|
777
|
+
const visitOccurrence = (token, _position, surface) => {
|
|
778
|
+
visit(token, surface ?? token);
|
|
771
779
|
};
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
780
|
+
visitFieldTexts(document, analysis.fields, (text) => visitTokens(text, analysis.language, tokenizing, visitOccurrence));
|
|
781
|
+
}
|
|
782
|
+
function visitDocumentSurfaceSpellings(document, analysis, visit) {
|
|
783
|
+
if (analysis.options?.customTokenizer !== void 0) return;
|
|
784
|
+
visitFieldTexts(document, analysis.fields, (text) => visitSurfaceSpellings(text, analysis.language, visit));
|
|
785
|
+
}
|
|
786
|
+
function countDocumentSpellings(counts, document, analysis, direction) {
|
|
787
|
+
visitDocumentSpellings(document, analysis, (token, spelling) => countSpelling(counts, token, spelling, direction));
|
|
777
788
|
}
|
|
778
789
|
function spellingCountsOf(documents, analysis) {
|
|
779
790
|
const counts = /* @__PURE__ */ new Map();
|
|
@@ -921,21 +932,21 @@ function readPartialWordTable(table, tokenTable, removedCounts) {
|
|
|
921
932
|
visit(tokenTable.tokenAt(spellingTokens[entry]), length);
|
|
922
933
|
}
|
|
923
934
|
},
|
|
924
|
-
entryOf(spelling
|
|
925
|
-
const tokenIndex = tokenTable.find(token);
|
|
926
|
-
if (tokenIndex < 0) return -1;
|
|
935
|
+
entryOf(spelling) {
|
|
927
936
|
const probe = partialWordProbe(spelling);
|
|
928
937
|
if (probe.codePoints.length <= PARTIAL_WORD_RUN_CODE_POINTS) return -1;
|
|
929
938
|
const run = narrowestRun(probe.codePoints);
|
|
930
939
|
if (run < 0) return -1;
|
|
940
|
+
let exhausted = -1;
|
|
931
941
|
for (let member = runOffsets[run]; member < runOffsets[run + 1]; member++) {
|
|
932
942
|
const entry = runEntries[member];
|
|
933
|
-
if (spellingTokens[entry] !== tokenIndex) continue;
|
|
934
943
|
const start = spellingOffsets[entry];
|
|
935
944
|
const end = spellingOffsets[entry + 1];
|
|
936
|
-
if (end - start
|
|
945
|
+
if (end - start !== probe.utf8.length || !utf8Contains(spellings, start, end, probe.utf8)) continue;
|
|
946
|
+
if ((removedCounts.get(entry) ?? 0) < spellingCounts[entry]) return entry;
|
|
947
|
+
if (exhausted < 0) exhausted = entry;
|
|
937
948
|
}
|
|
938
|
-
return
|
|
949
|
+
return exhausted;
|
|
939
950
|
},
|
|
940
951
|
bytes() {
|
|
941
952
|
return partialWordTableBytes(table);
|
|
@@ -958,6 +969,38 @@ function addCounts(target, source) {
|
|
|
958
969
|
for (const [spelling, count] of bySpelling) countSpelling(target, token, spelling, count);
|
|
959
970
|
}
|
|
960
971
|
}
|
|
972
|
+
function countRemovedSpellings(removed, document, analysis) {
|
|
973
|
+
const count = (spelling) => {
|
|
974
|
+
removed.set(spelling, (removed.get(spelling) ?? 0) + 1);
|
|
975
|
+
};
|
|
976
|
+
if (analysis.options?.customTokenizer !== void 0) {
|
|
977
|
+
visitDocumentSpellings(document, analysis, (_token, spelling) => count(spelling));
|
|
978
|
+
return;
|
|
979
|
+
}
|
|
980
|
+
visitDocumentSurfaceSpellings(document, analysis, count);
|
|
981
|
+
}
|
|
982
|
+
function unstemmedSpellingsOf(analysis) {
|
|
983
|
+
const { language } = analysis;
|
|
984
|
+
if (language.normalizer === void 0 || language.tokenizer?.normalizeDiacritics === true) return () => null;
|
|
985
|
+
const stopWords = resolveStopWords(language, analysis.options?.stopWordOverride);
|
|
986
|
+
return (spelling) => {
|
|
987
|
+
if (stopWords.has(spelling)) return null;
|
|
988
|
+
const normalized = transformToken(spelling, language, false, false);
|
|
989
|
+
return normalized === spelling ? null : normalized;
|
|
990
|
+
};
|
|
991
|
+
}
|
|
992
|
+
function subtractRemovedSpellings(counts, removed, analysis) {
|
|
993
|
+
const tokenOf = /* @__PURE__ */ new Map();
|
|
994
|
+
for (const [token, bySpelling] of counts) {
|
|
995
|
+
for (const spelling of bySpelling.keys()) tokenOf.set(spelling, token);
|
|
996
|
+
}
|
|
997
|
+
const unstemmedSpelling = unstemmedSpellingsOf(analysis);
|
|
998
|
+
for (const [spelling, count] of removed) {
|
|
999
|
+
const stored = tokenOf.has(spelling) ? spelling : unstemmedSpelling(spelling) ?? spelling;
|
|
1000
|
+
const token = tokenOf.get(stored);
|
|
1001
|
+
if (token !== void 0) countSpelling(counts, token, stored, -count);
|
|
1002
|
+
}
|
|
1003
|
+
}
|
|
961
1004
|
function createFrozenPartialWordSlot(initial, tokenTable, documents) {
|
|
962
1005
|
let table = initial;
|
|
963
1006
|
let uncounted = null;
|
|
@@ -970,13 +1013,16 @@ function createFrozenPartialWordSlot(initial, tokenTable, documents) {
|
|
|
970
1013
|
function subtractPendingRemovals(current, analysis) {
|
|
971
1014
|
if (pendingRemovals.length === 0) return;
|
|
972
1015
|
const removed = /* @__PURE__ */ new Map();
|
|
973
|
-
for (const ordinal of pendingRemovals)
|
|
1016
|
+
for (const ordinal of pendingRemovals) countRemovedSpellings(removed, documents.documentAt(ordinal), analysis);
|
|
974
1017
|
pendingRemovals = [];
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
1018
|
+
const unstemmedSpelling = unstemmedSpellingsOf(analysis);
|
|
1019
|
+
for (const [spelling, count] of removed) {
|
|
1020
|
+
let entry = current.entryOf(spelling);
|
|
1021
|
+
if (entry < 0) {
|
|
1022
|
+
const fallback = unstemmedSpelling(spelling);
|
|
1023
|
+
if (fallback !== null) entry = current.entryOf(fallback);
|
|
979
1024
|
}
|
|
1025
|
+
if (entry >= 0) removedCounts.set(entry, (removedCounts.get(entry) ?? 0) + count);
|
|
980
1026
|
}
|
|
981
1027
|
}
|
|
982
1028
|
function rebuild(analysis) {
|
|
@@ -1030,11 +1076,16 @@ function createFrozenPartialWordSlot(initial, tokenTable, documents) {
|
|
|
1030
1076
|
}
|
|
1031
1077
|
const corrected = /* @__PURE__ */ new Map();
|
|
1032
1078
|
countTableSpellings(corrected, table, tokenTable);
|
|
1079
|
+
const removed = /* @__PURE__ */ new Map();
|
|
1080
|
+
const returned = [];
|
|
1033
1081
|
for (let ordinal = 0; ordinal < mergedOrdinals.length; ordinal++) {
|
|
1034
1082
|
const kept = mergedOrdinals[ordinal] >= 0;
|
|
1035
1083
|
if (kept === isCounted(ordinal)) continue;
|
|
1036
|
-
|
|
1084
|
+
if (kept) returned.push(ordinal);
|
|
1085
|
+
else countRemovedSpellings(removed, documents.documentAt(ordinal), analysis);
|
|
1037
1086
|
}
|
|
1087
|
+
subtractRemovedSpellings(corrected, removed, analysis);
|
|
1088
|
+
for (const ordinal of returned) countDocumentSpellings(corrected, documents.documentAt(ordinal), analysis, 1);
|
|
1038
1089
|
addCounts(counts, corrected);
|
|
1039
1090
|
},
|
|
1040
1091
|
spellings() {
|
|
@@ -1397,4 +1448,4 @@ var partialWordSearchModule = {
|
|
|
1397
1448
|
highlightWords: partialWordsToHighlight
|
|
1398
1449
|
};
|
|
1399
1450
|
|
|
1400
|
-
export { PRUNING_REJECTION_SLACK, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, SEGMENT_FIELD_SCRATCH_START, SEGMENT_POSITION_POOL_START, SEGMENT_POSTING_LOG_START, SEGMENT_TOKEN_TABLE_START, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT, TARGET_BLOCK_ENTRIES, clearNormalizationCache, codePointOrder, configureNormalizationCache, encodeLookupKey, encodeStringBlob, expandTermPrefix,
|
|
1451
|
+
export { FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION, PRUNING_REJECTION_SLACK, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, SEGMENT_FIELD_SCRATCH_START, SEGMENT_POSITION_POOL_START, SEGMENT_POSTING_LOG_START, SEGMENT_TOKEN_TABLE_START, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT, TARGET_BLOCK_ENTRIES, clearNormalizationCache, codePointOrder, configureNormalizationCache, encodeLookupKey, encodeStringBlob, expandTermPrefix, getFlatSchema, getNestedValue, getNormalizationCacheSize, getOrCreateFieldNameIndex, parseVectorDimension, partialWordSearchModule, partitionDocumentFrequency, resetNormalizationCache, sameFields, suggestDisplayTerms, textFieldsChanged, tokenize, tokenizeOptions, visitDocumentSpellings, visitDocumentSurfaceSpellings, visitTokens };
|