@delali/narsil 0.3.1-next.130 → 0.3.1-next.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-JN23AZQP.mjs +43 -0
- package/dist/analysis-change-entry-5JSIMEKG.mjs +1 -0
- package/dist/browser/analysis-change-57MIRJWR.mjs +499 -0
- package/dist/browser/analysis-change-entry-ICGRR3IN.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-3EYDURFD.mjs} +39 -27
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-4MFH226Q.mjs} +1 -1
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-5LYWZJE2.mjs} +2 -2
- package/dist/browser/chunk-67KSBUYR.mjs +173 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-6BSB4N6B.mjs} +2 -2
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-CGVBGHBX.mjs} +81 -6
- package/dist/browser/chunk-E5SFTOQR.mjs +21 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-QHNNULPW.mjs +265 -0
- package/dist/browser/chunk-XVMOQQ34.mjs +384 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-ZAIZQOTM.mjs} +256 -777
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-MTIAN6VT.mjs} +145 -50
- package/dist/browser/import-Q7MXD6BS.mjs +8 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-5J3PEUEB.mjs} +7 -6
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-DISFECE4.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-YBJ5MF2M.mjs} +36 -167
- package/dist/browser/{write-YIC2MKRY.mjs → write-RTWI5MXL.mjs} +8 -6
- package/dist/{capabilities-Bl8G9taj.d.ts → capabilities-DXAmysqJ.d.ts} +11 -11
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-4GCN34JV.mjs} +2 -2
- package/dist/{chunk-SOHMMNCH.mjs → chunk-6VHWXAOI.mjs} +4 -4
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/{chunk-QQTXT65X.mjs → chunk-B5I3IL4H.mjs} +12 -7
- package/dist/chunk-BDIDUW2B.mjs +387 -0
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-CCCGEPJM.mjs} +2 -2
- package/dist/{chunk-2JWNJSNE.mjs → chunk-CPCGEIOM.mjs} +11 -8
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/{chunk-HCGJKDZV.mjs → chunk-FFVLPCLH.mjs} +1 -1
- package/dist/{chunk-WVRDRLIC.mjs → chunk-FKARMUWH.mjs} +6 -6
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/chunk-HMDT3I63.mjs +21 -0
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-IBATGS3J.mjs} +87 -61
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-IP4HJOGC.mjs} +90 -16
- package/dist/{chunk-EUCX5K74.mjs → chunk-J3EHATEZ.mjs} +3 -3
- package/dist/{chunk-X5CNUKD5.mjs → chunk-K3AQOZVM.mjs} +37 -20
- package/dist/chunk-LK3MOTXF.mjs +174 -0
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-LLDFIKGP.mjs} +4 -4
- package/dist/{chunk-K77FJXKS.mjs → chunk-MTQTX4BP.mjs} +81 -8
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-PNNVTKMT.mjs} +2 -2
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-XABMCBHA.mjs → chunk-SBMAWTRG.mjs} +684 -99
- package/dist/chunk-TCWQJL7O.mjs +188 -0
- package/dist/{chunk-2MKX2SCO.mjs → chunk-U4X4AWKL.mjs} +7 -7
- package/dist/{chunk-PU6SKH7O.mjs → chunk-UBXJQJII.mjs} +2 -2
- package/dist/chunk-UEV53BBO.mjs +480 -0
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/{chunk-F6AAXI7T.mjs → chunk-VSUYTPME.mjs} +3 -3
- package/dist/{chunk-TGHNXN7W.mjs → chunk-W2XDK66Z.mjs} +180 -541
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-6M6SDH3S.mjs → chunk-XMQNOD5V.mjs} +8 -8
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-JT3YZGCE.mjs → chunk-XXKWEG53.mjs} +3 -188
- package/dist/{chunk-YBWSV5DV.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-Cx-TiqPE.d.ts → client-DApBeLbk.d.ts} +18 -3
- package/dist/client.d.ts +7 -8
- package/dist/client.mjs +11 -1
- package/dist/{events-DErop1LM.d.ts → config-B0H-vp4U.d.ts} +424 -2
- package/dist/{config-CzB9YvF_.d.ts → config-Brysl9NO.d.ts} +3 -3
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-J5UDLPL3.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -9
- package/dist/distribution.mjs +882 -192
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-AYOTAPNQ.mjs} +165 -70
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-B_W9sdZK.d.ts → engine-CuLMbH1-.d.ts} +64 -7
- package/dist/import-QRZAQ5RH.mjs +43 -0
- package/dist/{index-CKI1fDEl.d.ts → index-HYnqS1SE.d.ts} +2 -2
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +6 -7
- package/dist/index.mjs +20 -20
- package/dist/{long-query-thread-WVSMIBYP.mjs → long-query-thread-U7K5A72G.mjs} +14 -14
- package/dist/{module-2E4XYSGT.mjs → module-L2OZO5PQ.mjs} +11 -11
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +12 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -23
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +11 -8
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-RYSRRFS4.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-CS-clLun.d.ts → schema-BTXbHCqA.d.ts} +4 -1
- package/dist/server.d.ts +200 -11
- package/dist/server.mjs +66 -23
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-BTRZ3MZK.mjs} +59 -190
- package/dist/synonyms.mjs +6 -4
- package/dist/tasks-DjGjK70Y.d.ts +720 -0
- package/dist/{thread-DNK5XWXG.mjs → thread-USB3GOPY.mjs} +13 -13
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-ZKR7SNRT.mjs → write-GHTA56OW.mjs} +25 -22
- package/openapi.json +5310 -0
- package/package.json +16 -8
- package/dist/config-hOWGBeC6.d.ts +0 -414
- package/dist/tasks-BfF-VRhm.d.ts +0 -302
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { readHeapStatistics, readResidentMemory } from './chunk-ZATRDAYU.mjs';
|
|
2
|
+
import { extractVectorFieldsFromSchema } from './chunk-UBXJQJII.mjs';
|
|
3
|
+
import { collectAndReturnPages } from './chunk-RIIUZJQ3.mjs';
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
5
|
+
|
|
6
|
+
// src/engine/snapshot/constants.ts
|
|
7
|
+
var RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE = 0.2;
|
|
8
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 7.86;
|
|
9
|
+
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR = 264;
|
|
10
|
+
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.24;
|
|
11
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 738;
|
|
12
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.2;
|
|
13
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE = 4.06;
|
|
14
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 14.5;
|
|
15
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR = 1482;
|
|
16
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 9.29;
|
|
17
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2141;
|
|
18
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 24.15;
|
|
19
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE = 0.16;
|
|
20
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 6.16;
|
|
21
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR = 204;
|
|
22
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.17;
|
|
23
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 695;
|
|
24
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.04;
|
|
25
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE = 2.14;
|
|
26
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 5.05;
|
|
27
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR = 1080;
|
|
28
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 0.72;
|
|
29
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 1744;
|
|
30
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 20.05;
|
|
31
|
+
var RESTORE_ESTIMATE_MARGIN = 1.25;
|
|
32
|
+
var MAX_SNAPSHOT_ANALYSIS_ATTEMPTS = 3;
|
|
33
|
+
|
|
34
|
+
// src/engine/snapshot/memory-budget.ts
|
|
35
|
+
var MEGABYTE_BYTES = 1048576;
|
|
36
|
+
var ARCHIVE_RATES = {
|
|
37
|
+
worst: {
|
|
38
|
+
heap: {
|
|
39
|
+
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
|
|
40
|
+
perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
|
|
41
|
+
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
42
|
+
},
|
|
43
|
+
external: {
|
|
44
|
+
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
45
|
+
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
|
|
46
|
+
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
floor: {
|
|
50
|
+
heap: {
|
|
51
|
+
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
|
|
52
|
+
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
|
|
53
|
+
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
54
|
+
},
|
|
55
|
+
external: {
|
|
56
|
+
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
57
|
+
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
|
|
58
|
+
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
var EARLIER_SNAPSHOT_RATES = {
|
|
63
|
+
worst: {
|
|
64
|
+
heap: {
|
|
65
|
+
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
66
|
+
perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
67
|
+
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
68
|
+
},
|
|
69
|
+
external: {
|
|
70
|
+
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
71
|
+
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
72
|
+
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
73
|
+
}
|
|
74
|
+
},
|
|
75
|
+
floor: {
|
|
76
|
+
heap: {
|
|
77
|
+
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
78
|
+
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
79
|
+
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
80
|
+
},
|
|
81
|
+
external: {
|
|
82
|
+
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
83
|
+
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
84
|
+
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
function bytesAt(rate, storedBytes, vectors) {
|
|
89
|
+
let bytes = storedBytes * rate.perStoredByte;
|
|
90
|
+
for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
|
|
91
|
+
return bytes;
|
|
92
|
+
}
|
|
93
|
+
function footprintAt(rates, replacesIndex, storedBytes, vectors) {
|
|
94
|
+
const chosen = replacesIndex ? rates.worst : rates.floor;
|
|
95
|
+
const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
|
|
96
|
+
return {
|
|
97
|
+
heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
|
|
98
|
+
externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
function deadCountOf(dead) {
|
|
102
|
+
if (dead === null) return 0;
|
|
103
|
+
let count = 0;
|
|
104
|
+
for (const byte of dead) {
|
|
105
|
+
for (let bits = byte; bits !== 0; bits &= bits - 1) count += 1;
|
|
106
|
+
}
|
|
107
|
+
return count;
|
|
108
|
+
}
|
|
109
|
+
function archiveFootprint(prepared, manifest, replacesIndex) {
|
|
110
|
+
let segmentBytes = 0;
|
|
111
|
+
for (const partition of manifest.partitions) {
|
|
112
|
+
for (const segment of partition.segments) segmentBytes += segment.bytes;
|
|
113
|
+
}
|
|
114
|
+
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
115
|
+
const vectors = [];
|
|
116
|
+
for (const field of manifest.vectors) {
|
|
117
|
+
const dimension = dimensions.get(field.fieldPath) ?? 0;
|
|
118
|
+
for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
|
|
119
|
+
}
|
|
120
|
+
return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
|
|
121
|
+
}
|
|
122
|
+
function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
|
|
123
|
+
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
124
|
+
const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
|
|
125
|
+
return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
|
|
126
|
+
}
|
|
127
|
+
function inMegabytes(bytes) {
|
|
128
|
+
return Math.ceil(bytes / MEGABYTE_BYTES);
|
|
129
|
+
}
|
|
130
|
+
function shortfallOf(heapBytes, processBytes, countInUse) {
|
|
131
|
+
const heap = readHeapStatistics();
|
|
132
|
+
if (heap !== null) {
|
|
133
|
+
const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
|
|
134
|
+
const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
|
|
135
|
+
if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
|
|
136
|
+
}
|
|
137
|
+
const resident = readResidentMemory();
|
|
138
|
+
if (resident !== null) {
|
|
139
|
+
const availableBytes = countInUse ? resident.limitBytes - resident.residentBytes : resident.limitBytes;
|
|
140
|
+
if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
|
|
141
|
+
}
|
|
142
|
+
return null;
|
|
143
|
+
}
|
|
144
|
+
function refuse(indexName, replacesIndex, shortfall) {
|
|
145
|
+
const { budget, neededBytes, availableBytes } = shortfall;
|
|
146
|
+
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
147
|
+
const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
|
|
148
|
+
throw new NarsilError(
|
|
149
|
+
ErrorCodes.RESTORE_MEMORY_EXCEEDED,
|
|
150
|
+
`The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
|
|
151
|
+
{ indexName, budget, neededBytes, availableBytes }
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
function requireAnalysisRebuildMemory(indexName, termBytes) {
|
|
155
|
+
if (shortfallOf(termBytes, termBytes, true) === null) return;
|
|
156
|
+
const shortfall = shortfallOf(termBytes, termBytes, collectAndReturnPages());
|
|
157
|
+
if (shortfall === null) return;
|
|
158
|
+
const { budget, neededBytes, availableBytes } = shortfall;
|
|
159
|
+
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
160
|
+
throw new NarsilError(
|
|
161
|
+
ErrorCodes.ANALYSIS_REBUILD_MEMORY_EXCEEDED,
|
|
162
|
+
`The analysis change of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where} for a second copy of its terms, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, so the index keeps its current settings. Give the process more memory, then change the settings again`,
|
|
163
|
+
{ indexName, budget, neededBytes, availableBytes }
|
|
164
|
+
);
|
|
165
|
+
}
|
|
166
|
+
function requireRestoreMemory(indexName, footprint, replacesIndex) {
|
|
167
|
+
const processBytes = footprint.heapBytes + footprint.externalBytes;
|
|
168
|
+
if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
|
|
169
|
+
const collected = collectAndReturnPages();
|
|
170
|
+
const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
|
|
171
|
+
if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, archiveFootprint, earlierSnapshotFootprint, requireAnalysisRebuildMemory, requireRestoreMemory };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-
|
|
2
|
-
import { getNestedValue } from './chunk-
|
|
3
|
-
import { requireValidFieldTypes } from './chunk-
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-W2XDK66Z.mjs';
|
|
2
|
+
import { getNestedValue } from './chunk-J3EHATEZ.mjs';
|
|
3
|
+
import { requireValidFieldTypes } from './chunk-UBXJQJII.mjs';
|
|
4
4
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
5
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
5
|
+
import { ErrorCodes, NarsilError } from './chunk-HNHCNTVG.mjs';
|
|
6
6
|
import { decode } from '@msgpack/msgpack';
|
|
7
7
|
|
|
8
8
|
function validateReplicationEntry(entry, localPrimaryTerm, log) {
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-
|
|
2
|
-
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, readSegmentFile, manifestKey, segmentsPrefix, readListedFile, requireListedChecksum, vectorFileKey, graphPartKey } from './chunk-
|
|
3
|
-
import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader, packChecksummedEnvelope } from './chunk-
|
|
4
|
-
import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-
|
|
5
|
-
import { DEFAULT_BM25_B, DEFAULT_BM25_K1 } from './chunk-
|
|
6
|
-
import { flattenSchema, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-
|
|
1
|
+
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-ICW45TTU.mjs';
|
|
2
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, readSegmentFile, manifestKey, segmentsPrefix, readListedFile, requireListedChecksum, vectorFileKey, graphPartKey } from './chunk-CPCGEIOM.mjs';
|
|
3
|
+
import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader, packChecksummedEnvelope } from './chunk-W2XDK66Z.mjs';
|
|
4
|
+
import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-IBATGS3J.mjs';
|
|
5
|
+
import { DEFAULT_BM25_B, DEFAULT_BM25_K1 } from './chunk-U4X4AWKL.mjs';
|
|
6
|
+
import { flattenSchema, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-UBXJQJII.mjs';
|
|
7
7
|
import { MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-U476IXKQ.mjs';
|
|
8
8
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
9
9
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
10
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
10
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
11
11
|
import { decode, encode } from '@msgpack/msgpack';
|
|
12
12
|
|
|
13
13
|
// src/engine/durability-metadata.ts
|
|
@@ -47,6 +47,12 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
|
47
47
|
if (config.stopWordList !== void 0) {
|
|
48
48
|
metadata.stopWordList = config.stopWordList;
|
|
49
49
|
}
|
|
50
|
+
if (config.analysisGeneration !== void 0) {
|
|
51
|
+
metadata.analysisGeneration = config.analysisGeneration;
|
|
52
|
+
}
|
|
53
|
+
if (config.pendingAnalysis !== void 0) {
|
|
54
|
+
metadata.pendingAnalysis = config.pendingAnalysis;
|
|
55
|
+
}
|
|
50
56
|
if (config.partitionLimits !== void 0) {
|
|
51
57
|
metadata.partitionLimits = config.partitionLimits;
|
|
52
58
|
}
|
|
@@ -101,6 +107,8 @@ function metadataConfigOf(entry) {
|
|
|
101
107
|
...typeof entry.config.tokenizer === "string" ? { tokenizer: entry.config.tokenizer } : {},
|
|
102
108
|
...typeof entry.config.stopWords === "string" ? { stopWords: entry.config.stopWords } : {},
|
|
103
109
|
...entry.config.stopWords instanceof Set ? { stopWordList: [...entry.config.stopWords].sort(compareCodePoints) } : {},
|
|
110
|
+
...entry.analysisGeneration > 0 ? { analysisGeneration: entry.analysisGeneration } : {},
|
|
111
|
+
...entry.pendingAnalysis !== null ? { pendingAnalysis: entry.pendingAnalysis } : {},
|
|
104
112
|
...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
|
|
105
113
|
...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
|
|
106
114
|
...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
|
|
@@ -111,6 +119,38 @@ function metadataConfigOf(entry) {
|
|
|
111
119
|
...entry.synonyms !== null ? { synonyms: rulesOfStoredSynonyms(entry.synonyms) } : {}
|
|
112
120
|
};
|
|
113
121
|
}
|
|
122
|
+
function durableMetadataConfigOf(entry) {
|
|
123
|
+
const terms = metadataConfigOf(entry);
|
|
124
|
+
const durable = entry.durableAnalysis;
|
|
125
|
+
if (durable === null) return terms;
|
|
126
|
+
const {
|
|
127
|
+
language,
|
|
128
|
+
tokenizer,
|
|
129
|
+
stopWords,
|
|
130
|
+
stopWordList,
|
|
131
|
+
analysisRevision: _analysisRevision,
|
|
132
|
+
arrayPositionGap: _arrayPositionGap,
|
|
133
|
+
analysisGeneration: _analysisGeneration,
|
|
134
|
+
...rest
|
|
135
|
+
} = terms;
|
|
136
|
+
return {
|
|
137
|
+
...rest,
|
|
138
|
+
language: durable.language,
|
|
139
|
+
...durable.tokenizer !== void 0 ? { tokenizer: durable.tokenizer } : {},
|
|
140
|
+
...durable.stopWords !== void 0 ? { stopWords: durable.stopWords } : {},
|
|
141
|
+
...durable.stopWordList !== void 0 ? { stopWordList: durable.stopWordList } : {},
|
|
142
|
+
...durable.analysisRevision !== null ? { analysisRevision: durable.analysisRevision } : {},
|
|
143
|
+
...durable.arrayPositionGap !== null ? { arrayPositionGap: durable.arrayPositionGap } : {},
|
|
144
|
+
...durable.generation > 0 ? { analysisGeneration: durable.generation } : {},
|
|
145
|
+
pendingAnalysis: {
|
|
146
|
+
generation: entry.analysisGeneration,
|
|
147
|
+
language,
|
|
148
|
+
...tokenizer !== void 0 ? { tokenizer } : {},
|
|
149
|
+
...stopWords !== void 0 ? { stopWords } : {},
|
|
150
|
+
...stopWordList !== void 0 ? { stopWordList } : {}
|
|
151
|
+
}
|
|
152
|
+
};
|
|
153
|
+
}
|
|
114
154
|
function indexMetadataOf(indexName, entry, partitionCount) {
|
|
115
155
|
return metadataFromConfig(indexName, metadataConfigOf(entry), partitionCount);
|
|
116
156
|
}
|
|
@@ -322,6 +362,19 @@ function metadataToWire(meta) {
|
|
|
322
362
|
if (meta.stopWordList !== void 0) {
|
|
323
363
|
wire.stop_word_list = meta.stopWordList;
|
|
324
364
|
}
|
|
365
|
+
if (meta.analysisGeneration !== void 0) {
|
|
366
|
+
wire.analysis_generation = meta.analysisGeneration;
|
|
367
|
+
}
|
|
368
|
+
if (meta.pendingAnalysis !== void 0) {
|
|
369
|
+
const pending = meta.pendingAnalysis;
|
|
370
|
+
wire.pending_analysis = {
|
|
371
|
+
generation: pending.generation,
|
|
372
|
+
language: pending.language,
|
|
373
|
+
...pending.tokenizer !== void 0 ? { tokenizer: pending.tokenizer } : {},
|
|
374
|
+
...pending.stopWords !== void 0 ? { stop_words: pending.stopWords } : {},
|
|
375
|
+
...pending.stopWordList !== void 0 ? { stop_word_list: pending.stopWordList } : {}
|
|
376
|
+
};
|
|
377
|
+
}
|
|
325
378
|
if (meta.partitionLimits !== void 0) {
|
|
326
379
|
wire.partition_limits = {
|
|
327
380
|
...meta.partitionLimits.maxDocsPerPartition !== void 0 ? { max_docs_per_partition: meta.partitionLimits.maxDocsPerPartition } : {},
|
|
@@ -417,6 +470,19 @@ function wireToMetadata(raw) {
|
|
|
417
470
|
if (Array.isArray(raw.stop_word_list) && raw.stop_word_list.every((word) => typeof word === "string")) {
|
|
418
471
|
meta.stopWordList = raw.stop_word_list;
|
|
419
472
|
}
|
|
473
|
+
if (isPositiveInteger(raw.analysis_generation)) {
|
|
474
|
+
meta.analysisGeneration = raw.analysis_generation;
|
|
475
|
+
}
|
|
476
|
+
const pending = raw.pending_analysis;
|
|
477
|
+
if (pending && isPositiveInteger(pending.generation) && typeof pending.language === "string") {
|
|
478
|
+
meta.pendingAnalysis = {
|
|
479
|
+
generation: pending.generation,
|
|
480
|
+
language: pending.language,
|
|
481
|
+
...typeof pending.tokenizer === "string" ? { tokenizer: pending.tokenizer } : {},
|
|
482
|
+
...typeof pending.stop_words === "string" ? { stopWords: pending.stop_words } : {},
|
|
483
|
+
...isStringList(pending.stop_word_list) ? { stopWordList: pending.stop_word_list } : {}
|
|
484
|
+
};
|
|
485
|
+
}
|
|
420
486
|
if (raw.partition_limits && typeof raw.partition_limits === "object") {
|
|
421
487
|
const limits = {
|
|
422
488
|
...isPositiveInteger(raw.partition_limits.max_docs_per_partition) ? { maxDocsPerPartition: raw.partition_limits.max_docs_per_partition } : {},
|
|
@@ -509,6 +575,9 @@ function wireToMetadata(raw) {
|
|
|
509
575
|
function isNonNegativeInteger(value) {
|
|
510
576
|
return typeof value === "number" && Number.isInteger(value) && value >= 0;
|
|
511
577
|
}
|
|
578
|
+
function isStringList(value) {
|
|
579
|
+
return Array.isArray(value) && value.every((word) => typeof word === "string");
|
|
580
|
+
}
|
|
512
581
|
function isPositiveInteger(value) {
|
|
513
582
|
return typeof value === "number" && Number.isInteger(value) && value >= 1;
|
|
514
583
|
}
|
|
@@ -788,12 +857,16 @@ function decodeManifestPayload(raw, indexName) {
|
|
|
788
857
|
if (!isRecord(schema)) invalidManifest("is missing its schema");
|
|
789
858
|
requireValidFieldTypes(schema);
|
|
790
859
|
if (typeof raw.language !== "string") invalidManifest("is missing its language");
|
|
860
|
+
if (raw.analysisGeneration !== void 0 && !isSequenceNumber(raw.analysisGeneration)) {
|
|
861
|
+
invalidManifest("has an invalid analysis generation");
|
|
862
|
+
}
|
|
791
863
|
return {
|
|
792
864
|
kind: "current",
|
|
793
865
|
manifest: {
|
|
794
866
|
version: SEGMENT_MANIFEST_VERSION,
|
|
795
867
|
schema,
|
|
796
868
|
language: raw.language,
|
|
869
|
+
...raw.analysisGeneration ? { analysisGeneration: raw.analysisGeneration } : {},
|
|
797
870
|
checkpoint: normalizeCheckpoint(raw.checkpoint),
|
|
798
871
|
partitions: normalizePartitions(raw.partitions, indexName),
|
|
799
872
|
vectors: normalizeVectorFields(raw.vectors, indexName)
|
|
@@ -1174,4 +1247,4 @@ async function reclaimUnlistedKeys(store, indexName, referenced) {
|
|
|
1174
1247
|
}
|
|
1175
1248
|
}
|
|
1176
1249
|
|
|
1177
|
-
export { SINGLE_NODE_PRIMARY_TERM, VECTOR_FILE_ENVELOPE_VERSION, createSeqOwner, decodeManifestPayload, decodeSegmentManifest, deserializePayloadV1, dropStoredVectorValues, growPartitionsTo, indexMetadataOf, invalidManifest, isId, isRecord, isSequenceNumber, isUint32, listedKeysOf, loadPartitionSegments, loadSegmentedCheckpoint, metadataConfigOf, metadataFromConfig, openManifestPayload, patternIndexesFromWire, readCurrentManifest, readGraphParts, readMetadataEnvelope, readSegmentManifest, readVectorFile, reclaimUnlistedKeys, sanitizeSurfaceForms, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeLiveVectors, writeMetadataEnvelope };
|
|
1250
|
+
export { SINGLE_NODE_PRIMARY_TERM, VECTOR_FILE_ENVELOPE_VERSION, createSeqOwner, decodeManifestPayload, decodeSegmentManifest, deserializePayloadV1, dropStoredVectorValues, durableMetadataConfigOf, growPartitionsTo, indexMetadataOf, invalidManifest, isId, isRecord, isSequenceNumber, isUint32, listedKeysOf, loadPartitionSegments, loadSegmentedCheckpoint, metadataConfigOf, metadataFromConfig, openManifestPayload, patternIndexesFromWire, readCurrentManifest, readGraphParts, readMetadataEnvelope, readSegmentManifest, readVectorFile, reclaimUnlistedKeys, sanitizeSurfaceForms, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeLiveVectors, writeMetadataEnvelope };
|
|
@@ -1,20 +1,3 @@
|
|
|
1
|
-
import { english } from './chunk-TGP5M5R7.mjs';
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
|
|
3
|
-
|
|
4
|
-
// src/languages/registry.ts
|
|
5
|
-
var languages = /* @__PURE__ */ new Map();
|
|
6
|
-
languages.set("english", english);
|
|
7
|
-
function registerLanguage(module) {
|
|
8
|
-
languages.set(module.name, module);
|
|
9
|
-
}
|
|
10
|
-
function getLanguage(name) {
|
|
11
|
-
const lang = languages.get(name);
|
|
12
|
-
if (!lang) {
|
|
13
|
-
throw new NarsilError(ErrorCodes.LANGUAGE_NOT_SUPPORTED, `Language "${name}" is not registered`, { language: name });
|
|
14
|
-
}
|
|
15
|
-
return lang;
|
|
16
|
-
}
|
|
17
|
-
|
|
18
1
|
// src/core/id-generator.ts
|
|
19
2
|
function generateId() {
|
|
20
3
|
const bytes = new Uint8Array(16);
|
|
@@ -32,4 +15,4 @@ function generateId() {
|
|
|
32
15
|
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
33
16
|
}
|
|
34
17
|
|
|
35
|
-
export { generateId
|
|
18
|
+
export { generateId };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { LOOKUP_KEY_SCRATCH_BYTES } from './chunk-4UMZ7KQL.mjs';
|
|
2
2
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
3
|
-
import { createNarsilError, ErrorCodes } from './chunk-
|
|
3
|
+
import { createNarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
4
4
|
|
|
5
5
|
// src/core/tokenizer/constants.ts
|
|
6
6
|
var TOKEN_CACHE_SIZE_FLOOR = 5e4;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { parseSynonymRules, encodeSynonymRules, decodeSynonymRules, SYNONYM_EXPANSION_CACHE_ENTRIES, SYNONYMS_ADDED_PER_QUERY } from './chunk-
|
|
2
|
-
import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-
|
|
1
|
+
import { getLanguage, parseSynonymRules, encodeSynonymRules, decodeSynonymRules, SYNONYM_EXPANSION_CACHE_ENTRIES, SYNONYMS_ADDED_PER_QUERY } from './chunk-WNBRW4Q2.mjs';
|
|
2
|
+
import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-NPNK57CA.mjs';
|
|
3
3
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
5
5
|
|
|
6
6
|
// src/analysis/registry.ts
|
|
7
7
|
var tokenizers = /* @__PURE__ */ new Map();
|
|
@@ -78,6 +78,26 @@ function resolveIndexAnalysis(config) {
|
|
|
78
78
|
};
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
+
// src/engine/analysis-change/pending-target.ts
|
|
82
|
+
function configUnderAnalysis(base, language, tokenizer, stopWords) {
|
|
83
|
+
const { tokenizer: _tokenizer, stopWords: _stopWords, ...rest } = base;
|
|
84
|
+
return {
|
|
85
|
+
...rest,
|
|
86
|
+
language,
|
|
87
|
+
...tokenizer !== void 0 ? { tokenizer } : {},
|
|
88
|
+
...stopWords !== void 0 ? { stopWords } : {}
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
function configOfNamedSettings(base, settings) {
|
|
92
|
+
const { language, tokenizer, stopWords, stopWordList } = settings;
|
|
93
|
+
const listed = stopWordList === void 0 ? void 0 : new Set(stopWordList);
|
|
94
|
+
return configUnderAnalysis(base, language, tokenizer, stopWords ?? listed);
|
|
95
|
+
}
|
|
96
|
+
function targetOfPending(entry, pending) {
|
|
97
|
+
const config = configOfNamedSettings(entry.config, pending);
|
|
98
|
+
return { config, language: getLanguage(pending.language), analysis: resolveIndexAnalysis(config) };
|
|
99
|
+
}
|
|
100
|
+
|
|
81
101
|
// src/engine/serial-index-update.ts
|
|
82
102
|
var updatingByCore = /* @__PURE__ */ new WeakMap();
|
|
83
103
|
async function withOpenIndex(core, indexName, action) {
|
|
@@ -394,6 +414,11 @@ async function replaceNow(core, indexName, value, strictWords) {
|
|
|
394
414
|
const entry = core.requireIndex(indexName);
|
|
395
415
|
const manager = core.requireManager(indexName);
|
|
396
416
|
const prepared = prepareSynonyms(value, entry.config, entry.language, strictWords);
|
|
417
|
+
if (entry.pendingAnalysis !== null) {
|
|
418
|
+
const target = targetOfPending(entry, entry.pendingAnalysis);
|
|
419
|
+
prepareSynonyms(value, target.config, target.language, strictWords);
|
|
420
|
+
}
|
|
421
|
+
const preparedUnder = manager.analysis;
|
|
397
422
|
const previous = entry.synonyms;
|
|
398
423
|
entry.synonyms = prepared?.encoded ?? null;
|
|
399
424
|
try {
|
|
@@ -402,7 +427,8 @@ async function replaceNow(core, indexName, value, strictWords) {
|
|
|
402
427
|
entry.synonyms = previous;
|
|
403
428
|
throw err;
|
|
404
429
|
}
|
|
405
|
-
manager.
|
|
430
|
+
const current = manager.analysis === preparedUnder ? prepared : prepareSynonyms(value, entry.config, entry.language, false);
|
|
431
|
+
manager.synonyms = current?.lookup ?? null;
|
|
406
432
|
await core.orchestrator.replicateToWorkers({
|
|
407
433
|
type: "replaceSynonyms",
|
|
408
434
|
indexName,
|
|
@@ -441,4 +467,4 @@ var synonymSearchModule = {
|
|
|
441
467
|
applyLoaded: applyLoadedSynonyms
|
|
442
468
|
};
|
|
443
469
|
|
|
444
|
-
export { getStopWords, getTokenizer, hasStopWords, hasTokenizer, oneAtATime, registerStopWords, registerTokenizer, resolveIndexAnalysis, synonymSearchModule, withOpenIndex };
|
|
470
|
+
export { configOfNamedSettings, configUnderAnalysis, getStopWords, getTokenizer, hasStopWords, hasTokenizer, oneAtATime, registerStopWords, registerTokenizer, resolveIndexAnalysis, synonymSearchModule, targetOfPending, withOpenIndex };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { sanitizeSurfaceForms, patternIndexesFromWire, dropStoredVectorValues } from './chunk-
|
|
2
|
-
import { requireValidFieldTypes } from './chunk-
|
|
1
|
+
import { sanitizeSurfaceForms, patternIndexesFromWire, dropStoredVectorValues } from './chunk-MTQTX4BP.mjs';
|
|
2
|
+
import { requireValidFieldTypes } from './chunk-UBXJQJII.mjs';
|
|
3
3
|
import { VALID_HNSW_METRICS } from './chunk-U476IXKQ.mjs';
|
|
4
4
|
import { decode } from '@msgpack/msgpack';
|
|
5
5
|
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { MAX_FILTER_ARRAY_SIZE } from './chunk-WQBCBKY3.mjs';
|
|
2
2
|
import { bitsetHas, bitsetSet, GrowableUint32, createBitSet, bitsetAnd, bitsetNot } from './chunk-JUKYYXSP.mjs';
|
|
3
3
|
import { tabulatedLevenshtein } from './chunk-GES7U2PM.mjs';
|
|
4
|
-
import { parsedTypeOf } from './chunk-
|
|
4
|
+
import { parsedTypeOf } from './chunk-ZSWHPKGM.mjs';
|
|
5
5
|
import { codePointCount, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
6
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
6
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
7
7
|
|
|
8
8
|
// src/core/pattern-index/constants.ts
|
|
9
9
|
var DEFAULT_PATTERN_WORK_CAP = 25e6;
|