@delali/narsil 0.3.1-next.131 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
- package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
- package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
- package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
- package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
- package/dist/browser/import-TLB2A5LJ.mjs +9 -0
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
- package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
- package/dist/chunk-46NZOPHS.mjs +21 -0
- package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
- package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
- package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
- package/dist/chunk-L76W3IZJ.mjs +193 -0
- package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-SRJBFG2T.mjs +881 -0
- package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
- package/dist/chunk-VJEGOELZ.mjs +177 -0
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
- package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
- package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
- package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +11 -1
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
- package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -8
- package/dist/distribution.mjs +982 -211
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
- package/dist/import-OOBUW3RR.mjs +44 -0
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +5 -5
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +22 -22
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
- package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +12 -10
- package/dist/server.mjs +54 -21
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
- package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
- package/openapi.json +114 -6
- package/package.json +12 -8
package/dist/server.mjs
CHANGED
|
@@ -1,36 +1,37 @@
|
|
|
1
|
-
import { answerRequest } from './chunk-
|
|
2
|
-
import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, PROMETHEUS_TEXT_CONTENT_TYPE, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, clearGateSlot, clearBodyBudgetSlot, createCloseAtExitWords, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, isRemoveOnlyBatch, validateBatch, serializeBatchResult, parseWriteJson, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, withServerRerankWindow, respondAnswer, parseTaskListQuery, parseJsonOptional, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, parseRefusalOf, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, SERVER_CAPABILITIES, toHttpError, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-
|
|
3
|
-
export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-
|
|
4
|
-
import { rerankOnHost } from './chunk-
|
|
5
|
-
import { writeBlockLookupOf, indexReadsOf, isRecord, validateIndexName } from './chunk-
|
|
6
|
-
import { readMemoryFigures } from './chunk-
|
|
1
|
+
import { answerRequest } from './chunk-ZMBFWQXN.mjs';
|
|
2
|
+
import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, PROMETHEUS_TEXT_CONTENT_TYPE, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, clearGateSlot, clearBodyBudgetSlot, createCloseAtExitWords, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, isRemoveOnlyBatch, validateBatch, serializeBatchResult, parseWriteJson, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, withServerRerankWindow, respondAnswer, parseTaskListQuery, parseJsonOptional, validateChangeAnalysis, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, parseRefusalOf, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, SERVER_CAPABILITIES, toHttpError, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-SBMAWTRG.mjs';
|
|
3
|
+
export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, CHANGE_ANALYSIS_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-SBMAWTRG.mjs';
|
|
4
|
+
import { rerankOnHost } from './chunk-HJJWY6PT.mjs';
|
|
5
|
+
import { writeBlockLookupOf, indexReadsOf, isRecord, validateIndexName } from './chunk-BRUUBXCB.mjs';
|
|
6
|
+
import { readMemoryFigures } from './chunk-DD2673TX.mjs';
|
|
7
|
+
import './chunk-L76W3IZJ.mjs';
|
|
7
8
|
import './chunk-IWVEG5CX.mjs';
|
|
8
9
|
import './chunk-DJHHF2L5.mjs';
|
|
9
|
-
import './chunk-
|
|
10
|
-
import { rerankWindowCapOf, engineCoreOf, providedDocId, resolveVectorText } from './chunk-
|
|
10
|
+
import './chunk-JDDXV6UF.mjs';
|
|
11
|
+
import { rerankWindowCapOf, engineCoreOf, providedDocId, resolveVectorText } from './chunk-5WYVUPGD.mjs';
|
|
12
|
+
import { MAX_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
|
|
11
13
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
12
|
-
import './chunk-
|
|
14
|
+
import './chunk-EEBCDP36.mjs';
|
|
15
|
+
import './chunk-N37VMC62.mjs';
|
|
13
16
|
import './chunk-ZATRDAYU.mjs';
|
|
14
17
|
import './chunk-EBIHRL7B.mjs';
|
|
15
|
-
import
|
|
16
|
-
import './chunk-RNSC4NFI.mjs';
|
|
17
|
-
import './chunk-TGP5M5R7.mjs';
|
|
18
|
-
import './chunk-U2BKYEC7.mjs';
|
|
19
|
-
import './chunk-2C3WXBQS.mjs';
|
|
18
|
+
import './chunk-74XB2SXI.mjs';
|
|
20
19
|
import './chunk-WQBCBKY3.mjs';
|
|
21
|
-
import './chunk-
|
|
20
|
+
import './chunk-KGOXHLLZ.mjs';
|
|
22
21
|
import './chunk-U476IXKQ.mjs';
|
|
23
22
|
import './chunk-FQBFZAWG.mjs';
|
|
24
23
|
import './chunk-VJS4JFBQ.mjs';
|
|
25
|
-
import './chunk-
|
|
26
|
-
import './chunk-
|
|
24
|
+
import './chunk-ZSWHPKGM.mjs';
|
|
25
|
+
import './chunk-WNBRW4Q2.mjs';
|
|
26
|
+
import './chunk-TGP5M5R7.mjs';
|
|
27
|
+
import './chunk-U2BKYEC7.mjs';
|
|
27
28
|
import './chunk-R6A773WK.mjs';
|
|
28
29
|
import './chunk-YTJCGKM4.mjs';
|
|
29
30
|
import { startIdleHeapCleanup } from './chunk-RIIUZJQ3.mjs';
|
|
30
31
|
import './chunk-22YPZVMG.mjs';
|
|
31
32
|
import './chunk-L5ORED7V.mjs';
|
|
32
|
-
import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-
|
|
33
|
-
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-
|
|
33
|
+
import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-HNHCNTVG.mjs';
|
|
34
|
+
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-HNHCNTVG.mjs';
|
|
34
35
|
import { randomUUID } from 'crypto';
|
|
35
36
|
import { createReadStream } from 'fs';
|
|
36
37
|
import { MessageChannel } from 'worker_threads';
|
|
@@ -151,6 +152,24 @@ function createAdminHandlers(deps) {
|
|
|
151
152
|
const record = await tasks.start("rebuildAnalysis", name, () => engine.rebuildAnalysis(name));
|
|
152
153
|
respondJson(ctx, record, 202);
|
|
153
154
|
}
|
|
155
|
+
async function changeAnalysis(ctx) {
|
|
156
|
+
const body = parseJsonOptional(ctx);
|
|
157
|
+
if (!body) return;
|
|
158
|
+
const failure = validateChangeAnalysis(body);
|
|
159
|
+
if (failure) {
|
|
160
|
+
rejectInvalid(ctx, failure);
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
const name = ctx.params[0];
|
|
164
|
+
try {
|
|
165
|
+
await requireIndexExists(name);
|
|
166
|
+
} catch (err) {
|
|
167
|
+
respondError(ctx, err);
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
const record = await tasks.start("changeAnalysis", name, () => engine.changeAnalysis(name, body));
|
|
171
|
+
respondJson(ctx, record, 202);
|
|
172
|
+
}
|
|
154
173
|
async function rebalance(ctx) {
|
|
155
174
|
const body = parseJsonOptional(ctx);
|
|
156
175
|
if (!body) return;
|
|
@@ -250,6 +269,7 @@ function createAdminHandlers(deps) {
|
|
|
250
269
|
optimize,
|
|
251
270
|
rebalance,
|
|
252
271
|
rebuildAnalysis,
|
|
272
|
+
changeAnalysis,
|
|
253
273
|
restore,
|
|
254
274
|
getTask,
|
|
255
275
|
listTasks,
|
|
@@ -767,15 +787,27 @@ function mapHttpIndexConfig(http) {
|
|
|
767
787
|
if (http.defaultScoring !== void 0) config.defaultScoring = http.defaultScoring;
|
|
768
788
|
if (http.bm25 !== void 0) config.bm25 = http.bm25;
|
|
769
789
|
if (http.trackPositions !== void 0) config.trackPositions = http.trackPositions;
|
|
790
|
+
if (http.textStorage !== void 0) config.textStorage = http.textStorage;
|
|
770
791
|
if (http.surfaceForms !== void 0) config.surfaceForms = http.surfaceForms === true;
|
|
771
792
|
if (http.vectorPromotion !== void 0) config.vectorPromotion = http.vectorPromotion;
|
|
772
793
|
if (http.strict !== void 0) config.strict = http.strict;
|
|
773
794
|
if (http.required !== void 0) config.required = http.required;
|
|
774
795
|
if (http.patternValueLimit !== void 0) config.patternValueLimit = http.patternValueLimit;
|
|
775
796
|
if (http.synonyms !== void 0) config.synonyms = http.synonyms;
|
|
776
|
-
if (http.
|
|
797
|
+
if (http.tokenizer !== void 0) {
|
|
798
|
+
if (typeof http.tokenizer !== "string") {
|
|
799
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Field "tokenizer" must be a registered name');
|
|
800
|
+
}
|
|
801
|
+
config.tokenizer = http.tokenizer;
|
|
802
|
+
}
|
|
803
|
+
if (typeof http.stopWords === "string") {
|
|
804
|
+
config.stopWords = http.stopWords;
|
|
805
|
+
} else if (http.stopWords !== void 0) {
|
|
777
806
|
if (!Array.isArray(http.stopWords) || http.stopWords.some((w) => typeof w !== "string")) {
|
|
778
|
-
throw new NarsilError(
|
|
807
|
+
throw new NarsilError(
|
|
808
|
+
ErrorCodes.CONFIG_INVALID,
|
|
809
|
+
'Field "stopWords" must be a registered name or an array of strings'
|
|
810
|
+
);
|
|
779
811
|
}
|
|
780
812
|
config.stopWords = new Set(http.stopWords);
|
|
781
813
|
}
|
|
@@ -1718,6 +1750,7 @@ function mainThreadHandlers(deps) {
|
|
|
1718
1750
|
multiGet: doc.multiGet,
|
|
1719
1751
|
checkpoint: admin.checkpoint,
|
|
1720
1752
|
rebuildAnalysis: admin.rebuildAnalysis,
|
|
1753
|
+
changeAnalysis: admin.changeAnalysis,
|
|
1721
1754
|
snapshot: admin.snapshot,
|
|
1722
1755
|
vectorMaintenance: admin.vectorMaintenance,
|
|
1723
1756
|
compact: admin.compact,
|
|
@@ -1,56 +1,63 @@
|
|
|
1
|
-
import { documentsOfPayloadV2 } from './chunk-
|
|
2
|
-
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import './chunk-
|
|
7
|
-
import './chunk-
|
|
1
|
+
import { documentsOfPayloadV2 } from './chunk-NJJCPFXN.mjs';
|
|
2
|
+
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive, writeArchiveFile } from './chunk-KZL72VTC.mjs';
|
|
3
|
+
import { indexMetadataOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, deserializePayloadV1, SINGLE_NODE_PRIMARY_TERM } from './chunk-YHUTF6EA.mjs';
|
|
4
|
+
import { readVectorFile, readGraphParts } from './chunk-NRUXBYPH.mjs';
|
|
5
|
+
import { createByteReader } from './chunk-CZQ33DLW.mjs';
|
|
6
|
+
import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-L76W3IZJ.mjs';
|
|
7
|
+
import './chunk-W2COJ5AH.mjs';
|
|
8
|
+
import './chunk-Z4HFMYNX.mjs';
|
|
9
|
+
import { decodeVectorIndexParts, bytesToVectors } from './chunk-ICW45TTU.mjs';
|
|
10
|
+
import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, storedBytesOfSegments, archiveFootprint } from './chunk-VJEGOELZ.mjs';
|
|
11
|
+
import './chunk-TNBIK2LQ.mjs';
|
|
8
12
|
import './chunk-37CD3OKS.mjs';
|
|
9
|
-
import
|
|
10
|
-
import './chunk-
|
|
11
|
-
import './chunk-
|
|
13
|
+
import './chunk-DJHHF2L5.mjs';
|
|
14
|
+
import './chunk-LA4WNDHL.mjs';
|
|
15
|
+
import './chunk-JDDXV6UF.mjs';
|
|
16
|
+
import { readSegmentFile } from './chunk-7AUASRE7.mjs';
|
|
17
|
+
import { manifestKey, encodeSegmentManifest, manifestFileRefs } from './chunk-UUZCQQG2.mjs';
|
|
18
|
+
import './chunk-KEIYVZUX.mjs';
|
|
12
19
|
import './chunk-YH5OIL7B.mjs';
|
|
13
|
-
import { encodeIndexConfiguration,
|
|
14
|
-
import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-
|
|
15
|
-
import
|
|
16
|
-
import './chunk-
|
|
20
|
+
import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, storedArrayPositionGap, decodeIndexConfiguration, prepareStoredSynonyms, arrayPositionsAreStale, insertDocumentVectors } from './chunk-IO5F7ZBE.mjs';
|
|
21
|
+
import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-VEFN6X6Q.mjs';
|
|
22
|
+
import './chunk-5WYVUPGD.mjs';
|
|
23
|
+
import './chunk-HJSN7UYG.mjs';
|
|
24
|
+
import './chunk-UIKN2OTQ.mjs';
|
|
17
25
|
import './chunk-AV4IJKD2.mjs';
|
|
18
|
-
import
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
import
|
|
26
|
+
import './chunk-EEBCDP36.mjs';
|
|
27
|
+
import { decodeEveryStoredDocument } from './chunk-WJGO5PGV.mjs';
|
|
28
|
+
import { ARRAY_POSITION_GAP, isCompositePartition } from './chunk-ZPWS2SKJ.mjs';
|
|
29
|
+
import './chunk-N37VMC62.mjs';
|
|
30
|
+
import './chunk-ZATRDAYU.mjs';
|
|
22
31
|
import './chunk-EBIHRL7B.mjs';
|
|
23
|
-
import './chunk-HJSN7UYG.mjs';
|
|
24
|
-
import './chunk-RNSC4NFI.mjs';
|
|
25
|
-
import './chunk-TGP5M5R7.mjs';
|
|
26
|
-
import './chunk-U2BKYEC7.mjs';
|
|
27
32
|
import './chunk-KOKM4PQN.mjs';
|
|
28
33
|
import './chunk-QSK35UC2.mjs';
|
|
29
|
-
import './chunk-
|
|
30
|
-
import './chunk-
|
|
34
|
+
import './chunk-74XB2SXI.mjs';
|
|
35
|
+
import './chunk-RKAVHNRW.mjs';
|
|
31
36
|
import './chunk-WQBCBKY3.mjs';
|
|
32
37
|
import './chunk-JUKYYXSP.mjs';
|
|
33
|
-
import { parseVectorDimension } from './chunk-
|
|
34
|
-
import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage,
|
|
38
|
+
import { parseVectorDimension } from './chunk-DQNKVOJ2.mjs';
|
|
39
|
+
import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, flattenSchema } from './chunk-KGOXHLLZ.mjs';
|
|
35
40
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
36
41
|
import './chunk-FQBFZAWG.mjs';
|
|
37
42
|
import './chunk-VJS4JFBQ.mjs';
|
|
38
43
|
import './chunk-GES7U2PM.mjs';
|
|
39
|
-
import './chunk-
|
|
40
|
-
import { resolveIndexAnalysis } from './chunk-
|
|
44
|
+
import './chunk-ZSWHPKGM.mjs';
|
|
45
|
+
import { resolveIndexAnalysis } from './chunk-O4LJQI4G.mjs';
|
|
41
46
|
import './chunk-X5CV2E2B.mjs';
|
|
42
|
-
import './chunk-
|
|
43
|
-
import './chunk-
|
|
47
|
+
import { getLanguage } from './chunk-WNBRW4Q2.mjs';
|
|
48
|
+
import './chunk-TGP5M5R7.mjs';
|
|
49
|
+
import './chunk-U2BKYEC7.mjs';
|
|
50
|
+
import './chunk-NPNK57CA.mjs';
|
|
44
51
|
import './chunk-4UMZ7KQL.mjs';
|
|
45
52
|
import './chunk-R6A773WK.mjs';
|
|
46
53
|
import './chunk-YTJCGKM4.mjs';
|
|
47
|
-
import
|
|
48
|
-
import './chunk-
|
|
49
|
-
import { loadVectorCodeFor } from './chunk-
|
|
54
|
+
import './chunk-RIIUZJQ3.mjs';
|
|
55
|
+
import './chunk-QHPYS427.mjs';
|
|
56
|
+
import { loadVectorCodeFor } from './chunk-GLZHNSGB.mjs';
|
|
50
57
|
import './chunk-22YPZVMG.mjs';
|
|
51
58
|
import './chunk-RFBLQLLZ.mjs';
|
|
52
59
|
import './chunk-L5ORED7V.mjs';
|
|
53
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
60
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
54
61
|
import { decode } from '@msgpack/msgpack';
|
|
55
62
|
|
|
56
63
|
// src/persistence/archive/validate.ts
|
|
@@ -76,6 +83,14 @@ function requireSchemaOfManifest(key, schema, manifest) {
|
|
|
76
83
|
async function validateStagedCheckpoint(store, manifest) {
|
|
77
84
|
for (const partition of manifest.partitions) {
|
|
78
85
|
for (const ref of partition.segments) {
|
|
86
|
+
const { directory } = store;
|
|
87
|
+
if (directory !== void 0) {
|
|
88
|
+
const { checkPagedSegmentFully, isPagedSegmentFile } = await import('./module-IYPLWHGC.mjs');
|
|
89
|
+
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
90
|
+
requireSchemaOfManifest(ref.key, await checkPagedSegmentFully(directory, ref), manifest);
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
79
94
|
const contents = await readSegmentFile(store, ref);
|
|
80
95
|
requireSchemaOfManifest(ref.key, contents.schema, manifest);
|
|
81
96
|
decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
@@ -92,161 +107,6 @@ async function validateStagedCheckpoint(store, manifest) {
|
|
|
92
107
|
}
|
|
93
108
|
}
|
|
94
109
|
|
|
95
|
-
// src/engine/snapshot/constants.ts
|
|
96
|
-
var RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE = 0.2;
|
|
97
|
-
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 7.86;
|
|
98
|
-
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR = 264;
|
|
99
|
-
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.24;
|
|
100
|
-
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 738;
|
|
101
|
-
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.2;
|
|
102
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE = 4.06;
|
|
103
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 14.5;
|
|
104
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR = 1482;
|
|
105
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 9.29;
|
|
106
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2141;
|
|
107
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 24.15;
|
|
108
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE = 0.16;
|
|
109
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 6.16;
|
|
110
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR = 204;
|
|
111
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.17;
|
|
112
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 695;
|
|
113
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.04;
|
|
114
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE = 2.14;
|
|
115
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 5.05;
|
|
116
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR = 1080;
|
|
117
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 0.72;
|
|
118
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 1744;
|
|
119
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 20.05;
|
|
120
|
-
var RESTORE_ESTIMATE_MARGIN = 1.25;
|
|
121
|
-
|
|
122
|
-
// src/engine/snapshot/memory-budget.ts
|
|
123
|
-
var MEGABYTE_BYTES = 1048576;
|
|
124
|
-
var ARCHIVE_RATES = {
|
|
125
|
-
worst: {
|
|
126
|
-
heap: {
|
|
127
|
-
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
|
|
128
|
-
perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
|
|
129
|
-
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
130
|
-
},
|
|
131
|
-
external: {
|
|
132
|
-
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
133
|
-
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
|
|
134
|
-
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
135
|
-
}
|
|
136
|
-
},
|
|
137
|
-
floor: {
|
|
138
|
-
heap: {
|
|
139
|
-
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
|
|
140
|
-
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
|
|
141
|
-
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
142
|
-
},
|
|
143
|
-
external: {
|
|
144
|
-
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
145
|
-
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
|
|
146
|
-
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
};
|
|
150
|
-
var EARLIER_SNAPSHOT_RATES = {
|
|
151
|
-
worst: {
|
|
152
|
-
heap: {
|
|
153
|
-
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
154
|
-
perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
155
|
-
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
156
|
-
},
|
|
157
|
-
external: {
|
|
158
|
-
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
159
|
-
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
160
|
-
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
161
|
-
}
|
|
162
|
-
},
|
|
163
|
-
floor: {
|
|
164
|
-
heap: {
|
|
165
|
-
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
166
|
-
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
167
|
-
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
168
|
-
},
|
|
169
|
-
external: {
|
|
170
|
-
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
171
|
-
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
172
|
-
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
};
|
|
176
|
-
function bytesAt(rate, storedBytes, vectors) {
|
|
177
|
-
let bytes = storedBytes * rate.perStoredByte;
|
|
178
|
-
for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
|
|
179
|
-
return bytes;
|
|
180
|
-
}
|
|
181
|
-
function footprintAt(rates, replacesIndex, storedBytes, vectors) {
|
|
182
|
-
const chosen = replacesIndex ? rates.worst : rates.floor;
|
|
183
|
-
const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
|
|
184
|
-
return {
|
|
185
|
-
heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
|
|
186
|
-
externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
|
|
187
|
-
};
|
|
188
|
-
}
|
|
189
|
-
function deadCountOf(dead) {
|
|
190
|
-
if (dead === null) return 0;
|
|
191
|
-
let count = 0;
|
|
192
|
-
for (const byte of dead) {
|
|
193
|
-
for (let bits = byte; bits !== 0; bits &= bits - 1) count += 1;
|
|
194
|
-
}
|
|
195
|
-
return count;
|
|
196
|
-
}
|
|
197
|
-
function archiveFootprint(prepared, manifest, replacesIndex) {
|
|
198
|
-
let segmentBytes = 0;
|
|
199
|
-
for (const partition of manifest.partitions) {
|
|
200
|
-
for (const segment of partition.segments) segmentBytes += segment.bytes;
|
|
201
|
-
}
|
|
202
|
-
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
203
|
-
const vectors = [];
|
|
204
|
-
for (const field of manifest.vectors) {
|
|
205
|
-
const dimension = dimensions.get(field.fieldPath) ?? 0;
|
|
206
|
-
for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
|
|
207
|
-
}
|
|
208
|
-
return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
|
|
209
|
-
}
|
|
210
|
-
function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
|
|
211
|
-
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
212
|
-
const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
|
|
213
|
-
return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
|
|
214
|
-
}
|
|
215
|
-
function inMegabytes(bytes) {
|
|
216
|
-
return Math.ceil(bytes / MEGABYTE_BYTES);
|
|
217
|
-
}
|
|
218
|
-
function shortfallOf(heapBytes, processBytes, countInUse) {
|
|
219
|
-
const heap = readHeapStatistics();
|
|
220
|
-
if (heap !== null) {
|
|
221
|
-
const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
|
|
222
|
-
const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
|
|
223
|
-
if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
|
|
224
|
-
}
|
|
225
|
-
const resident = readResidentMemory();
|
|
226
|
-
if (resident !== null) {
|
|
227
|
-
const availableBytes = countInUse ? resident.limitBytes - resident.residentBytes : resident.limitBytes;
|
|
228
|
-
if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
|
|
229
|
-
}
|
|
230
|
-
return null;
|
|
231
|
-
}
|
|
232
|
-
function refuse(indexName, replacesIndex, shortfall) {
|
|
233
|
-
const { budget, neededBytes, availableBytes } = shortfall;
|
|
234
|
-
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
235
|
-
const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
|
|
236
|
-
throw new NarsilError(
|
|
237
|
-
ErrorCodes.RESTORE_MEMORY_EXCEEDED,
|
|
238
|
-
`The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
|
|
239
|
-
{ indexName, budget, neededBytes, availableBytes }
|
|
240
|
-
);
|
|
241
|
-
}
|
|
242
|
-
function requireRestoreMemory(indexName, footprint, replacesIndex) {
|
|
243
|
-
const processBytes = footprint.heapBytes + footprint.externalBytes;
|
|
244
|
-
if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
|
|
245
|
-
const collected = collectAndReturnPages();
|
|
246
|
-
const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
|
|
247
|
-
if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
|
|
248
|
-
}
|
|
249
|
-
|
|
250
110
|
// src/workers/detached-index.ts
|
|
251
111
|
function attachablePolicy(shared) {
|
|
252
112
|
let attached = false;
|
|
@@ -286,6 +146,7 @@ function prepareRestoredIndex(configuration, deps, reject) {
|
|
|
286
146
|
requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
|
|
287
147
|
validateVectorPromotion(configFields.vectorPromotion);
|
|
288
148
|
validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
|
|
149
|
+
validateTextStorage(configFields.textStorage, deps.filesystemDurability);
|
|
289
150
|
if (configFields.required !== void 0 && configFields.required.length > 0) {
|
|
290
151
|
validateRequiredFieldsInSchema(configFields.required, schema);
|
|
291
152
|
}
|
|
@@ -312,12 +173,14 @@ function prepareRestoredIndex(configuration, deps, reject) {
|
|
|
312
173
|
}
|
|
313
174
|
const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
|
|
314
175
|
const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
|
|
176
|
+
const analysisGeneration = Number.isSafeInteger(configuration.analysisGeneration) ? Math.max(0, configuration.analysisGeneration) : 0;
|
|
315
177
|
return {
|
|
316
178
|
config,
|
|
317
179
|
language,
|
|
318
180
|
embeddingAdapter,
|
|
319
181
|
adapterName,
|
|
320
182
|
storedRevision,
|
|
183
|
+
analysisGeneration,
|
|
321
184
|
storedArrayPositionGap: storedArrayPositionGap(configuration.arrayPositionGap),
|
|
322
185
|
synonyms
|
|
323
186
|
};
|
|
@@ -352,7 +215,10 @@ function restoredRegistryEntry(prepared, manager, deps) {
|
|
|
352
215
|
partitionCount: manager.partitionCount,
|
|
353
216
|
analysisRevision: prepared.storedRevision,
|
|
354
217
|
arrayPositionGap: prepared.storedArrayPositionGap,
|
|
355
|
-
synonyms: prepared.synonyms?.encoded ?? null
|
|
218
|
+
synonyms: prepared.synonyms?.encoded ?? null,
|
|
219
|
+
analysisGeneration: prepared.analysisGeneration,
|
|
220
|
+
pendingAnalysis: null,
|
|
221
|
+
durableAnalysis: null
|
|
356
222
|
};
|
|
357
223
|
}
|
|
358
224
|
|
|
@@ -426,7 +292,14 @@ async function commitToStorage(indexName, restored, previousPartitions, deps) {
|
|
|
426
292
|
}
|
|
427
293
|
await staging.finish();
|
|
428
294
|
const relocate = await staging.relocation().catch(() => null);
|
|
429
|
-
if (relocate
|
|
295
|
+
if (relocate === null) return;
|
|
296
|
+
for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
|
|
297
|
+
const { manager } = restored.detached;
|
|
298
|
+
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
|
|
299
|
+
const partition = manager.getPartition(partitionId);
|
|
300
|
+
if (!isCompositePartition(partition)) continue;
|
|
301
|
+
for (const segment of partition.frozenSegments()) segment.paged?.relocate(relocate);
|
|
302
|
+
}
|
|
430
303
|
}
|
|
431
304
|
function activate(indexName, restored, entry, deps) {
|
|
432
305
|
deps.executor.installDetachedIndex(restored.detached);
|
|
@@ -448,6 +321,7 @@ function activate(indexName, restored, entry, deps) {
|
|
|
448
321
|
async function replaceExisting(indexName, restored, entry, deps) {
|
|
449
322
|
const release = { reloads: false, held: [], settled: Promise.resolve(), timedOut: false };
|
|
450
323
|
try {
|
|
324
|
+
await deps.abandonAnalysisChange(indexName);
|
|
451
325
|
await withinLimit(indexName, release, releaseCopies(indexName, release, deps), deps);
|
|
452
326
|
await deps.indexState.replace(indexName, async (wasOpen) => {
|
|
453
327
|
await deps.abandonAnalysisRebuild(indexName);
|
|
@@ -548,6 +422,9 @@ function partitionCountOf(manifest) {
|
|
|
548
422
|
function missingPart(what) {
|
|
549
423
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive holds no ${what}`);
|
|
550
424
|
}
|
|
425
|
+
function keepsTextOnDisk(prepared, staging) {
|
|
426
|
+
return prepared.config.textStorage === "disk" && staging.directory !== void 0;
|
|
427
|
+
}
|
|
551
428
|
async function stageArchive(indexName, reader, staging, deps) {
|
|
552
429
|
const staged = { prepared: null, manifest: null };
|
|
553
430
|
await readSnapshotArchive(reader, {
|
|
@@ -558,17 +435,29 @@ async function stageArchive(indexName, reader, staging, deps) {
|
|
|
558
435
|
if (staged.prepared === null) missingPart("configuration");
|
|
559
436
|
requireSameSchema(staged.prepared, manifest2);
|
|
560
437
|
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
561
|
-
|
|
438
|
+
const segmentBytes = keepsTextOnDisk(staged.prepared, staging) ? 0 : storedBytesOfSegments(manifest2);
|
|
439
|
+
requireRestoreMemory(
|
|
440
|
+
indexName,
|
|
441
|
+
archiveFootprint(staged.prepared, manifest2, replacesIndex, segmentBytes),
|
|
442
|
+
replacesIndex
|
|
443
|
+
);
|
|
562
444
|
staged.manifest = renamed(manifest2, sourceName, indexName);
|
|
563
445
|
},
|
|
564
|
-
file: async (entryName, _ref,
|
|
446
|
+
file: async (entryName, _ref, chunks) => {
|
|
565
447
|
if (staged.manifest === null) missingPart("manifest");
|
|
566
|
-
await staging
|
|
448
|
+
await writeArchiveFile(staging, `${indexName}/${entryName}`, chunks);
|
|
567
449
|
}
|
|
568
450
|
});
|
|
569
451
|
const { prepared, manifest } = staged;
|
|
570
452
|
if (prepared === null) missingPart("configuration");
|
|
571
453
|
if (manifest === null) missingPart("manifest");
|
|
454
|
+
const { directory } = staging;
|
|
455
|
+
if (keepsTextOnDisk(prepared, staging) && directory !== void 0) {
|
|
456
|
+
const { bytesThatSegmentsHoldInMemory } = await import('./module-IYPLWHGC.mjs');
|
|
457
|
+
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
458
|
+
const segmentBytes = await bytesThatSegmentsHoldInMemory(directory, manifest);
|
|
459
|
+
requireRestoreMemory(indexName, archiveFootprint(prepared, manifest, replacesIndex, segmentBytes), replacesIndex);
|
|
460
|
+
}
|
|
572
461
|
await validateStagedCheckpoint(staging, manifest);
|
|
573
462
|
return { prepared, manifest };
|
|
574
463
|
}
|
|
@@ -751,9 +640,24 @@ async function checkpointForArchive(deps) {
|
|
|
751
640
|
});
|
|
752
641
|
}
|
|
753
642
|
async function createSnapshot(deps) {
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
643
|
+
for (let attempt = 0; attempt < MAX_SNAPSHOT_ANALYSIS_ATTEMPTS; attempt += 1) {
|
|
644
|
+
const checkpoint = await checkpointForArchive(deps);
|
|
645
|
+
try {
|
|
646
|
+
if ((checkpoint.manifest.analysisGeneration ?? 0) === deps.entry.analysisGeneration) {
|
|
647
|
+
const config = await encodeIndexConfiguration(deps.entry);
|
|
648
|
+
return archiveStream({ ...checkpoint, config, sourceIndexName: deps.indexName });
|
|
649
|
+
}
|
|
650
|
+
} catch (error) {
|
|
651
|
+
checkpoint.release();
|
|
652
|
+
throw error;
|
|
653
|
+
}
|
|
654
|
+
checkpoint.release();
|
|
655
|
+
}
|
|
656
|
+
throw new NarsilError(
|
|
657
|
+
ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
|
|
658
|
+
`The analysis of index "${deps.indexName}" changed while the engine wrote its snapshot, so try the snapshot again once the change has a durable checkpoint`,
|
|
659
|
+
{ indexName: deps.indexName }
|
|
660
|
+
);
|
|
757
661
|
}
|
|
758
662
|
function requireSnapshotSource(data) {
|
|
759
663
|
if (data instanceof Uint8Array) return data;
|
package/dist/synonyms.mjs
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
import { synonymSearchModule } from './chunk-
|
|
1
|
+
import { synonymSearchModule } from './chunk-O4LJQI4G.mjs';
|
|
2
2
|
import './chunk-X5CV2E2B.mjs';
|
|
3
|
-
import './chunk-
|
|
4
|
-
import './chunk-
|
|
3
|
+
import './chunk-WNBRW4Q2.mjs';
|
|
4
|
+
import './chunk-TGP5M5R7.mjs';
|
|
5
|
+
import './chunk-U2BKYEC7.mjs';
|
|
6
|
+
import './chunk-NPNK57CA.mjs';
|
|
5
7
|
import './chunk-4UMZ7KQL.mjs';
|
|
6
8
|
import './chunk-R6A773WK.mjs';
|
|
7
9
|
import './chunk-YTJCGKM4.mjs';
|
|
8
|
-
import './chunk-
|
|
10
|
+
import './chunk-HNHCNTVG.mjs';
|
|
9
11
|
|
|
10
12
|
// src/synonyms/index.ts
|
|
11
13
|
var synonyms = synonymSearchModule;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { I as IndexInfo, P as PartitionStatsResult, V as VectorMaintenanceResult, M as MemoryStats } from './config-
|
|
2
|
-
import { A as AnyDocument, I as InsertOptions, B as BM25Params, W as WriteOptions } from './schema-
|
|
1
|
+
import { I as IndexInfo, P as PartitionStatsResult, V as VectorMaintenanceResult, M as MemoryStats } from './config-BhZxIeb1.js';
|
|
2
|
+
import { A as AnyDocument, I as InsertOptions, B as BM25Params, W as WriteOptions } from './schema-B-Runv0n.js';
|
|
3
3
|
import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
|
|
4
|
-
import { N as NodeRole, c as NodeMemoryReport } from './index-
|
|
5
|
-
import { a as ClusterNodeInfo, N as NodeReadiness } from './config-
|
|
4
|
+
import { N as NodeRole, c as NodeMemoryReport } from './index-DsEurhSl.js';
|
|
5
|
+
import { a as ClusterNodeInfo, N as NodeReadiness } from './config-CIG5yDyg.js';
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* The body of `POST /indexes`, which the server sends once the index exists.
|
|
@@ -425,6 +425,20 @@ interface BatchBody {
|
|
|
425
425
|
/** These settings apply to the whole batch. */
|
|
426
426
|
options?: InsertOptions;
|
|
427
427
|
}
|
|
428
|
+
/**
|
|
429
|
+
* The body of `POST /indexes/{name}/_change-analysis`, where a key that you
|
|
430
|
+
* leave out keeps its current value.
|
|
431
|
+
*
|
|
432
|
+
* @public
|
|
433
|
+
*/
|
|
434
|
+
interface ChangeAnalysisBody {
|
|
435
|
+
/** This is the name of a language registered on the server. */
|
|
436
|
+
language?: string;
|
|
437
|
+
/** This is a name registered on the server, or `null` for the language default. */
|
|
438
|
+
tokenizer?: string | null;
|
|
439
|
+
/** This is a name registered on the server, a list of words, or `null` for the language default. */
|
|
440
|
+
stopWords?: string | string[] | null;
|
|
441
|
+
}
|
|
428
442
|
/**
|
|
429
443
|
* The body of `POST /indexes/{name}/_rebalance`.
|
|
430
444
|
*
|
|
@@ -459,10 +473,14 @@ interface HttpIndexConfig {
|
|
|
459
473
|
defaultScoring?: 'local' | 'dfs' | 'broadcast';
|
|
460
474
|
/** These parameters tune BM25 for this index and for each of its text fields. */
|
|
461
475
|
bm25?: BM25Params;
|
|
462
|
-
/** This
|
|
463
|
-
|
|
476
|
+
/** This is the name of a tokeniser registered on the server, which then splits text in place of the language's own rules. */
|
|
477
|
+
tokenizer?: string;
|
|
478
|
+
/** This name of a stop word set registered on the server, or this list, replaces the language's stop words. */
|
|
479
|
+
stopWords?: string | string[];
|
|
464
480
|
/** Setting this records term positions in each posting, which the `.nrsl` format carries for readers that match phrases. Highlighting works without it. */
|
|
465
481
|
trackPositions?: boolean;
|
|
482
|
+
/** Setting this to `disk` lets the index answer from segment files that the server reads by position, which needs filesystem durability. The default is `memory`. */
|
|
483
|
+
textStorage?: 'memory' | 'disk';
|
|
466
484
|
/** Setting this to false returns index stems from suggestions and prefix expansion, instead of the words users typed. */
|
|
467
485
|
surfaceForms?: boolean;
|
|
468
486
|
/** These settings control when vector fields move to an HNSW graph, and how that graph is built. */
|
|
@@ -551,7 +569,7 @@ interface MemoryStatsResponse extends MemoryStats {
|
|
|
551
569
|
*
|
|
552
570
|
* @public
|
|
553
571
|
*/
|
|
554
|
-
type TaskType = 'optimizeVectors' | 'rebalance' | 'restore' | 'import' | 'importExport' | 'rebuildAnalysis';
|
|
572
|
+
type TaskType = 'optimizeVectors' | 'rebalance' | 'restore' | 'import' | 'importExport' | 'rebuildAnalysis' | 'changeAnalysis';
|
|
555
573
|
/**
|
|
556
574
|
* Where a long-running operation stands.
|
|
557
575
|
*
|
|
@@ -701,4 +719,4 @@ interface TaskStore {
|
|
|
701
719
|
shutdown?(): Promise<void>;
|
|
702
720
|
}
|
|
703
721
|
|
|
704
|
-
export type {
|
|
722
|
+
export type { IndexPartitionsResponse as A, BatchBody as B, CapabilitiesResponse as C, DocumentBody as D, ErrorDetail as E, IndexStateResponse as F, MultiGetResponse as G, HttpIndexConfig as H, ImportError as I, ReadinessResponse as J, ReplicationFactorResponse as K, LivenessResponse as L, MemoryStatsResponse as M, VectorMaintenanceResponse as N, OkResponse as O, VersionResponse as P, BatchFailure as Q, RebalanceBody as R, SynonymSetBody as S, TaskListPage as T, ClusterMemberReport as U, VectorFieldBody as V, ClusterNodeReport as W, PartitionAllocationReport as X, ChangeAnalysisBody as a, CreateIndexEmbedding as b, ImportResult as c, TaskListQuery as d, TaskProgress as e, TaskRecord as f, TaskStatus as g, TaskType as h, TaskStore as i, CreateIndexRequest as j, InsertBody as k, MultiGetBody as l, ReplicationFactorBody as m, BatchResponse as n, ClusterAllocationResponse as o, ClusterTopologyResponse as p, DocumentCountResponse as q, DocumentExistsResponse as r, DocumentIdResponse as s, DocumentPutResponse as t, DocumentRemovedResponse as u, DocumentResponse as v, IndexClearedResponse as w, IndexCreatedResponse as x, IndexDroppedResponse as y, IndexListResponse as z };
|