@delali/narsil 0.3.1-next.143 → 0.3.1-next.145
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +101 -13
- package/dist/adapters/gcs.d.ts +49 -0
- package/dist/adapters/gcs.mjs +213 -0
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +2 -7
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/memory.mjs +5 -0
- package/dist/adapters/s3.d.ts +53 -0
- package/dist/adapters/s3.mjs +263 -0
- package/dist/{adapters-BQFQ4PRz.d.ts → adapters-pljAqmrC.d.ts} +33 -0
- package/dist/{analysis-change-HQUT2Z3I.mjs → analysis-change-D5APH4Z3.mjs} +24 -33
- package/dist/{browser/analysis-change-entry-7HR4LWL3.mjs → analysis-change-entry-V4SQQSXQ.mjs} +1 -1
- package/dist/browser/{analysis-change-JPV775LR.mjs → analysis-change-QTCQ5ETU.mjs} +5 -5
- package/dist/{analysis-change-entry-TZTG4AZB.mjs → browser/analysis-change-entry-KYEQSP3I.mjs} +1 -1
- package/dist/{chunk-46NZOPHS.mjs → browser/chunk-5WYWLTM5.mjs} +2 -2
- package/dist/browser/{chunk-ZEV26AQ6.mjs → chunk-6OFS7RUN.mjs} +1 -1
- package/dist/browser/{chunk-MZI3TEOW.mjs → chunk-75PMIFQA.mjs} +1 -1
- package/dist/browser/{chunk-R4WUDZQV.mjs → chunk-AOKLYVTT.mjs} +2 -2
- package/dist/browser/{chunk-LVY2LI35.mjs → chunk-BTMRV5MY.mjs} +2 -2
- package/dist/browser/{chunk-3ZBJYAS4.mjs → chunk-FHO5N7FD.mjs} +1 -1
- package/dist/browser/{chunk-KTOK275N.mjs → chunk-FXPMF65P.mjs} +12 -12
- package/dist/browser/{chunk-VFYMPCOV.mjs → chunk-GP4QLUQM.mjs} +5 -5
- package/dist/browser/{chunk-35B42UFM.mjs → chunk-IFGTRRPU.mjs} +9 -0
- package/dist/browser/{chunk-6RHTAVZX.mjs → chunk-K7UFXEJS.mjs} +31 -14
- package/dist/browser/{chunk-MVM33ZUL.mjs → chunk-LGJ6XMH5.mjs} +5 -5
- package/dist/{chunk-NN33CZRE.mjs → browser/chunk-PMBSBSCN.mjs} +2 -2
- package/dist/browser/{chunk-D33A5VMU.mjs → chunk-QNDWMPV2.mjs} +1 -1
- package/dist/browser/{chunk-HKX2G6WH.mjs → chunk-U77XQJOY.mjs} +3 -3
- package/dist/browser/{chunk-TCLEOZ4P.mjs → chunk-XIHLIMFX.mjs} +2 -2
- package/dist/browser/{chunk-WCFCHIBB.mjs → chunk-XWMEGKDC.mjs} +2 -2
- package/dist/browser/{chunk-5DB2VDGI.mjs → chunk-ZALEKEUH.mjs} +27 -27
- package/dist/browser/{chunk-MWUK3DA5.mjs → chunk-ZKVDQ5ZH.mjs} +8 -8
- package/dist/browser/{cold-module-MJMKNUYJ.mjs → cold-module-Y2IJAP7S.mjs} +14 -14
- package/dist/browser/{durability-wiring-SNOWIT6Z.mjs → durability-wiring-MLXZ5QQF.mjs} +78 -61
- package/dist/browser/import-FTMLTPGQ.mjs +9 -0
- package/dist/browser/{module-BUFNZMEU.mjs → module-5LMDGXQH.mjs} +10 -10
- package/dist/browser/{module-BVP4TKZU.mjs → module-Q6VFGLMG.mjs} +71 -11
- package/dist/browser/{phrase-module-HGVBO24W.mjs → phrase-module-KXNRTWCS.mjs} +1 -1
- package/dist/{restore-queue-FEYAC4SB.mjs → browser/restore-queue-SOAHKZSQ.mjs} +2 -2
- package/dist/browser/{runtime-VEUH674B.mjs → runtime-ZNEGK33G.mjs} +2 -2
- package/dist/browser/{snapshot-QXBKHJAG.mjs → snapshot-2EGS2FYU.mjs} +16 -15
- package/dist/browser/{write-WHFBDWQS.mjs → write-WMZA6NSF.mjs} +11 -11
- package/dist/{capabilities-BCdUwK_L.d.ts → capabilities-CNnzGtqP.d.ts} +3 -3
- package/dist/{chunk-ZMBFWQXN.mjs → chunk-2E2ZFOPT.mjs} +2 -2
- package/dist/{chunk-BOJZGUS6.mjs → chunk-5K2JHKM3.mjs} +1 -1
- package/dist/{chunk-YH5OIL7B.mjs → chunk-5QMHMELB.mjs} +6 -1
- package/dist/{browser/chunk-CCOAOO73.mjs → chunk-6FOJKNUR.mjs} +2 -2
- package/dist/{chunk-GLZHNSGB.mjs → chunk-6Z22L5RN.mjs} +1 -1
- package/dist/{chunk-AST3SHFA.mjs → chunk-7523VCM4.mjs} +2 -2
- package/dist/{chunk-KZL72VTC.mjs → chunk-7AJAW6NN.mjs} +11 -10
- package/dist/{chunk-DQNKVOJ2.mjs → chunk-BTARFTEN.mjs} +695 -8
- package/dist/{chunk-YLNSJQLX.mjs → chunk-DHEJAGVW.mjs} +5 -5
- package/dist/{chunk-Z4HFMYNX.mjs → chunk-DKMZ7JFW.mjs} +29 -12
- package/dist/{chunk-NRUXBYPH.mjs → chunk-F442VZUL.mjs} +5 -5
- package/dist/{chunk-O4LJQI4G.mjs → chunk-FRJ5RGX7.mjs} +126 -4
- package/dist/{chunk-L76W3IZJ.mjs → chunk-FTQLF5JI.mjs} +1 -1
- package/dist/{chunk-UIKN2OTQ.mjs → chunk-G6Q5DEF5.mjs} +37 -3
- package/dist/{chunk-DD2673TX.mjs → chunk-GRFXSEFF.mjs} +2 -2
- package/dist/{chunk-ZSWHPKGM.mjs → chunk-GVPYMSLO.mjs} +69 -2
- package/dist/{chunk-VEFN6X6Q.mjs → chunk-HOC5JP2H.mjs} +9 -10
- package/dist/{chunk-SRJBFG2T.mjs → chunk-HTMUGEFU.mjs} +6 -7
- package/dist/{chunk-J6YVB2XM.mjs → chunk-IJPK5PGM.mjs} +1 -1
- package/dist/{chunk-HNHCNTVG.mjs → chunk-IQHZMCLX.mjs} +9 -0
- package/dist/{chunk-C3YDYX6C.mjs → chunk-ISUOSACK.mjs} +1 -1
- package/dist/{chunk-YHUTF6EA.mjs → chunk-IV2XTSS2.mjs} +14 -14
- package/dist/chunk-K5HSUPYS.mjs +11 -0
- package/dist/{chunk-LA4WNDHL.mjs → chunk-KG5N7LE6.mjs} +12 -14
- package/dist/{chunk-NPNK57CA.mjs → chunk-LWFK3TC2.mjs} +1 -1
- package/dist/{chunk-RKAVHNRW.mjs → chunk-LYKUJNAV.mjs} +10 -5
- package/dist/{chunk-TNBIK2LQ.mjs → chunk-MJTEU2XK.mjs} +3 -4
- package/dist/chunk-NWTEJ2T7.mjs +100 -0
- package/dist/{chunk-WJGO5PGV.mjs → chunk-OH7SX5MY.mjs} +28 -16
- package/dist/{chunk-W2COJ5AH.mjs → chunk-ONDEEBTA.mjs} +2 -2
- package/dist/{chunk-PWIWSUCX.mjs → chunk-POUYICOU.mjs} +1 -1
- package/dist/chunk-PRID7CA6.mjs +104 -0
- package/dist/{chunk-ICW45TTU.mjs → chunk-QLDLQW36.mjs} +1 -1
- package/dist/{chunk-KEIYVZUX.mjs → chunk-QU2XF7VI.mjs} +3 -4
- package/dist/{chunk-74XB2SXI.mjs → chunk-QXH2CS4V.mjs} +35 -2
- package/dist/{chunk-ZPWS2SKJ.mjs → chunk-QYMAYU7M.mjs} +165 -11
- package/dist/{chunk-NJJCPFXN.mjs → chunk-RNUP6WZ6.mjs} +2 -2
- package/dist/{chunk-SBMAWTRG.mjs → chunk-RTVAWNUL.mjs} +80 -14
- package/dist/{chunk-7AUASRE7.mjs → chunk-RVBUQITB.mjs} +7 -7
- package/dist/{chunk-VJEGOELZ.mjs → chunk-SA52N7G3.mjs} +3 -3
- package/dist/{chunk-NOT5QOHU.mjs → chunk-SE7MKJVK.mjs} +1 -1
- package/dist/{chunk-BRUUBXCB.mjs → chunk-SNB4DYL3.mjs} +10 -38
- package/dist/chunk-SVO6NV4D.mjs +248 -0
- package/dist/{chunk-GWOS5DYC.mjs → chunk-UGQULMSU.mjs} +11 -13
- package/dist/{chunk-6SNBQESJ.mjs → chunk-UHVTEVF5.mjs} +5 -5
- package/dist/{chunk-HJJWY6PT.mjs → chunk-UQS5PAS5.mjs} +1 -1
- package/dist/{chunk-UUZCQQG2.mjs → chunk-W6M4RRAT.mjs} +3 -3
- package/dist/{browser/chunk-XNWPP5NS.mjs → chunk-WDEWU46C.mjs} +2 -2
- package/dist/{chunk-XZ3SRNDJ.mjs → chunk-WMIV7PJ7.mjs} +65 -5
- package/dist/{chunk-RHKRTYGO.mjs → chunk-X5SBWARI.mjs} +3 -3
- package/dist/{chunk-XS2VM7UE.mjs → chunk-XLAI4OIO.mjs} +2 -2
- package/dist/{chunk-IO5F7ZBE.mjs → chunk-YUCGFPMS.mjs} +454 -37
- package/dist/{client-B12YFALP.d.ts → client-DoYPHcUn.d.ts} +62 -5
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +44 -0
- package/dist/{cold-module-4X7WJCFM.mjs → cold-module-NUD332YD.mjs} +32 -41
- package/dist/{config-CIG5yDyg.d.ts → config-CuUfls_H.d.ts} +150 -3
- package/dist/{config-BhZxIeb1.d.ts → config-Dzs46im6.d.ts} +11 -2
- package/dist/direct-executor-FDQYQEJQ.mjs +22 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +3 -3
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +16219 -14535
- package/dist/{durability-wiring-324D2DDD.mjs → durability-wiring-K7RDXWL6.mjs} +103 -95
- package/dist/embeddings/openai.d.ts +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-CtzbHOmG.d.ts → engine-C0ifUEcm.d.ts} +3 -3
- package/dist/import-6KXFY6XI.mjs +35 -0
- package/dist/{index-DsEurhSl.d.ts → index-MsF7e5hL.d.ts} +2 -2
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +21 -30
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-4SW7RWDE.mjs → long-query-thread-FKVHRCHF.mjs} +18 -23
- package/dist/{module-DXSJ5LT2.mjs → module-GJUKW5VU.mjs} +14 -16
- package/dist/{module-IYPLWHGC.mjs → module-TMFIWOD2.mjs} +28 -37
- package/dist/{paged-partition-archive-HFP45PLU.mjs → paged-partition-archive-WNUEMYWM.mjs} +34 -39
- package/dist/partial-words.mjs +7 -9
- package/dist/pattern.mjs +4 -6
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +27 -36
- package/dist/{phrase-module-5QHBD2VJ.mjs → phrase-module-OMY3QQV7.mjs} +1 -1
- package/dist/react.d.ts +11 -11
- package/dist/react.mjs +9 -0
- package/dist/{rerank-KTLC3E6N.mjs → rerank-ZZ5CHKLW.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-ONYNYUDE.mjs → restore-queue-UAVGBSJG.mjs} +2 -2
- package/dist/{runtime-DYV75DCB.mjs → runtime-RAOCKSYA.mjs} +5 -5
- package/dist/{schema-B-Runv0n.d.ts → schema-D33CyXCy.d.ts} +1 -1
- package/dist/server.d.ts +24 -12
- package/dist/server.mjs +153 -36
- package/dist/{snapshot-CGM65VCD.mjs → snapshot-DB5ZVCDN.mjs} +39 -48
- package/dist/synonyms.mjs +3 -4
- package/dist/{tasks-CjRJRGTb.d.ts → tasks-ChWp0wrF.d.ts} +59 -7
- package/dist/{thread-KGNAIKPD.mjs → thread-OV6T555H.mjs} +17 -22
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/vector/search-worker.mjs +5 -6
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-2DZPEQL2.mjs → write-KZVTLGCI.mjs} +28 -37
- package/openapi.json +349 -2
- package/package.json +27 -8
- package/dist/browser/import-TLB2A5LJ.mjs +0 -9
- package/dist/chunk-5WYVUPGD.mjs +0 -427
- package/dist/chunk-AV4IJKD2.mjs +0 -35
- package/dist/chunk-GES7U2PM.mjs +0 -68
- package/dist/chunk-KGOXHLLZ.mjs +0 -693
- package/dist/chunk-N37VMC62.mjs +0 -18
- package/dist/chunk-QSK35UC2.mjs +0 -35
- package/dist/chunk-WNBRW4Q2.mjs +0 -128
- package/dist/chunk-WQBCBKY3.mjs +0 -8
- package/dist/chunk-ZATRDAYU.mjs +0 -160
- package/dist/direct-executor-7XOQ7VQP.mjs +0 -28
- package/dist/import-OOBUW3RR.mjs +0 -44
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { withRerankQueryText } from './chunk-KZ3NX2P7.mjs';
|
|
2
|
-
import { sendRelayed, parseJson, validateQuery, rejectInvalid, refuseRegex, respondJson, withServerRerankWindow, respondError, loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedBodyBudget, createSharedRequestGate, registerServerRoutes, closeWhenTheProcessExits, statusLine, sendError, REQUEST_THREAD_MOVE_AFTER_MS, MAX_LONG_QUERIES_IN_FLIGHT, createDocumentReadHandlers, SERVER_CAPABILITIES, versionReport, serverRoutes, validateSuggest, SEARCH_CLOCK_SAMPLE_UNITS, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, encodeJson } from './chunk-
|
|
3
|
-
import { executeListDocuments, executeSuggest, executeQuery, lazyQueryBinding, executePreflight } from './chunk-
|
|
4
|
-
import { holdsQuotedText, resolveProjection, projectionKeepsField, setNestedValue } from './chunk-
|
|
5
|
-
import { extractVectorFieldsFromSchema } from './chunk-
|
|
6
|
-
import { ServerErrorCodes, ErrorCodes, NarsilError, isNarsilError } from './chunk-
|
|
2
|
+
import { sendRelayed, parseJson, validateQuery, rejectInvalid, refuseRegex, respondJson, withServerRerankWindow, respondError, loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedBodyBudget, createSharedRequestGate, registerServerRoutes, closeWhenTheProcessExits, statusLine, sendError, REQUEST_THREAD_MOVE_AFTER_MS, MAX_LONG_QUERIES_IN_FLIGHT, createDocumentReadHandlers, SERVER_CAPABILITIES, versionReport, serverRoutes, validateSuggest, SEARCH_CLOCK_SAMPLE_UNITS, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, encodeJson } from './chunk-RTVAWNUL.mjs';
|
|
3
|
+
import { executeListDocuments, executeSuggest, executeQuery, lazyQueryBinding, executePreflight } from './chunk-HOC5JP2H.mjs';
|
|
4
|
+
import { holdsQuotedText, resolveProjection, projectionKeepsField, setNestedValue } from './chunk-QYMAYU7M.mjs';
|
|
5
|
+
import { extractVectorFieldsFromSchema } from './chunk-BTARFTEN.mjs';
|
|
6
|
+
import { ServerErrorCodes, ErrorCodes, NarsilError, isNarsilError } from './chunk-IQHZMCLX.mjs';
|
|
7
7
|
|
|
8
8
|
// src/server/request-threads/messages.ts
|
|
9
9
|
function isLongQueryPortNote(message) {
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-
|
|
2
|
-
import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes, createOsqScratch, osqQuantize } from './chunk-
|
|
1
|
+
import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-QLDLQW36.mjs';
|
|
2
|
+
import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes, createOsqScratch, osqQuantize } from './chunk-MJTEU2XK.mjs';
|
|
3
3
|
import { osqRecordBytes, writeRecord } from './chunk-37CD3OKS.mjs';
|
|
4
4
|
import { MAX_DOC_ID_LENGTH } from './chunk-EEBCDP36.mjs';
|
|
5
5
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
6
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
6
|
+
import { NarsilError, ErrorCodes } from './chunk-IQHZMCLX.mjs';
|
|
7
7
|
|
|
8
8
|
// src/vector/vector-index/constants.ts
|
|
9
9
|
var DEFAULT_PROMOTION_THRESHOLD = 1024;
|
|
@@ -279,6 +279,12 @@ function sameCentroid(state, planned) {
|
|
|
279
279
|
}
|
|
280
280
|
return true;
|
|
281
281
|
}
|
|
282
|
+
function codesOfReadVectors(state, centroid, vectors, ordinals) {
|
|
283
|
+
const quantizer = state.osq;
|
|
284
|
+
const layout = state.store.handles.codeLayout;
|
|
285
|
+
if (centroid === null || quantizer === null || layout === null) return null;
|
|
286
|
+
return codesAgainstPlannedCentroid(quantizer, layout.slotStride, centroid, vectors, ordinals);
|
|
287
|
+
}
|
|
282
288
|
function codesAgainstPlannedCentroid(quantizer, slotStride, centroid, vectors, ordinals) {
|
|
283
289
|
const planned = Float32Array.from(centroid);
|
|
284
290
|
const dimension = planned.length;
|
|
@@ -567,7 +573,11 @@ function plannedVectorOf(state, ordinal) {
|
|
|
567
573
|
const docId = state.store.docIdForOrdinal(ordinal) ?? state.store.view.docIdAt(ordinal) ?? "";
|
|
568
574
|
return { ordinal, docId, tombstoned: !isLive(state, ordinal) };
|
|
569
575
|
}
|
|
570
|
-
function
|
|
576
|
+
function heldOrdinalOf(state, vector, pinned) {
|
|
577
|
+
if (pinned || vector.tombstoned || state.store.holdsOrdinal(vector.ordinal)) return vector.ordinal;
|
|
578
|
+
return state.store.getOrdinal(vector.docId);
|
|
579
|
+
}
|
|
580
|
+
function readFile(state, planned, centroid, pinned) {
|
|
571
581
|
const dimension = state.dimension;
|
|
572
582
|
const vectors = new Float32Array(planned.length * dimension);
|
|
573
583
|
const magnitudes = new Float64Array(planned.length);
|
|
@@ -575,14 +585,14 @@ function readFile(state, planned, centroid) {
|
|
|
575
585
|
const tombstonedAt = /* @__PURE__ */ new Map();
|
|
576
586
|
const docIds = planned.map((vector) => vector.docId);
|
|
577
587
|
for (let i = 0; i < planned.length; i++) {
|
|
578
|
-
const
|
|
579
|
-
|
|
580
|
-
const entry =
|
|
581
|
-
if (
|
|
588
|
+
const held = heldOrdinalOf(state, planned[i], pinned);
|
|
589
|
+
if (held === void 0) continue;
|
|
590
|
+
const entry = pinned ? state.store.entryForPinnedOrdinal(held) : state.store.entryForOrdinal(held);
|
|
591
|
+
if (entry === void 0) continue;
|
|
582
592
|
vectors.set(entry.vector, i * dimension);
|
|
583
593
|
magnitudes[i] = entry.magnitude;
|
|
584
594
|
ordinals[i] = held;
|
|
585
|
-
if (tombstoned) tombstonedAt.set(i, held);
|
|
595
|
+
if (planned[i].tombstoned) tombstonedAt.set(i, held);
|
|
586
596
|
}
|
|
587
597
|
return {
|
|
588
598
|
payload: {
|
|
@@ -590,7 +600,7 @@ function readFile(state, planned, centroid) {
|
|
|
590
600
|
fieldName: state.fieldName,
|
|
591
601
|
dimension,
|
|
592
602
|
docIds,
|
|
593
|
-
codes: partCodes(state, docIds, centroid, tombstonedAt, vectors, ordinals),
|
|
603
|
+
codes: pinned ? codesOfReadVectors(state, centroid, vectors, ordinals) : partCodes(state, docIds, centroid, tombstonedAt, vectors, ordinals),
|
|
594
604
|
vectors: vectorsToBytes(vectors)
|
|
595
605
|
},
|
|
596
606
|
ordinals,
|
|
@@ -598,7 +608,9 @@ function readFile(state, planned, centroid) {
|
|
|
598
608
|
magnitudes
|
|
599
609
|
};
|
|
600
610
|
}
|
|
601
|
-
function planCheckpoint(state, listedKeys, rewritesGraph = false) {
|
|
611
|
+
function planCheckpoint(state, listedKeys, rewritesGraph = false, pinsRetiredVectors = false) {
|
|
612
|
+
if (pinsRetiredVectors) state.store.pinRetiredVectors();
|
|
613
|
+
let pinned = pinsRetiredVectors;
|
|
602
614
|
const signature = signatureOf(state);
|
|
603
615
|
const calibration = signature.calibration;
|
|
604
616
|
const trusted = registryMatches(state, listedKeys);
|
|
@@ -639,11 +651,16 @@ function planCheckpoint(state, listedKeys, rewritesGraph = false) {
|
|
|
639
651
|
readNewFile(index) {
|
|
640
652
|
const start = index * capacity;
|
|
641
653
|
const end = start + capacity;
|
|
642
|
-
const file = readFile(state, planned.slice(start, end), centroid);
|
|
654
|
+
const file = readFile(state, planned.slice(start, end), centroid, pinsRetiredVectors);
|
|
643
655
|
for (let i = 0; i < file.ordinals.length; i++) {
|
|
644
656
|
if (file.ordinals[i] === NO_LIVE_VECTOR_AT_POSITION) vanished.add(firstWrittenNumber + start + i);
|
|
645
657
|
}
|
|
646
658
|
return file;
|
|
659
|
+
},
|
|
660
|
+
release() {
|
|
661
|
+
if (!pinned) return;
|
|
662
|
+
pinned = false;
|
|
663
|
+
state.store.unpinRetiredVectors();
|
|
647
664
|
}
|
|
648
665
|
};
|
|
649
666
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead } from './chunk-
|
|
2
|
-
import { readListedFile, requireListedChecksum, requireListedSize } from './chunk-
|
|
3
|
-
import { HEADER_SIZE, openChecksummedEnvelope } from './chunk-
|
|
4
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead } from './chunk-QLDLQW36.mjs';
|
|
2
|
+
import { readListedFile, requireListedChecksum, requireListedSize } from './chunk-W6M4RRAT.mjs';
|
|
3
|
+
import { HEADER_SIZE, openChecksummedEnvelope } from './chunk-YUCGFPMS.mjs';
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-IQHZMCLX.mjs';
|
|
5
5
|
import { decode } from '@msgpack/msgpack';
|
|
6
6
|
|
|
7
7
|
var VECTOR_FILE_ENVELOPE_VERSION = 2;
|
|
@@ -64,7 +64,7 @@ function holdsEveryHead(ref) {
|
|
|
64
64
|
async function loadFieldOnDemand(store, ref, vectorIndex, resolvePartition) {
|
|
65
65
|
const { pathOf } = store;
|
|
66
66
|
if (pathOf === void 0 || vectorIndex.storage !== "disk" || !holdsEveryHead(ref)) return false;
|
|
67
|
-
const { loadFilesByParts, openColdVectorField, osqBitsOf, waitBeforeColdLoad } = await import('./cold-module-
|
|
67
|
+
const { loadFilesByParts, openColdVectorField, osqBitsOf, waitBeforeColdLoad } = await import('./cold-module-NUD332YD.mjs');
|
|
68
68
|
await Promise.all([
|
|
69
69
|
...ref.files.map((file) => requireListedSize(store, file.key, { bytes: file.bytes, what: "vector file" })),
|
|
70
70
|
...ref.graphParts.map((part) => requireListedSize(store, part.key, { bytes: part.bytes, what: "graph part" }))
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-
|
|
1
|
+
import { english } from './chunk-TGP5M5R7.mjs';
|
|
2
|
+
import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-LWFK3TC2.mjs';
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-IQHZMCLX.mjs';
|
|
3
4
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
4
|
-
import {
|
|
5
|
+
import { decode, encode } from '@msgpack/msgpack';
|
|
5
6
|
|
|
6
7
|
// src/analysis/registry.ts
|
|
7
8
|
var tokenizers = /* @__PURE__ */ new Map();
|
|
@@ -78,6 +79,20 @@ function resolveIndexAnalysis(config) {
|
|
|
78
79
|
};
|
|
79
80
|
}
|
|
80
81
|
|
|
82
|
+
// src/languages/registry.ts
|
|
83
|
+
var languages = /* @__PURE__ */ new Map();
|
|
84
|
+
languages.set("english", english);
|
|
85
|
+
function registerLanguage(module) {
|
|
86
|
+
languages.set(module.name, module);
|
|
87
|
+
}
|
|
88
|
+
function getLanguage(name) {
|
|
89
|
+
const lang = languages.get(name);
|
|
90
|
+
if (!lang) {
|
|
91
|
+
throw new NarsilError(ErrorCodes.LANGUAGE_NOT_SUPPORTED, `Language "${name}" is not registered`, { language: name });
|
|
92
|
+
}
|
|
93
|
+
return lang;
|
|
94
|
+
}
|
|
95
|
+
|
|
81
96
|
// src/engine/analysis-change/pending-target.ts
|
|
82
97
|
function configUnderAnalysis(base, language, tokenizer, stopWords) {
|
|
83
98
|
const { tokenizer: _tokenizer, stopWords: _stopWords, ...rest } = base;
|
|
@@ -127,6 +142,16 @@ async function oneAtATime(core, indexName, action) {
|
|
|
127
142
|
}
|
|
128
143
|
}
|
|
129
144
|
|
|
145
|
+
// src/core/synonyms/constants.ts
|
|
146
|
+
var MAX_SYNONYM_RULES = 1e5;
|
|
147
|
+
var MAX_SYNONYM_WORDS = 5e5;
|
|
148
|
+
var MAX_SYNONYM_WORD_BYTES = 4194304;
|
|
149
|
+
var MAX_SYNONYM_RULE_WORDS = 100;
|
|
150
|
+
var MIN_TWO_WAY_SYNONYM_WORDS = 2;
|
|
151
|
+
var MIN_ONE_WAY_SYNONYMS = 1;
|
|
152
|
+
var SYNONYMS_ADDED_PER_QUERY = 50;
|
|
153
|
+
var SYNONYM_EXPANSION_CACHE_ENTRIES = 1024;
|
|
154
|
+
|
|
130
155
|
// src/core/synonyms/combined-postings.ts
|
|
131
156
|
var NO_DELETED = { size: 0, has: () => false };
|
|
132
157
|
var FIELD_SLOTS = 65536;
|
|
@@ -384,6 +409,103 @@ function buildSynonymLookup(rules, analysis, raise) {
|
|
|
384
409
|
if (ruleTokenIds.length === 0) return null;
|
|
385
410
|
return packedLookup(interned.tokens, ruleStart, ruleTokenIds, ruleSource);
|
|
386
411
|
}
|
|
412
|
+
var FIRST_PRINTABLE_CODE = 32;
|
|
413
|
+
function holdsControlCharacter(word) {
|
|
414
|
+
for (let i = 0; i < word.length; i++) if (word.charCodeAt(i) < FIRST_PRINTABLE_CODE) return true;
|
|
415
|
+
return false;
|
|
416
|
+
}
|
|
417
|
+
function utf8Length(word) {
|
|
418
|
+
let bytes = 0;
|
|
419
|
+
for (let i = 0; i < word.length; i++) {
|
|
420
|
+
const unit = word.charCodeAt(i);
|
|
421
|
+
if (unit < 128) bytes += 1;
|
|
422
|
+
else if (unit < 2048) bytes += 2;
|
|
423
|
+
else if (unit >= 55296 && unit < 56320 && i + 1 < word.length) {
|
|
424
|
+
const low = word.charCodeAt(i + 1);
|
|
425
|
+
if (low >= 56320 && low < 57344) {
|
|
426
|
+
bytes += 4;
|
|
427
|
+
i++;
|
|
428
|
+
} else bytes += 3;
|
|
429
|
+
} else bytes += 3;
|
|
430
|
+
}
|
|
431
|
+
return bytes;
|
|
432
|
+
}
|
|
433
|
+
function hasOnlyKeys(value, keys) {
|
|
434
|
+
const own = Object.keys(value);
|
|
435
|
+
return own.length === keys.length && keys.every((key) => Object.hasOwn(value, key));
|
|
436
|
+
}
|
|
437
|
+
function readWord(value, rule, tally, raise) {
|
|
438
|
+
if (typeof value !== "string") {
|
|
439
|
+
raise(`Synonym rule ${rule} holds a word that is not a string`, { rule });
|
|
440
|
+
}
|
|
441
|
+
if (holdsControlCharacter(value)) {
|
|
442
|
+
raise(`Synonym rule ${rule} holds a word with a control character`, { rule });
|
|
443
|
+
}
|
|
444
|
+
tally.words++;
|
|
445
|
+
tally.bytes += utf8Length(value);
|
|
446
|
+
if (tally.words > MAX_SYNONYM_WORDS) {
|
|
447
|
+
raise(`A synonym set holds at most ${MAX_SYNONYM_WORDS} words`, { maxWords: MAX_SYNONYM_WORDS });
|
|
448
|
+
}
|
|
449
|
+
if (tally.bytes > MAX_SYNONYM_WORD_BYTES) {
|
|
450
|
+
raise(`The words of a synonym set hold at most ${MAX_SYNONYM_WORD_BYTES} bytes in UTF-8`, {
|
|
451
|
+
maxWordBytes: MAX_SYNONYM_WORD_BYTES
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
return value;
|
|
455
|
+
}
|
|
456
|
+
function readWords(value, rule, key, minimum, maximum, tally, raise) {
|
|
457
|
+
if (!Array.isArray(value) || value.length < minimum || value.length > maximum) {
|
|
458
|
+
raise(`Synonym rule ${rule} needs "${key}" as a list of ${minimum} to ${maximum} words`, { rule });
|
|
459
|
+
}
|
|
460
|
+
const words = new Array(value.length);
|
|
461
|
+
for (let i = 0; i < value.length; i++) words[i] = readWord(value[i], rule, tally, raise);
|
|
462
|
+
return words;
|
|
463
|
+
}
|
|
464
|
+
function readRule(value, rule, tally, raise) {
|
|
465
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
466
|
+
raise(`Synonym rule ${rule} must be an object`, { rule });
|
|
467
|
+
}
|
|
468
|
+
if (hasOnlyKeys(value, ["words"])) {
|
|
469
|
+
const words = Reflect.get(value, "words");
|
|
470
|
+
return { words: readWords(words, rule, "words", MIN_TWO_WAY_SYNONYM_WORDS, MAX_SYNONYM_RULE_WORDS, tally, raise) };
|
|
471
|
+
}
|
|
472
|
+
if (hasOnlyKeys(value, ["word", "synonyms"])) {
|
|
473
|
+
const word = readWord(Reflect.get(value, "word"), rule, tally, raise);
|
|
474
|
+
const synonyms = readWords(
|
|
475
|
+
Reflect.get(value, "synonyms"),
|
|
476
|
+
rule,
|
|
477
|
+
"synonyms",
|
|
478
|
+
MIN_ONE_WAY_SYNONYMS,
|
|
479
|
+
MAX_SYNONYM_RULE_WORDS - 1,
|
|
480
|
+
tally,
|
|
481
|
+
raise
|
|
482
|
+
);
|
|
483
|
+
return { word, synonyms };
|
|
484
|
+
}
|
|
485
|
+
raise(`Synonym rule ${rule} must hold either "words" alone, or "word" and "synonyms"`, { rule });
|
|
486
|
+
}
|
|
487
|
+
function parseSynonymRules(value, raise) {
|
|
488
|
+
if (!Array.isArray(value)) raise("A synonym set must be a list of rules", {});
|
|
489
|
+
if (value.length > MAX_SYNONYM_RULES) {
|
|
490
|
+
raise(`A synonym set holds at most ${MAX_SYNONYM_RULES} rules`, { maxRules: MAX_SYNONYM_RULES });
|
|
491
|
+
}
|
|
492
|
+
const tally = { words: 0, bytes: 0 };
|
|
493
|
+
const rules = new Array(value.length);
|
|
494
|
+
for (let rule = 0; rule < value.length; rule++) rules[rule] = readRule(value[rule], rule, tally, raise);
|
|
495
|
+
return rules;
|
|
496
|
+
}
|
|
497
|
+
function encodeSynonymRules(rules) {
|
|
498
|
+
return encode(rules);
|
|
499
|
+
}
|
|
500
|
+
function decodeSynonymRules(bytes, raise) {
|
|
501
|
+
let decoded;
|
|
502
|
+
try {
|
|
503
|
+
decoded = decode(bytes);
|
|
504
|
+
} catch {
|
|
505
|
+
raise("A stored synonym set is not valid MessagePack", {});
|
|
506
|
+
}
|
|
507
|
+
return parseSynonymRules(decoded, raise);
|
|
508
|
+
}
|
|
387
509
|
|
|
388
510
|
// src/core/synonyms/prepare.ts
|
|
389
511
|
function invalidIn(source) {
|
|
@@ -467,4 +589,4 @@ var synonymSearchModule = {
|
|
|
467
589
|
applyLoaded: applyLoadedSynonyms
|
|
468
590
|
};
|
|
469
591
|
|
|
470
|
-
export { configOfNamedSettings, configUnderAnalysis, getStopWords, getTokenizer, hasStopWords, hasTokenizer, oneAtATime, registerStopWords, registerTokenizer, resolveIndexAnalysis, synonymSearchModule, targetOfPending, withOpenIndex };
|
|
592
|
+
export { configOfNamedSettings, configUnderAnalysis, decodeSynonymRules, getLanguage, getStopWords, getTokenizer, hasStopWords, hasTokenizer, oneAtATime, registerLanguage, registerStopWords, registerTokenizer, resolveIndexAnalysis, synonymSearchModule, targetOfPending, withOpenIndex };
|
|
@@ -1,5 +1,39 @@
|
|
|
1
|
-
import { SCRATCH_SLOTS_PER_THREAD_POOL } from './chunk-
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { SCRATCH_SLOTS_PER_THREAD_POOL } from './chunk-QXH2CS4V.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-IQHZMCLX.mjs';
|
|
3
|
+
|
|
4
|
+
// src/runtime/detect.ts
|
|
5
|
+
function detectRuntimeName() {
|
|
6
|
+
if ("Bun" in globalThis) {
|
|
7
|
+
return "bun";
|
|
8
|
+
}
|
|
9
|
+
if ("Deno" in globalThis) {
|
|
10
|
+
return "deno";
|
|
11
|
+
}
|
|
12
|
+
const proc = globalThis.process;
|
|
13
|
+
if (typeof proc === "object" && proc !== null && typeof proc.versions === "object" && typeof proc.versions?.node === "string") {
|
|
14
|
+
return "node";
|
|
15
|
+
}
|
|
16
|
+
return "browser";
|
|
17
|
+
}
|
|
18
|
+
function getCpuCount() {
|
|
19
|
+
const nav = globalThis.navigator;
|
|
20
|
+
if (nav && typeof nav.hardwareConcurrency === "number" && nav.hardwareConcurrency >= 1) {
|
|
21
|
+
return nav.hardwareConcurrency;
|
|
22
|
+
}
|
|
23
|
+
return 1;
|
|
24
|
+
}
|
|
25
|
+
function detectRuntime() {
|
|
26
|
+
const runtime = detectRuntimeName();
|
|
27
|
+
return {
|
|
28
|
+
runtime,
|
|
29
|
+
cpuCount: getCpuCount(),
|
|
30
|
+
supportsWorkerThreads: runtime === "node" || runtime === "bun",
|
|
31
|
+
supportsWebWorkers: runtime === "browser" || runtime === "deno" || runtime === "bun",
|
|
32
|
+
supportsFileSystem: runtime === "node" || runtime === "bun" || runtime === "deno",
|
|
33
|
+
supportsIndexedDB: runtime === "browser" || "indexedDB" in globalThis,
|
|
34
|
+
supportsBroadcastChannel: "BroadcastChannel" in globalThis
|
|
35
|
+
};
|
|
36
|
+
}
|
|
3
37
|
|
|
4
38
|
// src/workers/constants.ts
|
|
5
39
|
var WORKER_YOUNG_GENERATION_MB = 24;
|
|
@@ -64,4 +98,4 @@ function workerResourceLimits() {
|
|
|
64
98
|
return { maxYoungGenerationSizeMb: WORKER_YOUNG_GENERATION_MB };
|
|
65
99
|
}
|
|
66
100
|
|
|
67
|
-
export { CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, missingWorkerEntry, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, splitWorkerBudget, workerResourceLimits };
|
|
101
|
+
export { CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, detectRuntime, missingWorkerEntry, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, splitWorkerBudget, workerResourceLimits };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { admittedHeapBytes, collectGarbageBeforeAnswerRefusal, collectYoungGenerationBeforeRefusal, reservedHeapBytes, admitHeapBytes } from './chunk-IWVEG5CX.mjs';
|
|
2
|
-
import { readOldGenerationStatistics, processLimitBytes } from './chunk-
|
|
2
|
+
import { readOldGenerationStatistics, processLimitBytes } from './chunk-QYMAYU7M.mjs';
|
|
3
3
|
import { engineThreadYoungGenerationBytes } from './chunk-EBIHRL7B.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes, ServerErrorCodes } from './chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes, ServerErrorCodes } from './chunk-IQHZMCLX.mjs';
|
|
5
5
|
|
|
6
6
|
// src/distribution/cluster-node/constants.ts
|
|
7
7
|
var DEFAULT_PARTITION_COUNT = 5;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { PARSED_FIELD_TYPE_CACHE_ENTRIES, VECTOR_PATTERN, MAX_STORED_VECTOR_DIMENSION } from './chunk-L5ORED7V.mjs';
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-IQHZMCLX.mjs';
|
|
3
3
|
|
|
4
4
|
// src/schema/validator/field-type.ts
|
|
5
5
|
var FIELD_OPTION_ORDER = ["sortable", "pattern", "partial"];
|
|
@@ -114,4 +114,71 @@ function isStringValuedBase(base) {
|
|
|
114
114
|
return base === "string" || base === "string[]" || base === "verbatim" || base === "verbatim[]";
|
|
115
115
|
}
|
|
116
116
|
|
|
117
|
-
|
|
117
|
+
// src/core/fuzzy.ts
|
|
118
|
+
var FIRST_SURROGATE = 55296;
|
|
119
|
+
var LAST_SURROGATE = 57343;
|
|
120
|
+
var LAST_BASIC_PLANE_CODE_POINT = 65535;
|
|
121
|
+
function hasSurrogate(text) {
|
|
122
|
+
for (let index = 0; index < text.length; index++) {
|
|
123
|
+
const unit = text.charCodeAt(index);
|
|
124
|
+
if (unit >= FIRST_SURROGATE && unit <= LAST_SURROGATE) return true;
|
|
125
|
+
}
|
|
126
|
+
return false;
|
|
127
|
+
}
|
|
128
|
+
function fuzzyPrefixOf(queryTerm, prefixLength) {
|
|
129
|
+
let end = 0;
|
|
130
|
+
for (let counted = 0; counted < prefixLength && end < queryTerm.length; counted++) {
|
|
131
|
+
const codePoint = queryTerm.codePointAt(end);
|
|
132
|
+
end += codePoint !== void 0 && codePoint > LAST_BASIC_PLANE_CODE_POINT ? 2 : 1;
|
|
133
|
+
}
|
|
134
|
+
return queryTerm.slice(0, end);
|
|
135
|
+
}
|
|
136
|
+
function fuzzyTermMatches(queryTerm, indexTerm, tolerance, prefixLength) {
|
|
137
|
+
if (queryTerm === indexTerm) return true;
|
|
138
|
+
if (tolerance <= 0) return false;
|
|
139
|
+
if (!indexTerm.startsWith(fuzzyPrefixOf(queryTerm, prefixLength))) return false;
|
|
140
|
+
return boundedLevenshtein(queryTerm, indexTerm, tolerance).withinTolerance;
|
|
141
|
+
}
|
|
142
|
+
function boundedLevenshtein(a, b, tolerance) {
|
|
143
|
+
if (tolerance < 0) return { distance: -1, withinTolerance: false, cells: 0 };
|
|
144
|
+
if (a === b) return { distance: 0, withinTolerance: true, cells: 0 };
|
|
145
|
+
return tabulatedLevenshtein(a, b, tolerance);
|
|
146
|
+
}
|
|
147
|
+
function tabulatedLevenshtein(a, b, tolerance) {
|
|
148
|
+
if (tolerance < 0) return { distance: -1, withinTolerance: false, cells: 0 };
|
|
149
|
+
if (hasSurrogate(a) || hasSurrogate(b)) return bandedDistance(Array.from(a), Array.from(b), tolerance);
|
|
150
|
+
return bandedDistance(a, b, tolerance);
|
|
151
|
+
}
|
|
152
|
+
function bandedDistance(a, b, tolerance) {
|
|
153
|
+
const m = a.length;
|
|
154
|
+
const n = b.length;
|
|
155
|
+
const beyond = tolerance + 1;
|
|
156
|
+
if (Math.abs(m - n) > tolerance) return { distance: beyond, withinTolerance: false, cells: 0 };
|
|
157
|
+
let previous = new Uint32Array(n + 1).fill(beyond);
|
|
158
|
+
let current = new Uint32Array(n + 1).fill(beyond);
|
|
159
|
+
const firstRowEnd = Math.min(n, tolerance);
|
|
160
|
+
for (let j = 0; j <= firstRowEnd; j++) previous[j] = j;
|
|
161
|
+
let cells = 0;
|
|
162
|
+
for (let i = 1; i <= m; i++) {
|
|
163
|
+
const low = Math.max(1, i - tolerance);
|
|
164
|
+
const high = Math.min(n, i + tolerance);
|
|
165
|
+
current[low - 1] = low === 1 ? i : beyond;
|
|
166
|
+
let rowMin = current[low - 1];
|
|
167
|
+
const character = a[i - 1];
|
|
168
|
+
if (high >= low) cells += high - low + 1;
|
|
169
|
+
for (let j = low; j <= high; j++) {
|
|
170
|
+
const substitution = previous[j - 1] + (character === b[j - 1] ? 0 : 1);
|
|
171
|
+
const cell = Math.min(previous[j] + 1, current[j - 1] + 1, substitution, beyond);
|
|
172
|
+
current[j] = cell;
|
|
173
|
+
if (cell < rowMin) rowMin = cell;
|
|
174
|
+
}
|
|
175
|
+
if (rowMin > tolerance) return { distance: beyond, withinTolerance: false, cells };
|
|
176
|
+
const finished = previous;
|
|
177
|
+
previous = current;
|
|
178
|
+
current = finished;
|
|
179
|
+
}
|
|
180
|
+
const distance = previous[n];
|
|
181
|
+
return { distance, withinTolerance: distance <= tolerance, cells };
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
export { boundedLevenshtein, fuzzyPrefixOf, fuzzyTermMatches, isListBase, isSingleStringFieldType, isStringValuedBase, isWordIndexedBase, isWordIndexedFieldType, parseFieldType, parsedTypeOf, tabulatedLevenshtein, withSortableOption };
|
|
@@ -1,15 +1,14 @@
|
|
|
1
1
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-EEBCDP36.mjs';
|
|
2
|
-
import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, bm25SetsValue, sortSignatureOf, requireSortableFields, analyseQueryPhrases, requireLoadedPhraseCode, fanOutQuery, loadPhraseCodeFor, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, mergeBM25Levels, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex, MAX_BM25_K1, MAX_BM25_B } from './chunk-
|
|
3
|
-
import { MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS } from './chunk-
|
|
4
|
-
import { flattenSchema, partialFieldsOf } from './chunk-
|
|
2
|
+
import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, bm25SetsValue, sortSignatureOf, requireSortableFields, analyseQueryPhrases, requireLoadedPhraseCode, fanOutQuery, loadPhraseCodeFor, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, mergeBM25Levels, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex, MAX_BM25_K1, MAX_BM25_B } from './chunk-QYMAYU7M.mjs';
|
|
3
|
+
import { MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS } from './chunk-QXH2CS4V.mjs';
|
|
4
|
+
import { flattenSchema, partialFieldsOf } from './chunk-BTARFTEN.mjs';
|
|
5
5
|
import { DEFAULT_MAX_PER_GROUP, DEFAULT_FUZZY_PREFIX_LENGTH, BM25_DISPLAYED_NAME_LENGTH, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-FQBFZAWG.mjs';
|
|
6
|
-
import {
|
|
7
|
-
import { isStringValuedBase, parsedTypeOf, isWordIndexedFieldType } from './chunk-
|
|
8
|
-
import {
|
|
6
|
+
import { tokenize } from './chunk-LWFK3TC2.mjs';
|
|
7
|
+
import { isStringValuedBase, parsedTypeOf, isWordIndexedFieldType, fuzzyTermMatches } from './chunk-GVPYMSLO.mjs';
|
|
8
|
+
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isPlainObject, isGeopointOrVectorType, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
9
|
+
import { NarsilError, ErrorCodes } from './chunk-IQHZMCLX.mjs';
|
|
9
10
|
import { compareComparableKeys } from './chunk-R6A773WK.mjs';
|
|
10
11
|
import { compareCodePoints, holdsUnpairedSurrogate } from './chunk-YTJCGKM4.mjs';
|
|
11
|
-
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isPlainObject, isGeopointOrVectorType, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
12
|
-
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
13
12
|
|
|
14
13
|
// src/search/cursor-binding.ts
|
|
15
14
|
var ABSENT_TAG = 0;
|
|
@@ -1550,7 +1549,7 @@ function requireCursorDepth(cursor, signature, binding) {
|
|
|
1550
1549
|
}
|
|
1551
1550
|
function hybridSearchOf(params, context, limit, offset, cursorDepth) {
|
|
1552
1551
|
if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
|
|
1553
|
-
return import('./runtime-
|
|
1552
|
+
return import('./runtime-RAOCKSYA.mjs').then(
|
|
1554
1553
|
(runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
|
|
1555
1554
|
);
|
|
1556
1555
|
}
|
|
@@ -1560,7 +1559,7 @@ function executeQuery(request, callerContext) {
|
|
|
1560
1559
|
if (loadingPhraseCode === null) return runQuery(request, callerContext);
|
|
1561
1560
|
return loadingPhraseCode.then(() => runQuery(request, callerContext));
|
|
1562
1561
|
}
|
|
1563
|
-
return import('./runtime-
|
|
1562
|
+
return import('./runtime-RAOCKSYA.mjs').then(
|
|
1564
1563
|
(runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
|
|
1565
1564
|
);
|
|
1566
1565
|
}
|
|
@@ -1,12 +1,11 @@
|
|
|
1
|
-
import { requireAnalysisRebuildMemory } from './chunk-
|
|
2
|
-
import { requireConfigurationWithinSizeLimit, DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, prepareConfiguredSynonyms, REBALANCE_CHUNK_SIZE, yieldToEventLoop, prepareDocumentVectors, removeDocumentVectors, insertDocumentVectors, vectorsEqual, rebuildTextIndexes, textRebuildTargetOf, extractVectorFromDoc, deleteNestedValue } from './chunk-
|
|
1
|
+
import { requireAnalysisRebuildMemory } from './chunk-SA52N7G3.mjs';
|
|
2
|
+
import { requireConfigurationWithinSizeLimit, DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, prepareConfiguredSynonyms, REBALANCE_CHUNK_SIZE, yieldToEventLoop, prepareDocumentVectors, removeDocumentVectors, insertDocumentVectors, vectorsEqual, rebuildTextIndexes, textRebuildTargetOf, extractVectorFromDoc, deleteNestedValue } from './chunk-YUCGFPMS.mjs';
|
|
3
3
|
import { ANALYSIS_CHANGE_MEMBERS, ANALYSIS_HOLD_APPLY_INTERVAL_MS } from './chunk-EEBCDP36.mjs';
|
|
4
|
-
import { resolvePartialWordAnalysis } from './chunk-
|
|
5
|
-
import { createPartitionIndex, ARRAY_POSITION_GAP } from './chunk-
|
|
6
|
-
import { configUnderAnalysis, resolveIndexAnalysis, targetOfPending } from './chunk-
|
|
7
|
-
import {
|
|
4
|
+
import { resolvePartialWordAnalysis } from './chunk-OH7SX5MY.mjs';
|
|
5
|
+
import { createPartitionIndex, ARRAY_POSITION_GAP } from './chunk-QYMAYU7M.mjs';
|
|
6
|
+
import { getLanguage, configUnderAnalysis, resolveIndexAnalysis, targetOfPending } from './chunk-FRJ5RGX7.mjs';
|
|
7
|
+
import { NarsilError, ErrorCodes } from './chunk-IQHZMCLX.mjs';
|
|
8
8
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
9
|
-
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
10
9
|
|
|
11
10
|
// src/engine/analysis-change/run.ts
|
|
12
11
|
function createHold(generation) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
2
|
-
import { TransportError, TransportErrorCodes } from './chunk-
|
|
2
|
+
import { TransportError, TransportErrorCodes } from './chunk-5QMHMELB.mjs';
|
|
3
3
|
import { encode, decode } from '@msgpack/msgpack';
|
|
4
4
|
|
|
5
5
|
// src/distribution/transport/tcp/types.ts
|
|
@@ -25,6 +25,15 @@ var ErrorCodes = {
|
|
|
25
25
|
PERSISTENCE_CRC_MISMATCH: "PERSISTENCE_CRC_MISMATCH",
|
|
26
26
|
PERSISTENCE_WAL_CORRUPT: "PERSISTENCE_WAL_CORRUPT",
|
|
27
27
|
PERSISTENCE_FSYNC_FAILED: "PERSISTENCE_FSYNC_FAILED",
|
|
28
|
+
PERSISTENCE_DEPENDENCY_MISSING: "PERSISTENCE_DEPENDENCY_MISSING",
|
|
29
|
+
BACKUP_NOT_FOUND: "BACKUP_NOT_FOUND",
|
|
30
|
+
BACKUP_ALREADY_EXISTS: "BACKUP_ALREADY_EXISTS",
|
|
31
|
+
BACKUP_DAMAGED: "BACKUP_DAMAGED",
|
|
32
|
+
BACKUP_DURABILITY_REQUIRED: "BACKUP_DURABILITY_REQUIRED",
|
|
33
|
+
BACKUP_IN_PROGRESS: "BACKUP_IN_PROGRESS",
|
|
34
|
+
BACKUP_STORE_CLAIMED: "BACKUP_STORE_CLAIMED",
|
|
35
|
+
BACKUP_STORE_NOT_CONFIGURED: "BACKUP_STORE_NOT_CONFIGURED",
|
|
36
|
+
BACKUP_NOT_READY: "BACKUP_NOT_READY",
|
|
28
37
|
SEARCH_INVALID_FIELD: "SEARCH_INVALID_FIELD",
|
|
29
38
|
VECTOR_DIMENSION_MISMATCH: "VECTOR_DIMENSION_MISMATCH",
|
|
30
39
|
SEARCH_INVALID_FILTER: "SEARCH_INVALID_FILTER",
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH } from './chunk-FQBFZAWG.mjs';
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-IQHZMCLX.mjs';
|
|
3
3
|
|
|
4
4
|
// src/search/hybrid-depth.ts
|
|
5
5
|
function fixedHybridDepthOf(hybrid) {
|
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-
|
|
2
|
-
import { SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-
|
|
3
|
-
import { VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-
|
|
4
|
-
import { readSegmentFile, STABLE_HEAD_ATTEMPTS } from './chunk-
|
|
5
|
-
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, manifestKey, segmentsPrefix, vectorFileKey, graphPartKey } from './chunk-
|
|
6
|
-
import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks } from './chunk-
|
|
7
|
-
import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-
|
|
8
|
-
import { DEFAULT_BM25_B, DEFAULT_BM25_K1 } from './chunk-
|
|
9
|
-
import { flattenSchema, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-
|
|
1
|
+
import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-F442VZUL.mjs';
|
|
2
|
+
import { SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-ONDEEBTA.mjs';
|
|
3
|
+
import { VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-QLDLQW36.mjs';
|
|
4
|
+
import { readSegmentFile, STABLE_HEAD_ATTEMPTS } from './chunk-RVBUQITB.mjs';
|
|
5
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, manifestKey, segmentsPrefix, vectorFileKey, graphPartKey } from './chunk-W6M4RRAT.mjs';
|
|
6
|
+
import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks } from './chunk-YUCGFPMS.mjs';
|
|
7
|
+
import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-OH7SX5MY.mjs';
|
|
8
|
+
import { DEFAULT_BM25_B, DEFAULT_BM25_K1 } from './chunk-QYMAYU7M.mjs';
|
|
9
|
+
import { flattenSchema, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-BTARFTEN.mjs';
|
|
10
|
+
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
10
11
|
import { MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-U476IXKQ.mjs';
|
|
12
|
+
import { NarsilError, ErrorCodes } from './chunk-IQHZMCLX.mjs';
|
|
11
13
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
12
|
-
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
13
|
-
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
14
14
|
import { decode, encode } from '@msgpack/msgpack';
|
|
15
15
|
|
|
16
16
|
// src/engine/durability-metadata.ts
|
|
@@ -1450,7 +1450,7 @@ function removeFromPartition(manager, partitionId, docIds) {
|
|
|
1450
1450
|
}
|
|
1451
1451
|
}
|
|
1452
1452
|
async function attachPagedSegment(directory, ref, partitionId, manager, options) {
|
|
1453
|
-
const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-
|
|
1453
|
+
const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-TMFIWOD2.mjs');
|
|
1454
1454
|
if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
|
|
1455
1455
|
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
1456
1456
|
requireAgreedSchema(options.schema, snapshot.schema, ref.key);
|
|
@@ -1464,7 +1464,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
|
|
|
1464
1464
|
const { directory } = store;
|
|
1465
1465
|
for (const ref of partition.segments) {
|
|
1466
1466
|
if (manager.config.textStorage === "disk" && directory !== void 0) {
|
|
1467
|
-
const { isPagedSegmentFile } = await import('./module-
|
|
1467
|
+
const { isPagedSegmentFile } = await import('./module-TMFIWOD2.mjs');
|
|
1468
1468
|
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
1469
1469
|
await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
|
|
1470
1470
|
await yieldToEventLoop();
|
|
@@ -1520,7 +1520,7 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
|
|
|
1520
1520
|
(field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
|
|
1521
1521
|
);
|
|
1522
1522
|
if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
|
|
1523
|
-
const { startReadingOpeningSectionsAhead } = await import('./module-
|
|
1523
|
+
const { startReadingOpeningSectionsAhead } = await import('./module-TMFIWOD2.mjs');
|
|
1524
1524
|
await startReadingOpeningSectionsAhead(directory, manifest);
|
|
1525
1525
|
}
|
|
1526
1526
|
for (const partition of manifest.partitions) {
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// src/persistence/constants.ts
|
|
2
|
+
var DEFAULT_INDEXEDDB_NAME = "narsil";
|
|
3
|
+
var DEFAULT_INDEXEDDB_STORE_NAME = "partitions";
|
|
4
|
+
var MAX_PERSISTENCE_KEY_LENGTH = 1024;
|
|
5
|
+
var INDEXEDDB_BLOCKED_TIMEOUT_MS = 5e3;
|
|
6
|
+
var OBJECT_STORE_PART_BYTES = 8388608;
|
|
7
|
+
var OBJECT_STORE_LIST_PAGE_KEYS = 1e3;
|
|
8
|
+
var OBJECT_STORE_CONFLICT_ATTEMPTS = 5;
|
|
9
|
+
var OBJECT_STORE_CONFLICT_RETRY_MS = 200;
|
|
10
|
+
|
|
11
|
+
export { DEFAULT_INDEXEDDB_NAME, DEFAULT_INDEXEDDB_STORE_NAME, INDEXEDDB_BLOCKED_TIMEOUT_MS, MAX_PERSISTENCE_KEY_LENGTH, OBJECT_STORE_CONFLICT_ATTEMPTS, OBJECT_STORE_CONFLICT_RETRY_MS, OBJECT_STORE_LIST_PAGE_KEYS, OBJECT_STORE_PART_BYTES };
|