@delali/narsil 0.3.1-next.163 → 0.3.1-next.169
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/browser/{chunk-MZHQ65VI.mjs → chunk-2Q45HI5H.mjs} +14 -10
- package/dist/browser/{chunk-MMHIIL5B.mjs → chunk-4ZICWS4V.mjs} +1 -1
- package/dist/browser/{chunk-KQXKCMGQ.mjs → chunk-JHFBQKUF.mjs} +119 -90
- package/dist/browser/{core-leaf-SIWXFZLV.mjs → core-leaf-WYFANMKC.mjs} +61 -13
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/{embed-RF2MPGAO.mjs → embed-24IPR4LP.mjs} +2 -2
- package/dist/browser/{module-PMQBRESM.mjs → module-XVLHJ7T7.mjs} +358 -62
- package/dist/browser/{optional-writes-WMZX4OEC.mjs → optional-writes-TB34FYUW.mjs} +118 -66
- package/dist/browser/{phrase-module-22TKMXWI.mjs → phrase-module-6UGFVW5J.mjs} +1 -0
- package/dist/browser/{resolve-vector-text-KI65IHMD.mjs → resolve-vector-text-EJ6DHFF5.mjs} +8 -0
- package/dist/browser/{storage-leaf-VGR2BWJJ.mjs → storage-leaf-QFXIWDFK.mjs} +208 -87
- package/dist/browser/{worker-copies-WQCPP3TS.mjs → worker-copies-ICHLYYXM.mjs} +1 -0
- package/dist/{capabilities-DkXAkb-X.d.ts → capabilities-BQIdsaWy.d.ts} +2 -2
- package/dist/{chunk-BKOMO7HT.mjs → chunk-26E5MO3Y.mjs} +5 -5
- package/dist/{chunk-EEYDJB4Z.mjs → chunk-2C3BXZGA.mjs} +213 -149
- package/dist/{chunk-NXMGUDZA.mjs → chunk-2KWTQ2P7.mjs} +2 -2
- package/dist/{chunk-TW543F33.mjs → chunk-34IOOYXA.mjs} +1 -1
- package/dist/{chunk-D5GNFN6X.mjs → chunk-7FC2ZILY.mjs} +98 -40
- package/dist/{chunk-XRBPHCCT.mjs → chunk-A4XYDUVN.mjs} +5 -5
- package/dist/{chunk-R4F7ZTVG.mjs → chunk-BIIROQ5Q.mjs} +2 -2
- package/dist/{chunk-Q2WQSAYH.mjs → chunk-CQX5B6L2.mjs} +54 -11
- package/dist/{chunk-XXWEUXE4.mjs → chunk-D6FYUTFN.mjs} +1 -1
- package/dist/{chunk-OUWZKNO3.mjs → chunk-DG6WIEB3.mjs} +1 -1
- package/dist/{chunk-GJKFXKMI.mjs → chunk-F3FDHXCJ.mjs} +12 -9
- package/dist/{chunk-3UGZBRDN.mjs → chunk-GO6TRSJ6.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-OK3WUA6N.mjs → chunk-I2WJCRIH.mjs} +1 -1
- package/dist/{chunk-Q7CUUSJ2.mjs → chunk-ISJ35523.mjs} +2 -2
- package/dist/{chunk-GQLOEACG.mjs → chunk-JCWC75NH.mjs} +2 -2
- package/dist/{chunk-ZGYF37MG.mjs → chunk-JDIATDPY.mjs} +1 -1
- package/dist/{chunk-3B775F3C.mjs → chunk-JP77OWMI.mjs} +3 -1
- package/dist/{chunk-KSOZJEYU.mjs → chunk-KHJPMZE2.mjs} +1 -1
- package/dist/{chunk-ZQ6QY42L.mjs → chunk-KL6W73BZ.mjs} +8 -0
- package/dist/{chunk-ES5EOXT6.mjs → chunk-LOQHA6QK.mjs} +3 -3
- package/dist/{chunk-AB27FWZM.mjs → chunk-LPRRFCVU.mjs} +2 -2
- package/dist/{chunk-EIR3IQCL.mjs → chunk-QFJMRB7D.mjs} +2 -1
- package/dist/{chunk-S3BSDBF7.mjs → chunk-QJGDB3WR.mjs} +1 -1
- package/dist/{chunk-EI7U43MX.mjs → chunk-R7PLSVK3.mjs} +2 -2
- package/dist/{chunk-5DUTOEK5.mjs → chunk-TBOZHYCF.mjs} +2 -2
- package/dist/{chunk-VJOS6MHF.mjs → chunk-TNEU32EN.mjs} +64 -16
- package/dist/{chunk-RXMR2SIQ.mjs → chunk-UISV46WX.mjs} +4 -4
- package/dist/{chunk-4CAMH7FO.mjs → chunk-X4UMDSB4.mjs} +2 -1
- package/dist/{chunk-R4CE4IVU.mjs → chunk-X5NHWOXT.mjs} +15 -11
- package/dist/{react-native/chunks/chunk-HEELWLKL.mjs → chunk-XFOL77WH.mjs} +1 -1
- package/dist/{chunk-TOZENMM7.mjs → chunk-XITVDRRH.mjs} +5 -5
- package/dist/{chunk-FSLQHOKZ.mjs → chunk-XNFBYAXI.mjs} +1 -1
- package/dist/{chunk-V4YIEMAI.mjs → chunk-XQQKETAF.mjs} +4 -4
- package/dist/{chunk-DSXFN5BF.mjs → chunk-XTSFRWAS.mjs} +8 -8
- package/dist/{chunk-K4BMDPPX.mjs → chunk-Y27E2MGC.mjs} +2 -2
- package/dist/{chunk-JV74RKCY.mjs → chunk-YM2HI5N5.mjs} +1 -1
- package/dist/{chunk-7OE73OO4.mjs → chunk-YPV2L7M5.mjs} +56 -6
- package/dist/{chunk-STLNPJXU.mjs → chunk-YXZDTKQS.mjs} +2 -2
- package/dist/{chunk-5P72V66D.mjs → chunk-ZB5X5WAK.mjs} +24 -20
- package/dist/{client-VJmV_AjZ.d.ts → client-CeUxvIvd.d.ts} +4 -4
- package/dist/client.d.ts +8 -8
- package/dist/{config-ZZg4cwQz.d.ts → config-CIpLXXxm.d.ts} +2 -2
- package/dist/{config-Cb3FOlE_.d.ts → config-DehU-VGL.d.ts} +2 -2
- package/dist/{core-leaf-LOXCD7MY.mjs → core-leaf-VIMO377O.mjs} +3 -3
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/{direct-executor-PF4UDSUW.mjs → direct-executor-2WQSGBN3.mjs} +7 -7
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +1 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +39 -39
- package/dist/{embed-4H7RTBVL.mjs → embed-UNHDFGDU.mjs} +2 -2
- package/dist/{engine-DdOs682x.d.ts → engine-P-4xhUe4.d.ts} +6 -2
- package/dist/{index-Ck9w5WHz.d.ts → index-CBUl4z3E.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +7 -7
- package/dist/index.react-native.mjs +6 -6
- package/dist/{long-query-thread-OTVLP2OI.mjs → long-query-thread-AAYTLYZK.mjs} +19 -19
- package/dist/{module-UIKSHY3G.mjs → module-5SXNUCD5.mjs} +315 -64
- package/dist/{msgpack-leaf-5WFRP6JM.mjs → msgpack-leaf-7KBYOVTF.mjs} +2 -2
- package/dist/{optional-reads-RSLNSGNG.mjs → optional-reads-ENKMVQ4Q.mjs} +1 -1
- package/dist/{optional-writes-4ITCV7YT.mjs → optional-writes-N2DFWWZ3.mjs} +39 -26
- package/dist/{paged-partition-archive-VMJ2AUA6.mjs → paged-partition-archive-X7L27E5Z.mjs} +19 -19
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +7 -7
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -17
- package/dist/{phrase-module-IWSG2DDZ.mjs → phrase-module-ZY3TLTQA.mjs} +2 -1
- package/dist/{query-actions-EW72OQXY.mjs → query-actions-2JOLPELC.mjs} +7 -7
- package/dist/query-suggestions/merge-worker.mjs +10 -10
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +8 -8
- package/dist/{chunk-ZBVHS5KF.mjs → react-native/chunks/chunk-57R74JWC.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-AUXXPSFB.mjs → chunk-F4MIGBBP.mjs} +214 -150
- package/dist/react-native/chunks/{chunk-3RUXKZJG.mjs → chunk-F6BJYCF2.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-RVHCQJQM.mjs → chunk-GGBATERA.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-I5W7A2JF.mjs → chunk-GVWLACAT.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-XOCM3BFU.mjs → chunk-H344ZXRH.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-VIPNDN4B.mjs → chunk-I7DTUOK2.mjs} +65 -15
- package/dist/react-native/chunks/{chunk-2JMAUU26.mjs → chunk-J2BQTJJE.mjs} +1 -1
- package/dist/{chunk-QBVTYY4J.mjs → react-native/chunks/chunk-OZ3XEM6D.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-4EXKQ2IB.mjs → chunk-SK3DPAI4.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-XBTFK3AZ.mjs → chunk-WFQ6YQZA.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-FTQ7JNMU.mjs → chunk-ZDDJVR7X.mjs} +15 -11
- package/dist/react-native/chunks/{core-leaf-O3236QY3.mjs → core-leaf-UDUMVC3M.mjs} +2 -2
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/{embed-3WQ2CCR5.mjs → embed-U7CIKEY4.mjs} +2 -2
- package/dist/react-native/chunks/{module-LRR7D5ZO.mjs → module-7BJ3VOWQ.mjs} +360 -66
- package/dist/react-native/chunks/{msgpack-leaf-TR66ZVAX.mjs → msgpack-leaf-GTKLF6GJ.mjs} +2 -2
- package/dist/react-native/chunks/{optional-reads-TOWGNEDH.mjs → optional-reads-74HYI4DO.mjs} +1 -1
- package/dist/react-native/chunks/{optional-writes-7AFBIZMN.mjs → optional-writes-TXTJ6HUZ.mjs} +37 -24
- package/dist/react-native/chunks/{phrase-module-3A4AKMKL.mjs → phrase-module-UC4YGEBB.mjs} +2 -1
- package/dist/react-native/chunks/{resolve-vector-text-LXRYXF7M.mjs → resolve-vector-text-FTPD5EY6.mjs} +8 -0
- package/dist/react-native/chunks/{restore-queue-7FSHTGJ5.mjs → restore-queue-SZLWIL3B.mjs} +9 -9
- package/dist/react-native/chunks/{segment-codec-leaf-4QV4HO4R.mjs → segment-codec-leaf-WMUZ5P4F.mjs} +7 -7
- package/dist/react-native/chunks/{segment-merge-leaf-5DUZZSXJ.mjs → segment-merge-leaf-Q2OEJIQ3.mjs} +8 -8
- package/dist/react-native/chunks/{storage-leaf-PNU6HWY7.mjs → storage-leaf-OXYV6O6P.mjs} +213 -100
- package/dist/react-native/chunks/{store-leaf-ESVUGYYO.mjs → store-leaf-ZMDXYK4E.mjs} +8 -8
- package/dist/react-native/chunks/{suggestions-leaf-MOVOPBGD.mjs → suggestions-leaf-FIMGQZ2T.mjs} +8 -8
- package/dist/react-native/chunks/{worker-copies-VQGF3WZX.mjs → worker-copies-72TGFYQB.mjs} +8 -7
- package/dist/react-native/partial-words.mjs +6 -6
- package/dist/react-native/pattern.mjs +3 -3
- package/dist/react-native/query-suggestions.mjs +6 -6
- package/dist/react-native/sync.mjs +6 -6
- package/dist/react-native/synonyms.mjs +6 -6
- package/dist/react.d.ts +9 -9
- package/dist/{rerank-QHJXPSAK.mjs → rerank-E2HFKKGJ.mjs} +7 -7
- package/dist/{resolve-vector-text-4PDRTPNQ.mjs → resolve-vector-text-6WPFFIX5.mjs} +1 -1
- package/dist/{restore-queue-5W5HX44H.mjs → restore-queue-PUVBN6L3.mjs} +10 -10
- package/dist/{schema-C_7yRJLt.d.ts → schema-CzG9OUw5.d.ts} +5 -1
- package/dist/{segment-codec-leaf-7VLVXKWO.mjs → segment-codec-leaf-24Z3MMQI.mjs} +8 -8
- package/dist/{segment-merge-leaf-22XR3IGA.mjs → segment-merge-leaf-FQQDU45E.mjs} +8 -8
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +26 -26
- package/dist/{storage-leaf-ZZ77C3DX.mjs → storage-leaf-KIA7XO6Y.mjs} +79 -81
- package/dist/{store-leaf-LJ6D2PMF.mjs → store-leaf-ESVLL4JO.mjs} +9 -9
- package/dist/{suggestions-leaf-KTQHO3PY.mjs → suggestions-leaf-FXF4AHAY.mjs} +11 -11
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +15 -15
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +3 -3
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +7 -7
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +7 -7
- package/dist/{tasks-CxA-J_Ty.d.ts → tasks-CXtQ3Zpp.d.ts} +4 -4
- package/dist/thread-OWXVP2ZA.mjs +30 -0
- package/dist/{types-BboB278g.d.ts → types-CDN70Y_C.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/{worker-copies-ODAAGQRC.mjs → worker-copies-XXX7PF4J.mjs} +10 -10
- package/dist/workers/entry.mjs +13 -13
- package/package.json +9 -8
- package/dist/thread-XWDK6TAR.mjs +0 -30
- /package/dist/{chunk-77AUVDQ5.mjs → chunk-CV3Q2EKO.mjs} +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { VECTOR_BLOCK_MAX_BYTES, MAX_DOC_ID_TABLE_BYTES, MAX_VECTOR_ORDINALS, WASM_PAGE_BYTES, VECTOR_SCRATCH_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, turnsAreBudgeted, backgroundTurnIsSpent, yieldBackgroundTurn, VECTOR_BLOCK_INITIAL_SLOTS, MAX_WASM_PAGES, NATIVE_ATTACH_RETRY_MS, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, GRAPH_LOCK_WAIT_SLICE_MS, NATIVE_THREAD_SEARCH_SLOTS, GRAPH_LOCK_WAIT_DEADLINE_MS, holdWrites, MAX_DOC_ID_LENGTH, threadSlotGone } from './chunk-
|
|
2
|
-
import { stopUsingTheNativeSearchCore, nativeSearchCoreIsEnabled, loadNativeCore, noteFallback, NATIVE_OK, NATIVE_NEEDS_ROOM, NATIVE_GRAPH_NOT_HELD_ALONE } from './chunk-
|
|
3
|
-
import { DEFAULT_EF_SEARCH, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, PLACEMENT_ROOM_REQUESTS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, MAX_M, INITIAL_UPPER_SLOTS, CONNECT_RESTARTS_BEFORE_ONE_PASS, FARTHEST_REJECTED_LINKS_KEPT, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, NEAREST_LINK_FALLBACK_HOSTS
|
|
1
|
+
import { VECTOR_BLOCK_MAX_BYTES, MAX_DOC_ID_TABLE_BYTES, MAX_VECTOR_ORDINALS, WASM_PAGE_BYTES, VECTOR_SCRATCH_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, turnsAreBudgeted, backgroundTurnIsSpent, yieldBackgroundTurn, VECTOR_BLOCK_INITIAL_SLOTS, MAX_WASM_PAGES, NATIVE_ATTACH_RETRY_MS, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, GRAPH_LOCK_WAIT_SLICE_MS, NATIVE_THREAD_SEARCH_SLOTS, GRAPH_LOCK_WAIT_DEADLINE_MS, holdWrites, MAX_DOC_ID_LENGTH, threadSlotGone } from './chunk-JP77OWMI.mjs';
|
|
2
|
+
import { stopUsingTheNativeSearchCore, nativeSearchCoreIsEnabled, loadNativeCore, noteFallback, NATIVE_OK, NATIVE_NEEDS_ROOM, NATIVE_GRAPH_NOT_HELD_ALONE } from './chunk-D6FYUTFN.mjs';
|
|
3
|
+
import { DEFAULT_EF_SEARCH, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, PLACEMENT_ROOM_REQUESTS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, MAX_M, MIN_M, INITIAL_UPPER_SLOTS, CONNECT_RESTARTS_BEFORE_ONE_PASS, FARTHEST_REJECTED_LINKS_KEPT, CONNECT_STEPS_BETWEEN_CLOCK_CHECKS, NEAREST_LINK_FALLBACK_HOSTS } from './chunk-ZXX5GZEX.mjs';
|
|
4
4
|
import { MAX_FILE_BYTES } from './chunk-4PY4JYMY.mjs';
|
|
5
5
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
6
6
|
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
@@ -188,14 +188,16 @@ function growBufferToExactly(buffer, bytes) {
|
|
|
188
188
|
if (buffer.byteLength >= bytes) return;
|
|
189
189
|
resizeBuffer(buffer, bytes, bytes);
|
|
190
190
|
}
|
|
191
|
+
function requireBytesWithinLimit(bytes, maxBytes) {
|
|
192
|
+
if (bytes <= maxBytes) return;
|
|
193
|
+
throw new NarsilError(
|
|
194
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
195
|
+
`A vector structure needs ${bytes} bytes where it may hold at most ${maxBytes}`,
|
|
196
|
+
{ bytes, maxBytes }
|
|
197
|
+
);
|
|
198
|
+
}
|
|
191
199
|
function resizeBuffer(buffer, bytes, target) {
|
|
192
|
-
|
|
193
|
-
throw new NarsilError(
|
|
194
|
-
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
195
|
-
`A vector structure needs ${bytes} bytes where it may hold at most ${buffer.maxByteLength}`,
|
|
196
|
-
{ bytes, maxBytes: buffer.maxByteLength }
|
|
197
|
-
);
|
|
198
|
-
}
|
|
200
|
+
requireBytesWithinLimit(bytes, buffer.maxByteLength);
|
|
199
201
|
if (buffer instanceof ArrayBuffer) {
|
|
200
202
|
buffer.resize(target);
|
|
201
203
|
return;
|
|
@@ -3002,6 +3004,16 @@ function markTombstone(state, ord) {
|
|
|
3002
3004
|
Atomics.add(state.header, GRAPH_TOMBSTONE_COUNT, 1);
|
|
3003
3005
|
}
|
|
3004
3006
|
}
|
|
3007
|
+
function reviveTombstone(state, ord) {
|
|
3008
|
+
if (!nodeExists(state, ord) || !isTombstoned(state, ord)) return;
|
|
3009
|
+
lockGraphShared(state.locks);
|
|
3010
|
+
try {
|
|
3011
|
+
clearTombstone(state, ord);
|
|
3012
|
+
} finally {
|
|
3013
|
+
unlockGraphShared(state.locks);
|
|
3014
|
+
}
|
|
3015
|
+
state.linkWrites += 1;
|
|
3016
|
+
}
|
|
3005
3017
|
function compactTombstones(state, mayFree2) {
|
|
3006
3018
|
if (Atomics.load(state.header, GRAPH_TOMBSTONE_COUNT) === 0) return;
|
|
3007
3019
|
countIncomingBeforeWriting(state, true);
|
|
@@ -4341,6 +4353,7 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
|
|
|
4341
4353
|
if (ord !== void 0) markTombstone(state, ord);
|
|
4342
4354
|
},
|
|
4343
4355
|
markTombstoneOrdinal: (ordinal) => markTombstone(state, ordinal),
|
|
4356
|
+
reviveOrdinal: (ordinal) => reviveTombstone(state, ordinal),
|
|
4344
4357
|
holdsNode: (ordinal) => hasNode(state.adjacency, ordinal),
|
|
4345
4358
|
holdsTombstone: (ordinal) => hasNode(state.adjacency, ordinal) && isTombstoned(state, ordinal),
|
|
4346
4359
|
has: (docId) => {
|
|
@@ -5220,6 +5233,30 @@ function decodeVectorIndexParts(raw) {
|
|
|
5220
5233
|
return raw.map(decodeVectorIndexPart);
|
|
5221
5234
|
}
|
|
5222
5235
|
|
|
5236
|
+
// src/vector/vector-index/upper-list-bound.ts
|
|
5237
|
+
var UPPER_LISTS_PER_NODE_FACTOR = 4;
|
|
5238
|
+
var UPPER_LISTS_ALLOWANCE = 64;
|
|
5239
|
+
function requireBoundedUpperLists(levelsOfParts, m, what) {
|
|
5240
|
+
let nodes = 0;
|
|
5241
|
+
let upperLists = 0;
|
|
5242
|
+
for (const levels of levelsOfParts) {
|
|
5243
|
+
for (let at = 0; at < levels.length; at++) {
|
|
5244
|
+
const level2 = levels[at];
|
|
5245
|
+
if (level2 === 0) continue;
|
|
5246
|
+
nodes++;
|
|
5247
|
+
upperLists += level2 - 1;
|
|
5248
|
+
}
|
|
5249
|
+
}
|
|
5250
|
+
const spread = m - 1;
|
|
5251
|
+
if (upperLists * spread <= UPPER_LISTS_PER_NODE_FACTOR * nodes + UPPER_LISTS_ALLOWANCE * spread) return;
|
|
5252
|
+
const limit = Math.floor(UPPER_LISTS_PER_NODE_FACTOR * nodes / spread) + UPPER_LISTS_ALLOWANCE;
|
|
5253
|
+
throw new NarsilError(
|
|
5254
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
5255
|
+
`${what} holds ${upperLists} upper-layer lists over ${nodes} nodes, while a graph of ${nodes} nodes at m ${m} holds at most ${limit}`,
|
|
5256
|
+
{ upperLists, nodes, m, limit }
|
|
5257
|
+
);
|
|
5258
|
+
}
|
|
5259
|
+
|
|
5223
5260
|
// src/vector/vector-index/checkpoint-payload.ts
|
|
5224
5261
|
var VECTOR_FILE_PAYLOAD_VERSION = 1;
|
|
5225
5262
|
var VECTOR_GRAPH_PAYLOAD_VERSION = 1;
|
|
@@ -5282,15 +5319,15 @@ function calibratedMeanLengthOf(name, value) {
|
|
|
5282
5319
|
}
|
|
5283
5320
|
return value;
|
|
5284
5321
|
}
|
|
5285
|
-
function decodeNumberedGraph(raw
|
|
5322
|
+
function decodeNumberedGraph(raw) {
|
|
5286
5323
|
const name = "vector graph";
|
|
5287
5324
|
if (!isRecord2(raw)) invalid2(name, "each graph must be a map");
|
|
5288
5325
|
const entryPoint = raw.entryPoint ?? null;
|
|
5289
5326
|
if (entryPoint !== null && !isIntegerUpTo(entryPoint, 4294967295))
|
|
5290
5327
|
invalid2(name, "entryPoint must be a uint32 or nil");
|
|
5291
5328
|
if (!isIntegerUpTo(raw.maxLayer, 255)) invalid2(name, "maxLayer must be a uint8");
|
|
5292
|
-
if (!isIntegerUpTo(raw.m, MAX_M) || raw.m <
|
|
5293
|
-
invalid2(name, `m must be a whole number from ${
|
|
5329
|
+
if (!isIntegerUpTo(raw.m, MAX_M) || raw.m < MIN_M) {
|
|
5330
|
+
invalid2(name, `m must be a whole number from ${MIN_M} to ${MAX_M}`);
|
|
5294
5331
|
}
|
|
5295
5332
|
if (!isIntegerUpTo(raw.efConstruction, 65535)) invalid2(name, "efConstruction must be a uint16");
|
|
5296
5333
|
if (!isMetric(raw.metric)) invalid2(name, "metric must name a supported metric", { metric: raw.metric });
|
|
@@ -5321,7 +5358,7 @@ function decodeVectorGraphPayload(raw) {
|
|
|
5321
5358
|
return {
|
|
5322
5359
|
v: VECTOR_GRAPH_PAYLOAD_VERSION,
|
|
5323
5360
|
fieldName: raw.fieldName,
|
|
5324
|
-
graphs: raw.graphs.map((graph) => decodeNumberedGraph(graph
|
|
5361
|
+
graphs: raw.graphs.map((graph) => decodeNumberedGraph(graph))
|
|
5325
5362
|
};
|
|
5326
5363
|
}
|
|
5327
5364
|
function decodeVectorGraphPartPayload(raw) {
|
|
@@ -5333,7 +5370,7 @@ function decodeVectorGraphPartPayload(raw) {
|
|
|
5333
5370
|
return {
|
|
5334
5371
|
v: VECTOR_GRAPH_PART_PAYLOAD_VERSION,
|
|
5335
5372
|
fieldName: raw.fieldName,
|
|
5336
|
-
graphs: raw.graphs.map((graph) => decodeNumberedGraph(graph
|
|
5373
|
+
graphs: raw.graphs.map((graph) => decodeNumberedGraph(graph))
|
|
5337
5374
|
};
|
|
5338
5375
|
}
|
|
5339
5376
|
function sameGraphHeader(a, b) {
|
|
@@ -5375,10 +5412,21 @@ function assembleGraphParts(parts) {
|
|
|
5375
5412
|
}
|
|
5376
5413
|
requireOneGraphPerVector(part, p);
|
|
5377
5414
|
}
|
|
5415
|
+
for (let g = 0; g < partLists.length; g++) {
|
|
5416
|
+
const list = partLists[g];
|
|
5417
|
+
requireBoundedUpperLists(
|
|
5418
|
+
list.map((graph) => graph.levels),
|
|
5419
|
+
first.graphs[g].m,
|
|
5420
|
+
`Graph ${g} of the vector graph parts of field "${first.fieldName}"`
|
|
5421
|
+
);
|
|
5422
|
+
}
|
|
5378
5423
|
const graphs = partLists.map((list) => ({ partSize: VECTOR_GRAPH_PART_VECTORS, parts: list }));
|
|
5379
5424
|
return { fieldName: first.fieldName, graphs };
|
|
5380
5425
|
}
|
|
5381
5426
|
function graphPartsOfEarlierPayload(payload) {
|
|
5427
|
+
payload.graphs.forEach((graph, g) => {
|
|
5428
|
+
requireBoundedUpperLists([graph.levels], graph.m, `Graph ${g} of the vector graph of field "${payload.fieldName}"`);
|
|
5429
|
+
});
|
|
5382
5430
|
return {
|
|
5383
5431
|
fieldName: payload.fieldName,
|
|
5384
5432
|
graphs: payload.graphs.map((graph) => ({ partSize: Math.max(graph.levels.length, 1), parts: [graph] }))
|
|
@@ -6030,4 +6078,4 @@ function defaultQuantizationFor(dimension) {
|
|
|
6030
6078
|
return "osq8";
|
|
6031
6079
|
}
|
|
6032
6080
|
|
|
6033
|
-
export { BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_VECTOR_METRIC, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, GRAPH_RETIRED, IN_MEMORY, MAX_EF_SEARCH, MIN_EF_SEARCH, MIN_OVERSAMPLE, NOTHING_STAGED, NO_LIVE_VECTOR_AT_POSITION, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, STORE_BLOCK_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_CODE_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, STORE_WRITE_SEQUENCE_INT64, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, VECTOR_MEMORY_LIMITS, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, addToOrdinalFilter, admissionAt, adoptBackgroundScheduler, adoptGraph, adoptLoadedCalibration, allLiveDocIds, arenaFloat32Distance, assembleGraphParts, assignStorePartitions, bindVectorCoreHost, blocksGrowthBytes, bytesToVectors, centroidOfCentroidSums, computeCodeBlockLayout, cosineSimilarityWithMagnitudes, countDead, countOfCentroidSums, coverOrdinal, createCentroidSums, createFixedBuffer, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createOsqScratch, createSharedVectorStoreHandles, createVectorBlock, createVectorFileReader, deadBytesFor, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, defaultEfSearch, defaultQuantizationFor, deserialize, detachNativeStore, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, graphNeedsRebuild, graphPartsOfEarlierPayload, growBufferTo, grownByteLength, handOverBuildTurns, holdAdmission, insertNode, isDead, liveOrdinals, liveSize, magnitude, mayFree, meanOfCentroidSums, nativeMetricCode, nativeScoresOf, nativeStoreFor, nativeStoreScratchBytes, nextSequence, noBuildHistory, nodeCountOf, oldestHeldPin, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqDistance, osqEstimate, osqQuantize, osqRecordBytes, planCheckpoint, publishSequence, readTrailer, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releasePinsHeldBy, releaseReplacedVectors, requireUsableVectorOptions, requireWholeNeighbourLists, rescoreDepth, resetStoreHeader, retiredAtOf, scalarQueryProducts, searchOrdinals, sequenceWritesOf, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, shuffleStep, signatureOf, simdModuleLoaded, slotByteOffset, squaredLengthsOfCentroidSums, stageQueryLevels, stampBorn, stampRetired, storePinsOf, threadedNativeCore, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount, vectorsPerFile, vectorsToBytes, withNativeThreadSlot, withTransientAdmission, writeSequenceOf, writeSequenceWords, yieldBuildTurn };
|
|
6081
|
+
export { BUILD_CHUNK_SIZE, BUILD_RETRY_FIRST_DELAY_MS, BUILD_RETRY_MAX_DELAY_MS, BUILD_STEPS_BETWEEN_CLOCK_CHECKS, DEFAULT_FILTER_THRESHOLD, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_VECTOR_METRIC, DISK_ADOPTION_YIELD_INTERVAL, DRIFT_CHECK_GROWTH_FACTOR, ESTIMATED_MS_PER_TOMBSTONE, ESTIMATED_MS_PER_VECTOR_REBUILD, GRAPH_RETIRED, IN_MEMORY, MAX_EF_SEARCH, MIN_EF_SEARCH, MIN_OVERSAMPLE, NOTHING_STAGED, NO_LIVE_VECTOR_AT_POSITION, QUERY_STAGED, RECALIBRATION_CHUNK_VECTORS, RECALIBRATION_DRIFT_RATIO, SEARCH_ATTEMPTS_BEFORE_HOLDING_WRITES, SEARCH_REFILL_RESERVE, SEARCH_SCAN_CHUNK_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, STORE_BLOCK_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, STORE_CODE_BLOCK_COUNT, STORE_CODE_COUNT, STORE_DOC_ID_BYTES, STORE_LIVE_COUNT, STORE_ORDINAL_LAYOUT_GENERATION, STORE_RELEASED_VECTORS_TO_DISK, STORE_REMOVED_COUNT, STORE_SLOTS, STORE_UNPLACED_COUNT, STORE_WRITE_SEQUENCE_INT64, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, VECTOR_MEMORY_LIMITS, VECTOR_WORKER_COPIES_ALLOWED, WORKER_COPY_MIN_VECTORS, addToOrdinalFilter, admissionAt, adoptBackgroundScheduler, adoptGraph, adoptLoadedCalibration, allLiveDocIds, arenaFloat32Distance, assembleGraphParts, assignStorePartitions, bindVectorCoreHost, blocksGrowthBytes, bytesToVectors, centroidOfCentroidSums, computeCodeBlockLayout, cosineSimilarityWithMagnitudes, countDead, countOfCentroidSums, coverOrdinal, createCentroidSums, createFixedBuffer, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createOsqScratch, createSharedVectorStoreHandles, createVectorBlock, createVectorFileReader, deadBytesFor, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, defaultEfSearch, defaultQuantizationFor, deserialize, detachNativeStore, dotProduct, emptyFieldBeforeRestore, ensureBlockSlots, euclideanDistance, fieldHandlesOf, filterForOptions, fixedView, graphNeedsRebuild, graphPartsOfEarlierPayload, growBufferTo, grownByteLength, handOverBuildTurns, holdAdmission, insertNode, isDead, liveOrdinals, liveSize, magnitude, mayFree, meanOfCentroidSums, nativeMetricCode, nativeScoresOf, nativeStoreFor, nativeStoreScratchBytes, nextSequence, noBuildHistory, nodeCountOf, oldestHeldPin, openGraphState, openSharedQuantizer, openVectorBlock, ordinalFilterForSubsets, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqDistance, osqEstimate, osqQuantize, osqRecordBytes, planCheckpoint, publishSequence, readTrailer, recalibrateFromStore, recordCheckpoint, releaseLocksHeldBy, releasePinsHeldBy, releaseReplacedVectors, requireBoundedUpperLists, requireBytesWithinLimit, requireUsableVectorOptions, requireWholeNeighbourLists, rescoreDepth, resetStoreHeader, retiredAtOf, scalarQueryProducts, searchOrdinals, sequenceWritesOf, sharedMemoryAvailable, sharedVectorStoreBytes, shuffleInPlace, shuffleStep, signatureOf, simdModuleLoaded, slotByteOffset, squaredLengthsOfCentroidSums, stageQueryLevels, stampBorn, stampRetired, storePinsOf, threadedNativeCore, threadsHoldCurrentLayout, toDistance, toScore, tombstoneCountOf, topLayerInParts, unplacedChunkCount, vectorsPerFile, vectorsToBytes, withNativeThreadSlot, withTransientAdmission, writeSequenceOf, writeSequenceWords, yieldBuildTurn };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, manifestFileRefs, graphPartKey, vectorFileKey } from './chunk-
|
|
2
|
-
import { VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-
|
|
3
|
-
import { openChecksummedEnvelope, HEADER_SIZE, QUERY_HISTORY_ARCHIVE_PREFIX, MAX_HISTORY_FILE_BYTES } from './chunk-
|
|
1
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, manifestFileRefs, graphPartKey, vectorFileKey } from './chunk-KHJPMZE2.mjs';
|
|
2
|
+
import { VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-TNEU32EN.mjs';
|
|
3
|
+
import { openChecksummedEnvelope, HEADER_SIZE, QUERY_HISTORY_ARCHIVE_PREFIX, MAX_HISTORY_FILE_BYTES } from './chunk-JCWC75NH.mjs';
|
|
4
4
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
5
|
-
import { requireValidFieldTypes } from './chunk-
|
|
5
|
+
import { requireValidFieldTypes } from './chunk-X5NHWOXT.mjs';
|
|
6
6
|
import { MAX_FILE_BYTES } from './chunk-4PY4JYMY.mjs';
|
|
7
7
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
8
8
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { MAX_PARTITIONS } from './chunk-JP77OWMI.mjs';
|
|
1
2
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
2
3
|
|
|
3
4
|
// src/engine/snapshot/restore-config.ts
|
|
@@ -82,7 +83,7 @@ function restoredPartitionConfig(raw, reject = rejectSnapshot) {
|
|
|
82
83
|
}
|
|
83
84
|
return {
|
|
84
85
|
...typeof maxDocsPerPartition === "number" ? { maxDocsPerPartition } : {},
|
|
85
|
-
...typeof maxPartitions === "number" ? { maxPartitions } : {},
|
|
86
|
+
...typeof maxPartitions === "number" ? { maxPartitions: Math.min(maxPartitions, MAX_PARTITIONS) } : {},
|
|
86
87
|
...typeof watermark === "number" ? { watermark } : {}
|
|
87
88
|
};
|
|
88
89
|
}
|
|
@@ -17,6 +17,11 @@ var PARSED_FIELD_TYPE_CACHE_ENTRIES = 1024;
|
|
|
17
17
|
|
|
18
18
|
// src/schema/validator/shared.ts
|
|
19
19
|
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
20
|
+
var VECTOR_TYPE_PATTERN = /^vector\[(.*)]$/;
|
|
21
|
+
function vectorDimensionOf(fieldType) {
|
|
22
|
+
const match = VECTOR_PATTERN.exec(fieldType);
|
|
23
|
+
return match === null ? null : Number.parseInt(match[1], 10);
|
|
24
|
+
}
|
|
20
25
|
function isGeopointOrVectorType(fieldType) {
|
|
21
26
|
return fieldType === "geopoint" || fieldType !== void 0 && VECTOR_PATTERN.test(fieldType);
|
|
22
27
|
}
|
|
@@ -58,15 +63,15 @@ function invalidType(path, type, message) {
|
|
|
58
63
|
}
|
|
59
64
|
function parseBase(baseName, path, type) {
|
|
60
65
|
if (isNamedBase(baseName)) return { base: baseName, dimension: 0 };
|
|
61
|
-
const vectorMatch =
|
|
66
|
+
const vectorMatch = VECTOR_TYPE_PATTERN.exec(baseName);
|
|
62
67
|
if (vectorMatch === null) {
|
|
63
68
|
invalidType(path, type, `Field "${path}" has an unsupported type: "${type}"`);
|
|
64
69
|
}
|
|
65
|
-
const dimension = Number.
|
|
66
|
-
if (dimension
|
|
70
|
+
const dimension = vectorDimensionOf(baseName) ?? Number.NaN;
|
|
71
|
+
if (!(dimension > 0 && dimension <= MAX_STORED_VECTOR_DIMENSION)) {
|
|
67
72
|
throw new NarsilError(
|
|
68
73
|
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
69
|
-
`Vector field "${path}" has ${
|
|
74
|
+
`Vector field "${path}" has ${vectorMatch[1]} dimensions, but a vector field holds from 1 to ${MAX_STORED_VECTOR_DIMENSION}`,
|
|
70
75
|
{ field: path, dimension, maximum: MAX_STORED_VECTOR_DIMENSION }
|
|
71
76
|
);
|
|
72
77
|
}
|
|
@@ -219,7 +224,7 @@ function validateVector(path, value, expectedDimension) {
|
|
|
219
224
|
);
|
|
220
225
|
}
|
|
221
226
|
for (let i = 0; i < arr.length; i++) {
|
|
222
|
-
if (typeof arr[i] !== "number" || !Number.isFinite(arr[i])) {
|
|
227
|
+
if (typeof arr[i] !== "number" || !Number.isFinite(Math.fround(arr[i]))) {
|
|
223
228
|
throw new NarsilError(
|
|
224
229
|
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
225
230
|
`Field "${path}" vector element at index ${i} is not a valid number`,
|
|
@@ -803,7 +808,6 @@ function getOrCreateFieldNameIndex(table, fieldName) {
|
|
|
803
808
|
table.indexMap.set(fieldName, idx);
|
|
804
809
|
return idx;
|
|
805
810
|
}
|
|
806
|
-
var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
|
|
807
811
|
function getNestedValue(obj, path) {
|
|
808
812
|
if (obj === null || typeof obj !== "object") return void 0;
|
|
809
813
|
if (!path.includes(".")) return Object.hasOwn(obj, path) ? obj[path] : void 0;
|
|
@@ -839,8 +843,7 @@ function textFieldsChanged(oldDoc, newDoc, fields) {
|
|
|
839
843
|
return false;
|
|
840
844
|
}
|
|
841
845
|
function parseVectorDimension(fieldType) {
|
|
842
|
-
|
|
843
|
-
return match ? Number.parseInt(match[1], 10) : null;
|
|
846
|
+
return vectorDimensionOf(fieldType);
|
|
844
847
|
}
|
|
845
848
|
function tokenizeOptions(options) {
|
|
846
849
|
return {
|
|
@@ -848,7 +851,8 @@ function tokenizeOptions(options) {
|
|
|
848
851
|
removeStopWords: true,
|
|
849
852
|
collectSurfaces: options?.collectSurfaces === true,
|
|
850
853
|
stopWordOverride: options?.stopWordOverride,
|
|
851
|
-
customTokenizer: options?.customTokenizer
|
|
854
|
+
customTokenizer: options?.customTokenizer,
|
|
855
|
+
textSource: "document"
|
|
852
856
|
};
|
|
853
857
|
}
|
|
854
858
|
function getFlatSchema(state, schema) {
|
|
@@ -863,7 +867,7 @@ var loaded = null;
|
|
|
863
867
|
var loading = null;
|
|
864
868
|
function loadMessagePack() {
|
|
865
869
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
866
|
-
loading ??= import('./msgpack-leaf-
|
|
870
|
+
loading ??= import('./msgpack-leaf-7KBYOVTF.mjs').then(
|
|
867
871
|
(code) => {
|
|
868
872
|
code.bindMsgpackHost({
|
|
869
873
|
BYTE_ORDER_MARK_WALK_BYTES_PER_HEADER,
|
|
@@ -896,4 +900,4 @@ function requireLoadedMessagePack() {
|
|
|
896
900
|
return loaded;
|
|
897
901
|
}
|
|
898
902
|
|
|
899
|
-
export { ANTIMERIDIAN_SPAN_DEGREES, DEG_TO_RAD, EARTH_RADIUS_METERS, FULL_TURN_DEGREES, LOCATION_CELLS_PER_AXIS, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_DOCUMENT_NESTING_DEPTH, MAX_LATITUDE_DEGREES, MAX_LONGITUDE_DEGREES, MAX_NESTING_DEPTH, MAX_PATTERN_VALUE_LIMIT, MAX_STORED_VECTOR_DIMENSION, MIN_PATTERN_VALUE_LIMIT, MIN_POLYGON_POINTS, PROTOTYPE_POLLUTION_KEYS, VECTOR_PATTERN, extractVectorFieldsFromSchema, flattenSchema, getFlatSchema, getNestedValue, getOrCreateFieldNameIndex, isGeopointOrVectorType, isLatitude, isListBase, isLongitude, isPlainObject, isQuantizationMode, isSingleStringFieldType, isStorageMode, isStringValuedBase, isWordIndexedBase, isWordIndexedFieldType, loadMessagePack, loadedMessagePack, messagePackUpperBoundBytes, parseVectorDimension, parsedTypeOf, partialFieldsOf, raiseEarlierReleaseGraphSetting, requireCreatableSchema, requireDocumentWithinSizeLimit, requireLoadedMessagePack, requireValidFieldTypes, schemaFieldsOf, textFieldsChanged, tokenizeOptions, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateTextStorage, validateVectorPromotion, validateVectorStorage, withSortableOption };
|
|
903
|
+
export { ANTIMERIDIAN_SPAN_DEGREES, DEG_TO_RAD, EARTH_RADIUS_METERS, FULL_TURN_DEGREES, LOCATION_CELLS_PER_AXIS, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_DOCUMENT_NESTING_DEPTH, MAX_LATITUDE_DEGREES, MAX_LONGITUDE_DEGREES, MAX_NESTING_DEPTH, MAX_PATTERN_VALUE_LIMIT, MAX_STORED_VECTOR_DIMENSION, MIN_PATTERN_VALUE_LIMIT, MIN_POLYGON_POINTS, PROTOTYPE_POLLUTION_KEYS, VECTOR_PATTERN, extractVectorFieldsFromSchema, flattenSchema, getFlatSchema, getNestedValue, getOrCreateFieldNameIndex, isGeopointOrVectorType, isLatitude, isListBase, isLongitude, isPlainObject, isQuantizationMode, isSingleStringFieldType, isStorageMode, isStringValuedBase, isWordIndexedBase, isWordIndexedFieldType, loadMessagePack, loadedMessagePack, messagePackUpperBoundBytes, parseVectorDimension, parsedTypeOf, partialFieldsOf, raiseEarlierReleaseGraphSetting, requireCreatableSchema, requireDocumentWithinSizeLimit, requireLoadedMessagePack, requireValidFieldTypes, schemaFieldsOf, textFieldsChanged, tokenizeOptions, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateTextStorage, validateVectorPromotion, validateVectorStorage, vectorDimensionOf, withSortableOption };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { loadStorageCode, resolvePartialWordAnalysis, resolvePartitionInsertOptions, yieldToEventLoop, createPartitionIndex, writePartitionDocument } from './chunk-
|
|
1
|
+
import { loadStorageCode, resolvePartialWordAnalysis, resolvePartitionInsertOptions, yieldToEventLoop, createPartitionIndex, writePartitionDocument } from './chunk-2C3BXZGA.mjs';
|
|
2
2
|
|
|
3
3
|
// src/partitioning/constants.ts
|
|
4
4
|
var REBALANCE_CHUNK_SIZE = 1e3;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { syncStorageOf, INDEX_CONFIGURATION_ENVELOPE_VERSION, MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, encodeIndexConfiguration } from './chunk-
|
|
2
|
-
import { archiveLayout } from './chunk-
|
|
3
|
-
import { manifestFileRefs } from './chunk-
|
|
1
|
+
import { syncStorageOf, INDEX_CONFIGURATION_ENVELOPE_VERSION, MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, encodeIndexConfiguration } from './chunk-LOQHA6QK.mjs';
|
|
2
|
+
import { archiveLayout } from './chunk-UISV46WX.mjs';
|
|
3
|
+
import { manifestFileRefs } from './chunk-KHJPMZE2.mjs';
|
|
4
4
|
import { DIRECTORY_IDENTITY_CHECK_INTERVAL_MS } from './chunk-QDZJM7NI.mjs';
|
|
5
5
|
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
6
|
-
import { openChecksummedEnvelope } from './chunk-
|
|
7
|
-
import { withOpenIndex, emitEngineEvent } from './chunk-
|
|
6
|
+
import { openChecksummedEnvelope } from './chunk-JCWC75NH.mjs';
|
|
7
|
+
import { withOpenIndex, emitEngineEvent } from './chunk-2C3BXZGA.mjs';
|
|
8
8
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
9
9
|
import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
10
10
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { bindResponseStop, sendError, RELAY_STREAM_CHUNKS_IN_FLIGHT } from './chunk-
|
|
1
|
+
import { bindResponseStop, sendError, RELAY_STREAM_CHUNKS_IN_FLIGHT } from './chunk-XTSFRWAS.mjs';
|
|
2
2
|
import { ServerErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
3
3
|
import { rm } from 'fs/promises';
|
|
4
4
|
import path from 'path';
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { restoredEmbedding, restoredPartitionConfig, readSharedConfigFields } from './chunk-
|
|
1
|
+
import { restoredEmbedding, restoredPartitionConfig, readSharedConfigFields } from './chunk-X4UMDSB4.mjs';
|
|
2
2
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-JDDXV6UF.mjs';
|
|
3
|
-
import { storedStateOf } from './chunk-
|
|
4
|
-
import { getLanguage, generateId, decodeSynonymRules } from './chunk-
|
|
3
|
+
import { storedStateOf } from './chunk-JCWC75NH.mjs';
|
|
4
|
+
import { getLanguage, generateId, decodeSynonymRules } from './chunk-2C3BXZGA.mjs';
|
|
5
5
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
6
6
|
import { isSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
7
|
-
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-
|
|
7
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-JP77OWMI.mjs';
|
|
8
8
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
9
9
|
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
10
10
|
import { encode } from '@msgpack/msgpack';
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { encodeJson, SYNC_CLOSE_SERVER_ERROR, SYNC_SUBPROTOCOL, encodeScore } from './chunk-FETZYCH7.mjs';
|
|
2
|
-
import { answerRefusalOf } from './chunk-
|
|
3
|
-
import { archiveRange } from './chunk-
|
|
4
|
-
import { readRecentMemoryFigures, readMemoryFigures, DEFAULT_RERANK_WINDOW } from './chunk-
|
|
5
|
-
import { collectGarbageBeforeRefusal, collectYoungGenerationBeforeRefusal, reportRefusalAfterCollection, reservedHeapBytes, admitHeapBytes } from './chunk-
|
|
6
|
-
import { MAX_FILTER_DEPTH } from './chunk-
|
|
7
|
-
import { bindSignalStop } from './chunk-
|
|
8
|
-
import { ANALYSIS_CHANGE_MEMBERS, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, MAX_SORT_FIELDS } from './chunk-
|
|
9
|
-
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-
|
|
2
|
+
import { answerRefusalOf } from './chunk-Y27E2MGC.mjs';
|
|
3
|
+
import { archiveRange } from './chunk-UISV46WX.mjs';
|
|
4
|
+
import { readRecentMemoryFigures, readMemoryFigures, DEFAULT_RERANK_WINDOW } from './chunk-2C3BXZGA.mjs';
|
|
5
|
+
import { collectGarbageBeforeRefusal, collectYoungGenerationBeforeRefusal, reportRefusalAfterCollection, reservedHeapBytes, admitHeapBytes } from './chunk-LPRRFCVU.mjs';
|
|
6
|
+
import { MAX_FILTER_DEPTH } from './chunk-YM2HI5N5.mjs';
|
|
7
|
+
import { bindSignalStop } from './chunk-DG6WIEB3.mjs';
|
|
8
|
+
import { ANALYSIS_CHANGE_MEMBERS, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, MAX_SORT_FIELDS } from './chunk-JP77OWMI.mjs';
|
|
9
|
+
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-D6FYUTFN.mjs';
|
|
10
10
|
import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
11
11
|
import { randomUUID } from 'crypto';
|
|
12
12
|
import { mkdtemp, open, rm } from 'fs/promises';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { readRecentMemoryFigures, readMemoryFigures } from './chunk-
|
|
2
|
-
import { collectGarbageBeforeAnswerRefusal, collectYoungGenerationBeforeRefusal, reservedHeapBytes, admitHeapBytes } from './chunk-
|
|
1
|
+
import { readRecentMemoryFigures, readMemoryFigures } from './chunk-2C3BXZGA.mjs';
|
|
2
|
+
import { collectGarbageBeforeAnswerRefusal, collectYoungGenerationBeforeRefusal, reservedHeapBytes, admitHeapBytes } from './chunk-LPRRFCVU.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes, ServerErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
4
4
|
|
|
5
5
|
// src/distribution/cluster-node/constants.ts
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { requireLoadedMessagePack, getNestedValue, parsedTypeOf } from './chunk-
|
|
1
|
+
import { requireLoadedMessagePack, getNestedValue, parsedTypeOf } from './chunk-X5NHWOXT.mjs';
|
|
2
2
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
3
3
|
import { codePointCount, exceedsCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
4
4
|
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { loadFailed, cachedFileBytes, COLD_LOAD_WAIT_WITHOUT_SEARCH_MS, readWholeSectionChecked, checkEverySectionPage, readFileRange, openPagedFile, readCacheOfThisThread, createSectionReader, closePagedFile, COLD_GRAPH_PART_VECTORS, decodeSections, readListedFile, requireListedChecksum, requireListedSize, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, failNativePage, COLD_WALK_NODES_READ_AHEAD } from './chunk-
|
|
2
|
-
import { isDead, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors, decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, osqRecordBytes, osqBitsOf, countDead, magnitude, rescoreDepth, defaultEfSearch, computeCodeBlockLayout, toScore, osqQuantize, createOsqScratch, stageQueryLevels, scalarQueryProducts, readTrailer, osqEstimate, osqDistance, toDistance, createFixedBuffer } from './chunk-
|
|
3
|
-
import { HEADER_SIZE, openChecksummedEnvelope, crc32, readHeader } from './chunk-
|
|
4
|
-
import { yieldToEventLoop, compareScores } from './chunk-
|
|
1
|
+
import { loadFailed, cachedFileBytes, COLD_LOAD_WAIT_WITHOUT_SEARCH_MS, readWholeSectionChecked, checkEverySectionPage, readFileRange, openPagedFile, readCacheOfThisThread, createSectionReader, closePagedFile, COLD_GRAPH_PART_VECTORS, decodeSections, readListedFile, requireListedChecksum, requireListedSize, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES, failNativePage, COLD_WALK_NODES_READ_AHEAD } from './chunk-ZB5X5WAK.mjs';
|
|
2
|
+
import { isDead, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors, decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, requireBoundedUpperLists, osqRecordBytes, osqBitsOf, countDead, magnitude, rescoreDepth, defaultEfSearch, computeCodeBlockLayout, toScore, osqQuantize, createOsqScratch, stageQueryLevels, scalarQueryProducts, readTrailer, osqEstimate, osqDistance, toDistance, createFixedBuffer } from './chunk-TNEU32EN.mjs';
|
|
3
|
+
import { HEADER_SIZE, openChecksummedEnvelope, crc32, readHeader } from './chunk-JCWC75NH.mjs';
|
|
4
|
+
import { yieldToEventLoop, compareScores } from './chunk-2C3BXZGA.mjs';
|
|
5
5
|
import { decode, decodeMulti } from './chunk-GXCDWLQI.mjs';
|
|
6
|
-
import { signalOfSearch, requireSearchActive } from './chunk-
|
|
7
|
-
import { MAX_LAYER_CAP } from './chunk-ZXX5GZEX.mjs';
|
|
6
|
+
import { signalOfSearch, requireSearchActive } from './chunk-DG6WIEB3.mjs';
|
|
7
|
+
import { MIN_M, MAX_M, MAX_LAYER_CAP } from './chunk-ZXX5GZEX.mjs';
|
|
8
8
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
9
9
|
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
10
10
|
|
|
@@ -663,6 +663,9 @@ function openParts(input) {
|
|
|
663
663
|
}
|
|
664
664
|
const { entryPoint, maxLayer, m } = layout;
|
|
665
665
|
if (typeof maxLayer !== "number" || typeof m !== "number") loadFailed(`The head of "${ref.key}" holds no layers`);
|
|
666
|
+
if (!Number.isInteger(m) || m < MIN_M || m > MAX_M) {
|
|
667
|
+
loadFailed(`The head of "${ref.key}" gives m ${m}, while m must be a whole number from ${MIN_M} to ${MAX_M}`);
|
|
668
|
+
}
|
|
666
669
|
if (entryPoint !== null && typeof entryPoint !== "number") loadFailed(`The head of "${ref.key}" holds no entry`);
|
|
667
670
|
if (graph === null) graph = { entryPoint, maxLayer, m };
|
|
668
671
|
const levels = readWholeSectionChecked(path, layout.levels, false);
|
|
@@ -674,6 +677,13 @@ function openParts(input) {
|
|
|
674
677
|
neighbours: createSectionReader(file, layout.neighbours)
|
|
675
678
|
});
|
|
676
679
|
}
|
|
680
|
+
if (graph !== null) {
|
|
681
|
+
requireBoundedUpperLists(
|
|
682
|
+
parts.map((part) => part.levels),
|
|
683
|
+
graph.m,
|
|
684
|
+
`The vector graph parts of field "${input.fieldPath}"`
|
|
685
|
+
);
|
|
686
|
+
}
|
|
677
687
|
} catch (error) {
|
|
678
688
|
closeHeldFiles(parts);
|
|
679
689
|
throw error;
|
|
@@ -714,7 +724,10 @@ function validateColdLayout(input, opened, parts, graph) {
|
|
|
714
724
|
// src/persistence/paged/cold-field/index.ts
|
|
715
725
|
var FIXSTR = 160;
|
|
716
726
|
var FIXSTR_MAX_BYTES = 31;
|
|
727
|
+
var STR8_MAX_BYTES = 255;
|
|
728
|
+
var STR16_MAX_BYTES = 65535;
|
|
717
729
|
var ASCII_MAX = 127;
|
|
730
|
+
var ID_OFFSET_BYTES = 4;
|
|
718
731
|
function shortAsciiIdOf(encoded) {
|
|
719
732
|
const length = encoded[0] - FIXSTR;
|
|
720
733
|
if (length < 0 || length > FIXSTR_MAX_BYTES || encoded.length !== length + 1) return null;
|
|
@@ -819,6 +832,35 @@ function createFieldData(input, opened, graph, parts) {
|
|
|
819
832
|
}
|
|
820
833
|
};
|
|
821
834
|
}
|
|
835
|
+
function utf8BytesOfEncodedId(encodedBytes) {
|
|
836
|
+
if (encodedBytes <= FIXSTR_MAX_BYTES + 1) return encodedBytes - 1;
|
|
837
|
+
if (encodedBytes <= STR8_MAX_BYTES + 2) return encodedBytes - 2;
|
|
838
|
+
if (encodedBytes <= STR16_MAX_BYTES + 3) return encodedBytes - 3;
|
|
839
|
+
return encodedBytes - 5;
|
|
840
|
+
}
|
|
841
|
+
function claimsOfLoad(data) {
|
|
842
|
+
const offsets = new Uint8Array(SEGMENT_PAGE_BYTES);
|
|
843
|
+
const view = new DataView(offsets.buffer);
|
|
844
|
+
const offsetsPerRead = offsets.byteLength / ID_OFFSET_BYTES - 1;
|
|
845
|
+
let slots = 0;
|
|
846
|
+
let docIdBytes = 0;
|
|
847
|
+
for (const file of data.files) {
|
|
848
|
+
for (let first = 0; first < file.count; first += offsetsPerRead) {
|
|
849
|
+
const count = Math.min(offsetsPerRead, file.count - first);
|
|
850
|
+
file.idOffsets.read(first * ID_OFFSET_BYTES, offsets.subarray(0, (count + 1) * ID_OFFSET_BYTES));
|
|
851
|
+
for (let at = 0; at < count; at++) {
|
|
852
|
+
const position = first + at;
|
|
853
|
+
if (isDead(file.dead, position) && data.topLayerOf(file.firstNumber + position) < 0) continue;
|
|
854
|
+
const start = view.getUint32(at * ID_OFFSET_BYTES, true);
|
|
855
|
+
const end = view.getUint32((at + 1) * ID_OFFSET_BYTES, true);
|
|
856
|
+
if (end <= start) loadFailed(`The ID offsets of "${file.file.path}" hold no ID at position ${position}`);
|
|
857
|
+
slots += 1;
|
|
858
|
+
docIdBytes += utf8BytesOfEncodedId(end - start);
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
return { slots, docIdBytes };
|
|
863
|
+
}
|
|
822
864
|
function positionOfId(file, docId, idAt, slot, view) {
|
|
823
865
|
let low = 0;
|
|
824
866
|
let high = file.count - 1;
|
|
@@ -875,6 +917,9 @@ function openColdVectorField(input) {
|
|
|
875
917
|
}
|
|
876
918
|
let residentBytes = 0;
|
|
877
919
|
for (const part of parts) residentBytes += part.levels.byteLength + part.nodeOffsets.byteLength;
|
|
920
|
+
let encodedIdBytes = 0;
|
|
921
|
+
for (const file of opened.files) encodedIdBytes += file.ids.length;
|
|
922
|
+
let claimsOnceRead = null;
|
|
878
923
|
const orderSlot = new Uint8Array(4);
|
|
879
924
|
const orderView = new DataView(orderSlot.buffer);
|
|
880
925
|
function liveNumberOf(docId) {
|
|
@@ -893,6 +938,11 @@ function openColdVectorField(input) {
|
|
|
893
938
|
for (const part of parts) bytes += cachedFileBytes(part.file);
|
|
894
939
|
return bytes;
|
|
895
940
|
},
|
|
941
|
+
claimsAtMost: { slots: data.total, docIdBytes: encodedIdBytes },
|
|
942
|
+
claims() {
|
|
943
|
+
claimsOnceRead ??= claimsOfLoad(data);
|
|
944
|
+
return claimsOnceRead;
|
|
945
|
+
},
|
|
896
946
|
numberOf: liveNumberOf,
|
|
897
947
|
vectorOfNumber: (number) => data.vectorAt(number),
|
|
898
948
|
reusedVectorOfNumber: (number) => data.reusedVectorAt(number),
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { MAX_HISTORY_FILE_BYTES, openChecksummedEnvelope, KEY_FINGERPRINT_BYTES, DAY_MS, dayFileKey, SCRAMBLED_ID_BYTES, restoreStagingPrefix, resolveSetting, QUERY_HISTORY_ARCHIVE_PREFIX, restoredFileKey, attachedRuntimeOf, MAX_INDEX_UUID_LENGTH, HEADER_SIZE, isGroup, isNormalPhrase, MAX_MAX_LENGTH, MAX_MAX_WORDS, listHistoryFolder, historyFileOf, MIN_PHRASE_TABLE_CAPACITY, PERSON_TABLE_MIN_SLOTS, PAIR_PAGE_BLOCKS, PAIR_BLOCK_SLOTS, packEnvelopeWithChecksum, crc32, MAX_PEOPLE_PER_PHRASE_IN_FILE } from './chunk-
|
|
2
|
-
import { loadSuggestionsCode, emitWriteEventAfter, dropEngineIndex } from './chunk-
|
|
1
|
+
import { MAX_HISTORY_FILE_BYTES, openChecksummedEnvelope, KEY_FINGERPRINT_BYTES, DAY_MS, dayFileKey, SCRAMBLED_ID_BYTES, restoreStagingPrefix, resolveSetting, QUERY_HISTORY_ARCHIVE_PREFIX, restoredFileKey, attachedRuntimeOf, MAX_INDEX_UUID_LENGTH, HEADER_SIZE, isGroup, isNormalPhrase, MAX_MAX_LENGTH, MAX_MAX_WORDS, listHistoryFolder, historyFileOf, MIN_PHRASE_TABLE_CAPACITY, PERSON_TABLE_MIN_SLOTS, PAIR_PAGE_BLOCKS, PAIR_BLOCK_SLOTS, packEnvelopeWithChecksum, crc32, MAX_PEOPLE_PER_PHRASE_IN_FILE } from './chunk-JCWC75NH.mjs';
|
|
2
|
+
import { loadSuggestionsCode, emitWriteEventAfter, dropEngineIndex } from './chunk-2C3BXZGA.mjs';
|
|
3
3
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
4
4
|
|
|
5
5
|
// src/query-suggestions/engine-hooks.ts
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { SEGMENT_MANIFEST_VERSION, MANIFEST_ENVELOPE_VERSION } from './chunk-
|
|
2
|
-
import { encode } from './chunk-
|
|
3
|
-
import { createFixedBuffer, createVectorFileReader } from './chunk-
|
|
4
|
-
import { crc32, packChecksummedEnvelope, HEADER_SIZE } from './chunk-
|
|
1
|
+
import { SEGMENT_MANIFEST_VERSION, MANIFEST_ENVELOPE_VERSION } from './chunk-KHJPMZE2.mjs';
|
|
2
|
+
import { encode } from './chunk-CV3Q2EKO.mjs';
|
|
3
|
+
import { createFixedBuffer, createVectorFileReader } from './chunk-TNEU32EN.mjs';
|
|
4
|
+
import { crc32, packChecksummedEnvelope, HEADER_SIZE } from './chunk-JCWC75NH.mjs';
|
|
5
5
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
6
|
-
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-
|
|
6
|
+
import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-D6FYUTFN.mjs';
|
|
7
7
|
import { ErrorCodes, NarsilError } from './chunk-Q3ZZH5Z7.mjs';
|
|
8
8
|
|
|
9
9
|
// src/persistence/durability/segment/store.ts
|
|
@@ -569,7 +569,7 @@ function openPagedFile(path, cache) {
|
|
|
569
569
|
const native = openNativePagedFile(path);
|
|
570
570
|
readersOfPath.set(path, (readersOfPath.get(path) ?? 0) + 1);
|
|
571
571
|
try {
|
|
572
|
-
return { path, fileNumber: cache.fileNumber(), cache, native, closed: false };
|
|
572
|
+
return { path, fileNumber: cache.fileNumber(), cache, native, closed: false, suspended: false };
|
|
573
573
|
} catch (error) {
|
|
574
574
|
native?.core.closePagedFile(native.handle);
|
|
575
575
|
stopReadingPath(path);
|
|
@@ -585,27 +585,31 @@ function stopReadingPath(path) {
|
|
|
585
585
|
readersOfPath.delete(path);
|
|
586
586
|
fileReaderOfThisThread().retainOnly(new Set(readersOfPath.keys()));
|
|
587
587
|
}
|
|
588
|
-
function
|
|
589
|
-
if (file.closed) return;
|
|
590
|
-
file.closed = true;
|
|
588
|
+
function closeNativeHandle(file) {
|
|
591
589
|
if (file.native !== null) file.cache.releasePageFile(file.native.handle.identity);
|
|
592
590
|
file.native?.core.closePagedFile(file.native.handle);
|
|
593
591
|
file.native = null;
|
|
592
|
+
}
|
|
593
|
+
function closePagedFile(file) {
|
|
594
|
+
if (file.closed) return;
|
|
595
|
+
file.closed = true;
|
|
596
|
+
closeNativeHandle(file);
|
|
594
597
|
file.cache.releaseFile(file.fileNumber);
|
|
598
|
+
if (file.suspended) return;
|
|
595
599
|
stopReadingPath(file.path);
|
|
596
600
|
}
|
|
597
|
-
function
|
|
598
|
-
if (
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
file.native?.core.closePagedFile(file.native.handle);
|
|
602
|
-
file.native = native;
|
|
601
|
+
function suspendPagedFile(file) {
|
|
602
|
+
if (file.closed || file.suspended) return;
|
|
603
|
+
file.suspended = true;
|
|
604
|
+
closeNativeHandle(file);
|
|
603
605
|
stopReadingPath(file.path);
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
606
|
+
}
|
|
607
|
+
function resumePagedFile(file, path) {
|
|
608
|
+
if (file.closed || !file.suspended) return;
|
|
609
|
+
file.suspended = false;
|
|
608
610
|
file.path = path;
|
|
611
|
+
readersOfPath.set(path, (readersOfPath.get(path) ?? 0) + 1);
|
|
612
|
+
file.native = openNativePagedFile(path);
|
|
609
613
|
}
|
|
610
614
|
function unreadable(path, offset, target) {
|
|
611
615
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The file "${path}" cannot be read at byte ${offset}`, {
|
|
@@ -785,4 +789,4 @@ function readWholeSectionChecked(path, section, shared) {
|
|
|
785
789
|
return target;
|
|
786
790
|
}
|
|
787
791
|
|
|
788
|
-
export { BLOCK_SUMMARY_BYTES, CACHED_DOCUMENT_BLOCK, CACHED_LIST, CACHED_POSITIONS, COLD_GRAPH_PART_VECTORS, COLD_LOAD_WAIT_WITHOUT_SEARCH_MS, COLD_WALK_NODES_READ_AHEAD, DOCUMENT_BLOCK_BYTES, DOCUMENT_BLOCK_MIN_RAW_BYTES, FILE_ASSEMBLY_CHUNK_BYTES, ID_SAMPLE_INTERVAL, LARGEST_REMOVAL_BUCKET_COUNT, LIST_VIEWS_HELD_PER_SEGMENT, MAX_DOCUMENT_BLOCK_RAW_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, MERGE_READ_CACHE_BYTES, PAGED_SEGMENT_ENVELOPE_VERSION, PAYLOAD_PREFIX_BYTES, POSITION_WIDTH_BYTES, POSTING_WIDTH_BYTES, POSTING_WRITE_POSITIONS, POSTING_WRITE_ROWS, READS_AHEAD_AT_ONCE, READ_AHEAD_CHUNK_BYTES, RECORD_PAGE_BYTES, REMOVAL_BUCKET_TARGET_BYTES, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, REWRITE_SPILL_BYTES, SECTION_ALIGNMENT_BYTES, SECTION_WRITE_BUFFER_BYTES, SEGMENT_PAGE_BYTES, SMALLEST_BLOCK_POSTINGS, STABLE_HEAD_ATTEMPTS, TOMBSTONES_BETWEEN_YIELDS, WRITER_YIELD_INTERVAL_TOKENS, adapterCheckpointStore, cachedFileBytes, checkEverySectionPage, closePagedFile, createReadCache, createSectionReader, decodePagedSegmentHead, decodeSections, directoryCheckpointStore, encodePagedSegmentHead, encodeSegmentManifest, failNativePage, loadFailed, openPagedFile, readCacheOfThisThread, readFileRange, readFileRangeLater, readListedFile, readWholeSectionChecked,
|
|
792
|
+
export { BLOCK_SUMMARY_BYTES, CACHED_DOCUMENT_BLOCK, CACHED_LIST, CACHED_POSITIONS, COLD_GRAPH_PART_VECTORS, COLD_LOAD_WAIT_WITHOUT_SEARCH_MS, COLD_WALK_NODES_READ_AHEAD, DOCUMENT_BLOCK_BYTES, DOCUMENT_BLOCK_MIN_RAW_BYTES, FILE_ASSEMBLY_CHUNK_BYTES, ID_SAMPLE_INTERVAL, LARGEST_REMOVAL_BUCKET_COUNT, LIST_VIEWS_HELD_PER_SEGMENT, MAX_DOCUMENT_BLOCK_RAW_BYTES, MERGED_SEGMENT_GROWTH_ALLOWANCE, MERGED_SEGMENT_TARGET_BYTES, MERGE_READ_CACHE_BYTES, PAGED_SEGMENT_ENVELOPE_VERSION, PAYLOAD_PREFIX_BYTES, POSITION_WIDTH_BYTES, POSTING_WIDTH_BYTES, POSTING_WRITE_POSITIONS, POSTING_WRITE_ROWS, READS_AHEAD_AT_ONCE, READ_AHEAD_CHUNK_BYTES, RECORD_PAGE_BYTES, REMOVAL_BUCKET_TARGET_BYTES, REMOVAL_BUCKET_WRITE_BUFFER_BYTES, REWRITE_SPILL_BYTES, SECTION_ALIGNMENT_BYTES, SECTION_WRITE_BUFFER_BYTES, SEGMENT_PAGE_BYTES, SMALLEST_BLOCK_POSTINGS, STABLE_HEAD_ATTEMPTS, TOMBSTONES_BETWEEN_YIELDS, WRITER_YIELD_INTERVAL_TOKENS, adapterCheckpointStore, cachedFileBytes, checkEverySectionPage, closePagedFile, createReadCache, createSectionReader, decodePagedSegmentHead, decodeSections, directoryCheckpointStore, encodePagedSegmentHead, encodeSegmentManifest, failNativePage, loadFailed, openPagedFile, readCacheOfThisThread, readFileRange, readFileRangeLater, readListedFile, readWholeSectionChecked, requireListedChecksum, requireListedSize, requireNativePage, resumePagedFile, sectionNamed, suspendPagedFile };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { P as PartitionConfig, B as BM25Params, A as AnyDocument, a as InsertOptions, W as WriteOptions } from './schema-
|
|
2
|
-
import { e as SnapshotOptions, V as VectorMaintenanceResult, B as BatchResult, L as ListParams, l as ListResult, I as IndexInfo, p as IndexStats, P as PartitionStatsResult, Q as QueryParams, j as QueryResult, o as PreflightResult, m as SuggestParams, n as SuggestResult, g as SyncStatus, f as SyncSetting } from './engine-
|
|
1
|
+
import { P as PartitionConfig, B as BM25Params, A as AnyDocument, a as InsertOptions, W as WriteOptions } from './schema-CzG9OUw5.js';
|
|
2
|
+
import { e as SnapshotOptions, V as VectorMaintenanceResult, B as BatchResult, L as ListParams, l as ListResult, I as IndexInfo, p as IndexStats, P as PartitionStatsResult, Q as QueryParams, j as QueryResult, o as PreflightResult, m as SuggestParams, n as SuggestResult, g as SyncStatus, f as SyncSetting } from './engine-P-4xhUe4.js';
|
|
3
3
|
import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
|
|
4
|
-
import { f as TaskRecord, a as ChangeAnalysisBody, M as MemoryStatsResponse, m as CreateBackupBody, R as RestoreBackupBody, c as ImportResult, H as HttpIndexConfig, d as TaskListQuery, T as TaskListPage } from './tasks-
|
|
5
|
-
import { B as BackupListing } from './config-
|
|
4
|
+
import { f as TaskRecord, a as ChangeAnalysisBody, M as MemoryStatsResponse, m as CreateBackupBody, R as RestoreBackupBody, c as ImportResult, H as HttpIndexConfig, d as TaskListQuery, T as TaskListPage } from './tasks-CXtQ3Zpp.js';
|
|
5
|
+
import { B as BackupListing } from './config-CIpLXXxm.js';
|
|
6
6
|
import { d as QuerySuggestionsSetting, e as QuerySuggestionsStatus, f as RecordQueryResult, c as QuerySuggestionsPreview, M as ManualQuerySuggestion } from './query-suggestions-CBYkrQ5T.js';
|
|
7
7
|
|
|
8
8
|
/**
|
package/dist/client.d.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
export { B as BackupListing, a as BackupSummary, D as DamagedBackup } from './config-
|
|
2
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, e as SnapshotOptions, f as SyncSetting, g as SyncStatus, i as isNarsilError } from './engine-
|
|
3
|
-
export { A as ASYNC_IMPORT_CAPABILITY, B as BM25_TUNING_CAPABILITY, C as CHANGE_ANALYSIS_CAPABILITY, D as DOCUMENT_EXPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, R as REBUILD_ANALYSIS_CAPABILITY, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-
|
|
4
|
-
export { B as BackupListResponse, C as CapabilitiesResponse, a as ChangeAnalysisBody, b as CreateIndexEmbedding, E as ErrorDetail, H as HttpIndexConfig, I as ImportError, c as ImportResult, M as MemoryStatsResponse, R as RestoreBackupBody, T as TaskListPage, d as TaskListQuery, e as TaskProgress, f as TaskRecord, g as TaskStatus, h as TaskStepProgress, i as TaskType } from './tasks-
|
|
1
|
+
export { B as BackupListing, a as BackupSummary, D as DamagedBackup } from './config-CIpLXXxm.js';
|
|
2
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, e as SnapshotOptions, f as SyncSetting, g as SyncStatus, i as isNarsilError } from './engine-P-4xhUe4.js';
|
|
3
|
+
export { A as ASYNC_IMPORT_CAPABILITY, B as BM25_TUNING_CAPABILITY, C as CHANGE_ANALYSIS_CAPABILITY, D as DOCUMENT_EXPORT_CAPABILITY, E as ErrorEnvelope, I as INDEX_LIFECYCLE_CAPABILITY, R as REBUILD_ANALYSIS_CAPABILITY, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-BQIdsaWy.js';
|
|
4
|
+
export { B as BackupListResponse, C as CapabilitiesResponse, a as ChangeAnalysisBody, b as CreateIndexEmbedding, E as ErrorDetail, H as HttpIndexConfig, I as ImportError, c as ImportResult, M as MemoryStatsResponse, R as RestoreBackupBody, T as TaskListPage, d as TaskListQuery, e as TaskProgress, f as TaskRecord, g as TaskStatus, h as TaskStepProgress, i as TaskType } from './tasks-CXtQ3Zpp.js';
|
|
5
5
|
export { M as ManualQuerySuggestion, Q as QuerySuggestionPreviewEntry, a as QuerySuggestionsBuild, b as QuerySuggestionsBuildTrigger, c as QuerySuggestionsPreview, d as QuerySuggestionsSetting, e as QuerySuggestionsStatus, R as RecordQueryOutcome, f as RecordQueryResult } from './query-suggestions-CBYkrQ5T.js';
|
|
6
|
-
export { A as AdminOperations, B as BackupOperations, a as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, b as IndexOperations, N as NarsilClient, c as NarsilClientOptions, P as PutResult, Q as QuerySuggestionOperations, R as RequestOptions, S as SearchOperations, d as ServerOperations, e as ServerVersion, f as SyncSettingOperations, g as SyncSnapshotDownload, h as SyncSnapshotResume, T as TaskOperations, W as WaitForTaskOptions, i as createNarsilClient } from './client-
|
|
7
|
-
import './config-
|
|
6
|
+
export { A as AdminOperations, B as BackupOperations, a as BulkOperations, D as DocumentOperations, F as FetchFunction, I as ImportSource, b as IndexOperations, N as NarsilClient, c as NarsilClientOptions, P as PutResult, Q as QuerySuggestionOperations, R as RequestOptions, S as SearchOperations, d as ServerOperations, e as ServerVersion, f as SyncSettingOperations, g as SyncSnapshotDownload, h as SyncSnapshotResume, T as TaskOperations, W as WaitForTaskOptions, i as createNarsilClient } from './client-CeUxvIvd.js';
|
|
7
|
+
import './config-DehU-VGL.js';
|
|
8
8
|
import './adapters-pljAqmrC.js';
|
|
9
|
-
import './schema-
|
|
9
|
+
import './schema-CzG9OUw5.js';
|
|
10
10
|
import './synonyms-CpOAHPa-.js';
|
|
11
|
-
import './index-
|
|
11
|
+
import './index-CBUl4z3E.js';
|
|
12
12
|
import './types-DvncmvuQ.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { N as NarsilConfig } from './config-
|
|
1
|
+
import { N as NarsilConfig } from './config-DehU-VGL.js';
|
|
2
2
|
import { P as PersistenceAdapter } from './adapters-pljAqmrC.js';
|
|
3
|
-
import { A as AllocationConstraints, a as AllocationTable, N as NodeRole, b as NodeRegistration, C as ClusterCoordinator, c as NodeCapacity } from './index-
|
|
3
|
+
import { A as AllocationConstraints, a as AllocationTable, N as NodeRole, b as NodeRegistration, C as ClusterCoordinator, c as NodeCapacity } from './index-CBUl4z3E.js';
|
|
4
4
|
import { N as NodeTransport } from './types-DvncmvuQ.js';
|
|
5
5
|
|
|
6
6
|
/**
|