@delali/narsil 0.3.1-next.163 → 0.3.1-next.169
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/browser/{chunk-MZHQ65VI.mjs → chunk-2Q45HI5H.mjs} +14 -10
- package/dist/browser/{chunk-MMHIIL5B.mjs → chunk-4ZICWS4V.mjs} +1 -1
- package/dist/browser/{chunk-KQXKCMGQ.mjs → chunk-JHFBQKUF.mjs} +119 -90
- package/dist/browser/{core-leaf-SIWXFZLV.mjs → core-leaf-WYFANMKC.mjs} +61 -13
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/{embed-RF2MPGAO.mjs → embed-24IPR4LP.mjs} +2 -2
- package/dist/browser/{module-PMQBRESM.mjs → module-XVLHJ7T7.mjs} +358 -62
- package/dist/browser/{optional-writes-WMZX4OEC.mjs → optional-writes-TB34FYUW.mjs} +118 -66
- package/dist/browser/{phrase-module-22TKMXWI.mjs → phrase-module-6UGFVW5J.mjs} +1 -0
- package/dist/browser/{resolve-vector-text-KI65IHMD.mjs → resolve-vector-text-EJ6DHFF5.mjs} +8 -0
- package/dist/browser/{storage-leaf-VGR2BWJJ.mjs → storage-leaf-QFXIWDFK.mjs} +208 -87
- package/dist/browser/{worker-copies-WQCPP3TS.mjs → worker-copies-ICHLYYXM.mjs} +1 -0
- package/dist/{capabilities-DkXAkb-X.d.ts → capabilities-BQIdsaWy.d.ts} +2 -2
- package/dist/{chunk-BKOMO7HT.mjs → chunk-26E5MO3Y.mjs} +5 -5
- package/dist/{chunk-EEYDJB4Z.mjs → chunk-2C3BXZGA.mjs} +213 -149
- package/dist/{chunk-NXMGUDZA.mjs → chunk-2KWTQ2P7.mjs} +2 -2
- package/dist/{chunk-TW543F33.mjs → chunk-34IOOYXA.mjs} +1 -1
- package/dist/{chunk-D5GNFN6X.mjs → chunk-7FC2ZILY.mjs} +98 -40
- package/dist/{chunk-XRBPHCCT.mjs → chunk-A4XYDUVN.mjs} +5 -5
- package/dist/{chunk-R4F7ZTVG.mjs → chunk-BIIROQ5Q.mjs} +2 -2
- package/dist/{chunk-Q2WQSAYH.mjs → chunk-CQX5B6L2.mjs} +54 -11
- package/dist/{chunk-XXWEUXE4.mjs → chunk-D6FYUTFN.mjs} +1 -1
- package/dist/{chunk-OUWZKNO3.mjs → chunk-DG6WIEB3.mjs} +1 -1
- package/dist/{chunk-GJKFXKMI.mjs → chunk-F3FDHXCJ.mjs} +12 -9
- package/dist/{chunk-3UGZBRDN.mjs → chunk-GO6TRSJ6.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-OK3WUA6N.mjs → chunk-I2WJCRIH.mjs} +1 -1
- package/dist/{chunk-Q7CUUSJ2.mjs → chunk-ISJ35523.mjs} +2 -2
- package/dist/{chunk-GQLOEACG.mjs → chunk-JCWC75NH.mjs} +2 -2
- package/dist/{chunk-ZGYF37MG.mjs → chunk-JDIATDPY.mjs} +1 -1
- package/dist/{chunk-3B775F3C.mjs → chunk-JP77OWMI.mjs} +3 -1
- package/dist/{chunk-KSOZJEYU.mjs → chunk-KHJPMZE2.mjs} +1 -1
- package/dist/{chunk-ZQ6QY42L.mjs → chunk-KL6W73BZ.mjs} +8 -0
- package/dist/{chunk-ES5EOXT6.mjs → chunk-LOQHA6QK.mjs} +3 -3
- package/dist/{chunk-AB27FWZM.mjs → chunk-LPRRFCVU.mjs} +2 -2
- package/dist/{chunk-EIR3IQCL.mjs → chunk-QFJMRB7D.mjs} +2 -1
- package/dist/{chunk-S3BSDBF7.mjs → chunk-QJGDB3WR.mjs} +1 -1
- package/dist/{chunk-EI7U43MX.mjs → chunk-R7PLSVK3.mjs} +2 -2
- package/dist/{chunk-5DUTOEK5.mjs → chunk-TBOZHYCF.mjs} +2 -2
- package/dist/{chunk-VJOS6MHF.mjs → chunk-TNEU32EN.mjs} +64 -16
- package/dist/{chunk-RXMR2SIQ.mjs → chunk-UISV46WX.mjs} +4 -4
- package/dist/{chunk-4CAMH7FO.mjs → chunk-X4UMDSB4.mjs} +2 -1
- package/dist/{chunk-R4CE4IVU.mjs → chunk-X5NHWOXT.mjs} +15 -11
- package/dist/{react-native/chunks/chunk-HEELWLKL.mjs → chunk-XFOL77WH.mjs} +1 -1
- package/dist/{chunk-TOZENMM7.mjs → chunk-XITVDRRH.mjs} +5 -5
- package/dist/{chunk-FSLQHOKZ.mjs → chunk-XNFBYAXI.mjs} +1 -1
- package/dist/{chunk-V4YIEMAI.mjs → chunk-XQQKETAF.mjs} +4 -4
- package/dist/{chunk-DSXFN5BF.mjs → chunk-XTSFRWAS.mjs} +8 -8
- package/dist/{chunk-K4BMDPPX.mjs → chunk-Y27E2MGC.mjs} +2 -2
- package/dist/{chunk-JV74RKCY.mjs → chunk-YM2HI5N5.mjs} +1 -1
- package/dist/{chunk-7OE73OO4.mjs → chunk-YPV2L7M5.mjs} +56 -6
- package/dist/{chunk-STLNPJXU.mjs → chunk-YXZDTKQS.mjs} +2 -2
- package/dist/{chunk-5P72V66D.mjs → chunk-ZB5X5WAK.mjs} +24 -20
- package/dist/{client-VJmV_AjZ.d.ts → client-CeUxvIvd.d.ts} +4 -4
- package/dist/client.d.ts +8 -8
- package/dist/{config-ZZg4cwQz.d.ts → config-CIpLXXxm.d.ts} +2 -2
- package/dist/{config-Cb3FOlE_.d.ts → config-DehU-VGL.d.ts} +2 -2
- package/dist/{core-leaf-LOXCD7MY.mjs → core-leaf-VIMO377O.mjs} +3 -3
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/{direct-executor-PF4UDSUW.mjs → direct-executor-2WQSGBN3.mjs} +7 -7
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +1 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +39 -39
- package/dist/{embed-4H7RTBVL.mjs → embed-UNHDFGDU.mjs} +2 -2
- package/dist/{engine-DdOs682x.d.ts → engine-P-4xhUe4.d.ts} +6 -2
- package/dist/{index-Ck9w5WHz.d.ts → index-CBUl4z3E.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +7 -7
- package/dist/index.react-native.mjs +6 -6
- package/dist/{long-query-thread-OTVLP2OI.mjs → long-query-thread-AAYTLYZK.mjs} +19 -19
- package/dist/{module-UIKSHY3G.mjs → module-5SXNUCD5.mjs} +315 -64
- package/dist/{msgpack-leaf-5WFRP6JM.mjs → msgpack-leaf-7KBYOVTF.mjs} +2 -2
- package/dist/{optional-reads-RSLNSGNG.mjs → optional-reads-ENKMVQ4Q.mjs} +1 -1
- package/dist/{optional-writes-4ITCV7YT.mjs → optional-writes-N2DFWWZ3.mjs} +39 -26
- package/dist/{paged-partition-archive-VMJ2AUA6.mjs → paged-partition-archive-X7L27E5Z.mjs} +19 -19
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +7 -7
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -17
- package/dist/{phrase-module-IWSG2DDZ.mjs → phrase-module-ZY3TLTQA.mjs} +2 -1
- package/dist/{query-actions-EW72OQXY.mjs → query-actions-2JOLPELC.mjs} +7 -7
- package/dist/query-suggestions/merge-worker.mjs +10 -10
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +8 -8
- package/dist/{chunk-ZBVHS5KF.mjs → react-native/chunks/chunk-57R74JWC.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-AUXXPSFB.mjs → chunk-F4MIGBBP.mjs} +214 -150
- package/dist/react-native/chunks/{chunk-3RUXKZJG.mjs → chunk-F6BJYCF2.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-RVHCQJQM.mjs → chunk-GGBATERA.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-I5W7A2JF.mjs → chunk-GVWLACAT.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-XOCM3BFU.mjs → chunk-H344ZXRH.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-VIPNDN4B.mjs → chunk-I7DTUOK2.mjs} +65 -15
- package/dist/react-native/chunks/{chunk-2JMAUU26.mjs → chunk-J2BQTJJE.mjs} +1 -1
- package/dist/{chunk-QBVTYY4J.mjs → react-native/chunks/chunk-OZ3XEM6D.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-4EXKQ2IB.mjs → chunk-SK3DPAI4.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-XBTFK3AZ.mjs → chunk-WFQ6YQZA.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-FTQ7JNMU.mjs → chunk-ZDDJVR7X.mjs} +15 -11
- package/dist/react-native/chunks/{core-leaf-O3236QY3.mjs → core-leaf-UDUMVC3M.mjs} +2 -2
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/{embed-3WQ2CCR5.mjs → embed-U7CIKEY4.mjs} +2 -2
- package/dist/react-native/chunks/{module-LRR7D5ZO.mjs → module-7BJ3VOWQ.mjs} +360 -66
- package/dist/react-native/chunks/{msgpack-leaf-TR66ZVAX.mjs → msgpack-leaf-GTKLF6GJ.mjs} +2 -2
- package/dist/react-native/chunks/{optional-reads-TOWGNEDH.mjs → optional-reads-74HYI4DO.mjs} +1 -1
- package/dist/react-native/chunks/{optional-writes-7AFBIZMN.mjs → optional-writes-TXTJ6HUZ.mjs} +37 -24
- package/dist/react-native/chunks/{phrase-module-3A4AKMKL.mjs → phrase-module-UC4YGEBB.mjs} +2 -1
- package/dist/react-native/chunks/{resolve-vector-text-LXRYXF7M.mjs → resolve-vector-text-FTPD5EY6.mjs} +8 -0
- package/dist/react-native/chunks/{restore-queue-7FSHTGJ5.mjs → restore-queue-SZLWIL3B.mjs} +9 -9
- package/dist/react-native/chunks/{segment-codec-leaf-4QV4HO4R.mjs → segment-codec-leaf-WMUZ5P4F.mjs} +7 -7
- package/dist/react-native/chunks/{segment-merge-leaf-5DUZZSXJ.mjs → segment-merge-leaf-Q2OEJIQ3.mjs} +8 -8
- package/dist/react-native/chunks/{storage-leaf-PNU6HWY7.mjs → storage-leaf-OXYV6O6P.mjs} +213 -100
- package/dist/react-native/chunks/{store-leaf-ESVUGYYO.mjs → store-leaf-ZMDXYK4E.mjs} +8 -8
- package/dist/react-native/chunks/{suggestions-leaf-MOVOPBGD.mjs → suggestions-leaf-FIMGQZ2T.mjs} +8 -8
- package/dist/react-native/chunks/{worker-copies-VQGF3WZX.mjs → worker-copies-72TGFYQB.mjs} +8 -7
- package/dist/react-native/partial-words.mjs +6 -6
- package/dist/react-native/pattern.mjs +3 -3
- package/dist/react-native/query-suggestions.mjs +6 -6
- package/dist/react-native/sync.mjs +6 -6
- package/dist/react-native/synonyms.mjs +6 -6
- package/dist/react.d.ts +9 -9
- package/dist/{rerank-QHJXPSAK.mjs → rerank-E2HFKKGJ.mjs} +7 -7
- package/dist/{resolve-vector-text-4PDRTPNQ.mjs → resolve-vector-text-6WPFFIX5.mjs} +1 -1
- package/dist/{restore-queue-5W5HX44H.mjs → restore-queue-PUVBN6L3.mjs} +10 -10
- package/dist/{schema-C_7yRJLt.d.ts → schema-CzG9OUw5.d.ts} +5 -1
- package/dist/{segment-codec-leaf-7VLVXKWO.mjs → segment-codec-leaf-24Z3MMQI.mjs} +8 -8
- package/dist/{segment-merge-leaf-22XR3IGA.mjs → segment-merge-leaf-FQQDU45E.mjs} +8 -8
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +26 -26
- package/dist/{storage-leaf-ZZ77C3DX.mjs → storage-leaf-KIA7XO6Y.mjs} +79 -81
- package/dist/{store-leaf-LJ6D2PMF.mjs → store-leaf-ESVLL4JO.mjs} +9 -9
- package/dist/{suggestions-leaf-KTQHO3PY.mjs → suggestions-leaf-FXF4AHAY.mjs} +11 -11
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +15 -15
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +3 -3
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +7 -7
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +7 -7
- package/dist/{tasks-CxA-J_Ty.d.ts → tasks-CXtQ3Zpp.d.ts} +4 -4
- package/dist/thread-OWXVP2ZA.mjs +30 -0
- package/dist/{types-BboB278g.d.ts → types-CDN70Y_C.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/{worker-copies-ODAAGQRC.mjs → worker-copies-XXX7PF4J.mjs} +10 -10
- package/dist/workers/entry.mjs +13 -13
- package/package.json +9 -8
- package/dist/thread-XWDK6TAR.mjs +0 -30
- /package/dist/{chunk-77AUVDQ5.mjs → chunk-CV3Q2EKO.mjs} +0 -0
|
@@ -10,6 +10,7 @@ var ANALYSIS_HOLD_APPLY_INTERVAL_MS;
|
|
|
10
10
|
var ARRAY_POSITION_GAP;
|
|
11
11
|
var DEFAULT_BM25_B;
|
|
12
12
|
var DEFAULT_BM25_K1;
|
|
13
|
+
var DEFAULT_MAX_PARTITIONS;
|
|
13
14
|
var EXPORT_CHUNK_BYTES;
|
|
14
15
|
var ErrorCodes;
|
|
15
16
|
var FOREIGN_STATS_TTL_MS;
|
|
@@ -21,6 +22,8 @@ var MAX_FILE_BYTES;
|
|
|
21
22
|
var MAX_IMPORT_REFUSALS_REPORTED;
|
|
22
23
|
var MAX_INDEX_NAME_LENGTH;
|
|
23
24
|
var MAX_LAYER_CAP;
|
|
25
|
+
var MAX_M;
|
|
26
|
+
var MAX_PARTITIONS;
|
|
24
27
|
var MIN_M;
|
|
25
28
|
var NarsilError;
|
|
26
29
|
var STATISTICS_BROADCAST_INTERVAL_MS;
|
|
@@ -122,6 +125,7 @@ function bindEngineNames(host) {
|
|
|
122
125
|
ARRAY_POSITION_GAP,
|
|
123
126
|
DEFAULT_BM25_B,
|
|
124
127
|
DEFAULT_BM25_K1,
|
|
128
|
+
DEFAULT_MAX_PARTITIONS,
|
|
125
129
|
EXPORT_CHUNK_BYTES,
|
|
126
130
|
ErrorCodes,
|
|
127
131
|
FOREIGN_STATS_TTL_MS,
|
|
@@ -133,6 +137,8 @@ function bindEngineNames(host) {
|
|
|
133
137
|
MAX_IMPORT_REFUSALS_REPORTED,
|
|
134
138
|
MAX_INDEX_NAME_LENGTH,
|
|
135
139
|
MAX_LAYER_CAP,
|
|
140
|
+
MAX_M,
|
|
141
|
+
MAX_PARTITIONS,
|
|
136
142
|
MIN_M,
|
|
137
143
|
NarsilError,
|
|
138
144
|
STATISTICS_BROADCAST_INTERVAL_MS,
|
|
@@ -396,6 +402,7 @@ var osqEstimate;
|
|
|
396
402
|
var osqQuantize;
|
|
397
403
|
var osqRecordBytes;
|
|
398
404
|
var readTrailer;
|
|
405
|
+
var requireBoundedUpperLists;
|
|
399
406
|
var rescoreDepth;
|
|
400
407
|
var scalarQueryProducts;
|
|
401
408
|
var stageQueryLevels;
|
|
@@ -435,6 +442,7 @@ function bindVectorCoreNames(vectorCore) {
|
|
|
435
442
|
osqQuantize,
|
|
436
443
|
osqRecordBytes,
|
|
437
444
|
readTrailer,
|
|
445
|
+
requireBoundedUpperLists,
|
|
438
446
|
rescoreDepth,
|
|
439
447
|
scalarQueryProducts,
|
|
440
448
|
stageQueryLevels,
|
|
@@ -1652,8 +1660,8 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
|
|
|
1652
1660
|
);
|
|
1653
1661
|
}
|
|
1654
1662
|
const entry = ctx.requireIndex(indexName);
|
|
1655
|
-
const maxPartitions = entry.config.partitions?.maxPartitions;
|
|
1656
|
-
if (
|
|
1663
|
+
const maxPartitions = entry.config.partitions?.maxPartitions ?? DEFAULT_MAX_PARTITIONS;
|
|
1664
|
+
if (targetPartitionCount > maxPartitions) {
|
|
1657
1665
|
throw new NarsilError(
|
|
1658
1666
|
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
1659
1667
|
`Target partition count (${targetPartitionCount}) is above the maximum of ${maxPartitions} partitions`,
|
|
@@ -1958,6 +1966,9 @@ async function driveRun(core, indexName, entry, run, previous, resumedGeneration
|
|
|
1958
1966
|
}
|
|
1959
1967
|
}
|
|
1960
1968
|
async function changeIndexAnalysis(core, indexName, change) {
|
|
1969
|
+
const changedTokenizer = change !== null && typeof change === "object" ? change.tokenizer : void 0;
|
|
1970
|
+
const loadingTokenizerCode = loadSchemaCodeFor(null, changedTokenizer);
|
|
1971
|
+
if (loadingTokenizerCode !== null) await loadingTokenizerCode;
|
|
1961
1972
|
const entry = core.requireIndex(indexName);
|
|
1962
1973
|
requireNotRebalancing(core, indexName);
|
|
1963
1974
|
requireNoRestoreInFlight(core, indexName);
|
|
@@ -1979,6 +1990,8 @@ async function changeIndexAnalysis(core, indexName, change) {
|
|
|
1979
1990
|
}
|
|
1980
1991
|
function resumeAnalysisChange(core, indexName) {
|
|
1981
1992
|
const entry = core.requireIndex(indexName);
|
|
1993
|
+
const loadingTokenizerCode = loadSchemaCodeFor(null, entry.pendingAnalysis?.tokenizer);
|
|
1994
|
+
if (loadingTokenizerCode !== null) return loadingTokenizerCode.then(() => resumeAnalysisChange(core, indexName));
|
|
1982
1995
|
const runs = runsOf(core);
|
|
1983
1996
|
const running = runs.get(indexName);
|
|
1984
1997
|
if (running !== void 0) return running.finished;
|
|
@@ -2028,6 +2041,8 @@ function startHeldRun(core, indexName, pending) {
|
|
|
2028
2041
|
return run;
|
|
2029
2042
|
}
|
|
2030
2043
|
async function buildHeldAnalysisChange(core, indexName, pending) {
|
|
2044
|
+
const loadingTokenizerCode = loadSchemaCodeFor(null, pending.tokenizer);
|
|
2045
|
+
if (loadingTokenizerCode !== null) await loadingTokenizerCode;
|
|
2031
2046
|
const running = runsOf(core).get(indexName);
|
|
2032
2047
|
const entry = core.indexRegistry.get(indexName);
|
|
2033
2048
|
if (running?.hold != null && entry !== void 0 && holdServes(running, entry, pending)) return running.hold.ready;
|
|
@@ -2828,7 +2843,7 @@ function wireToMetadata(raw) {
|
|
|
2828
2843
|
if (raw.partition_limits && typeof raw.partition_limits === "object") {
|
|
2829
2844
|
const limits = {
|
|
2830
2845
|
...isPositiveInteger(raw.partition_limits.max_docs_per_partition) ? { maxDocsPerPartition: raw.partition_limits.max_docs_per_partition } : {},
|
|
2831
|
-
...isPositiveInteger(raw.partition_limits.max_partitions) ? { maxPartitions: raw.partition_limits.max_partitions } : {},
|
|
2846
|
+
...isPositiveInteger(raw.partition_limits.max_partitions) ? { maxPartitions: Math.min(raw.partition_limits.max_partitions, MAX_PARTITIONS) } : {},
|
|
2832
2847
|
...isWatermarkFraction(raw.partition_limits.watermark) ? { watermark: raw.partition_limits.watermark } : {}
|
|
2833
2848
|
};
|
|
2834
2849
|
if (Object.keys(limits).length > 0) {
|
|
@@ -4721,7 +4736,7 @@ function openPagedFile(path, cache) {
|
|
|
4721
4736
|
const native = openNativePagedFile(path);
|
|
4722
4737
|
readersOfPath.set(path, (readersOfPath.get(path) ?? 0) + 1);
|
|
4723
4738
|
try {
|
|
4724
|
-
return { path, fileNumber: cache.fileNumber(), cache, native, closed: false };
|
|
4739
|
+
return { path, fileNumber: cache.fileNumber(), cache, native, closed: false, suspended: false };
|
|
4725
4740
|
} catch (error) {
|
|
4726
4741
|
native?.core.closePagedFile(native.handle);
|
|
4727
4742
|
stopReadingPath(path);
|
|
@@ -4737,27 +4752,31 @@ function stopReadingPath(path) {
|
|
|
4737
4752
|
readersOfPath.delete(path);
|
|
4738
4753
|
fileReaderOfThisThread().retainOnly(new Set(readersOfPath.keys()));
|
|
4739
4754
|
}
|
|
4740
|
-
function
|
|
4741
|
-
if (file.closed) return;
|
|
4742
|
-
file.closed = true;
|
|
4755
|
+
function closeNativeHandle(file) {
|
|
4743
4756
|
if (file.native !== null) file.cache.releasePageFile(file.native.handle.identity);
|
|
4744
4757
|
file.native?.core.closePagedFile(file.native.handle);
|
|
4745
4758
|
file.native = null;
|
|
4759
|
+
}
|
|
4760
|
+
function closePagedFile(file) {
|
|
4761
|
+
if (file.closed) return;
|
|
4762
|
+
file.closed = true;
|
|
4763
|
+
closeNativeHandle(file);
|
|
4746
4764
|
file.cache.releaseFile(file.fileNumber);
|
|
4765
|
+
if (file.suspended) return;
|
|
4747
4766
|
stopReadingPath(file.path);
|
|
4748
4767
|
}
|
|
4749
|
-
function
|
|
4750
|
-
if (
|
|
4751
|
-
|
|
4752
|
-
|
|
4753
|
-
file.native?.core.closePagedFile(file.native.handle);
|
|
4754
|
-
file.native = native;
|
|
4768
|
+
function suspendPagedFile(file) {
|
|
4769
|
+
if (file.closed || file.suspended) return;
|
|
4770
|
+
file.suspended = true;
|
|
4771
|
+
closeNativeHandle(file);
|
|
4755
4772
|
stopReadingPath(file.path);
|
|
4756
|
-
|
|
4757
|
-
|
|
4758
|
-
|
|
4759
|
-
|
|
4773
|
+
}
|
|
4774
|
+
function resumePagedFile(file, path) {
|
|
4775
|
+
if (file.closed || !file.suspended) return;
|
|
4776
|
+
file.suspended = false;
|
|
4760
4777
|
file.path = path;
|
|
4778
|
+
readersOfPath.set(path, (readersOfPath.get(path) ?? 0) + 1);
|
|
4779
|
+
file.native = openNativePagedFile(path);
|
|
4761
4780
|
}
|
|
4762
4781
|
function unreadable(path, offset, target) {
|
|
4763
4782
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The file "${path}" cannot be read at byte ${offset}`, {
|
|
@@ -5382,8 +5401,11 @@ function assembleOpenedSegment(file, snapshot, cache) {
|
|
|
5382
5401
|
close() {
|
|
5383
5402
|
closePagedFile(file);
|
|
5384
5403
|
},
|
|
5385
|
-
|
|
5386
|
-
|
|
5404
|
+
suspend() {
|
|
5405
|
+
suspendPagedFile(file);
|
|
5406
|
+
},
|
|
5407
|
+
resume(rewrite) {
|
|
5408
|
+
resumePagedFile(file, rewrite(file.path));
|
|
5387
5409
|
snapshot.path = file.path;
|
|
5388
5410
|
}
|
|
5389
5411
|
};
|
|
@@ -7513,6 +7535,9 @@ function openParts(input) {
|
|
|
7513
7535
|
}
|
|
7514
7536
|
const { entryPoint, maxLayer, m } = layout;
|
|
7515
7537
|
if (typeof maxLayer !== "number" || typeof m !== "number") loadFailed(`The head of "${ref.key}" holds no layers`);
|
|
7538
|
+
if (!Number.isInteger(m) || m < MIN_M || m > MAX_M) {
|
|
7539
|
+
loadFailed(`The head of "${ref.key}" gives m ${m}, while m must be a whole number from ${MIN_M} to ${MAX_M}`);
|
|
7540
|
+
}
|
|
7516
7541
|
if (entryPoint !== null && typeof entryPoint !== "number") loadFailed(`The head of "${ref.key}" holds no entry`);
|
|
7517
7542
|
if (graph === null) graph = { entryPoint, maxLayer, m };
|
|
7518
7543
|
const levels = readWholeSectionChecked(path, layout.levels, false);
|
|
@@ -7524,6 +7549,13 @@ function openParts(input) {
|
|
|
7524
7549
|
neighbours: createSectionReader(file, layout.neighbours)
|
|
7525
7550
|
});
|
|
7526
7551
|
}
|
|
7552
|
+
if (graph !== null) {
|
|
7553
|
+
requireBoundedUpperLists(
|
|
7554
|
+
parts.map((part) => part.levels),
|
|
7555
|
+
graph.m,
|
|
7556
|
+
`The vector graph parts of field "${input.fieldPath}"`
|
|
7557
|
+
);
|
|
7558
|
+
}
|
|
7527
7559
|
} catch (error) {
|
|
7528
7560
|
closeHeldFiles(parts);
|
|
7529
7561
|
throw error;
|
|
@@ -7564,7 +7596,10 @@ function validateColdLayout(input, opened, parts, graph) {
|
|
|
7564
7596
|
// src/persistence/paged/cold-field/index.ts
|
|
7565
7597
|
var FIXSTR2 = 160;
|
|
7566
7598
|
var FIXSTR_MAX_BYTES = 31;
|
|
7599
|
+
var STR8_MAX_BYTES = 255;
|
|
7600
|
+
var STR16_MAX_BYTES = 65535;
|
|
7567
7601
|
var ASCII_MAX2 = 127;
|
|
7602
|
+
var ID_OFFSET_BYTES = 4;
|
|
7568
7603
|
function shortAsciiIdOf(encoded) {
|
|
7569
7604
|
const length = encoded[0] - FIXSTR2;
|
|
7570
7605
|
if (length < 0 || length > FIXSTR_MAX_BYTES || encoded.length !== length + 1) return null;
|
|
@@ -7669,6 +7704,35 @@ function createFieldData(input, opened, graph, parts) {
|
|
|
7669
7704
|
}
|
|
7670
7705
|
};
|
|
7671
7706
|
}
|
|
7707
|
+
function utf8BytesOfEncodedId(encodedBytes) {
|
|
7708
|
+
if (encodedBytes <= FIXSTR_MAX_BYTES + 1) return encodedBytes - 1;
|
|
7709
|
+
if (encodedBytes <= STR8_MAX_BYTES + 2) return encodedBytes - 2;
|
|
7710
|
+
if (encodedBytes <= STR16_MAX_BYTES + 3) return encodedBytes - 3;
|
|
7711
|
+
return encodedBytes - 5;
|
|
7712
|
+
}
|
|
7713
|
+
function claimsOfLoad(data) {
|
|
7714
|
+
const offsets = new Uint8Array(SEGMENT_PAGE_BYTES);
|
|
7715
|
+
const view = new DataView(offsets.buffer);
|
|
7716
|
+
const offsetsPerRead = offsets.byteLength / ID_OFFSET_BYTES - 1;
|
|
7717
|
+
let slots = 0;
|
|
7718
|
+
let docIdBytes = 0;
|
|
7719
|
+
for (const file of data.files) {
|
|
7720
|
+
for (let first = 0; first < file.count; first += offsetsPerRead) {
|
|
7721
|
+
const count = Math.min(offsetsPerRead, file.count - first);
|
|
7722
|
+
file.idOffsets.read(first * ID_OFFSET_BYTES, offsets.subarray(0, (count + 1) * ID_OFFSET_BYTES));
|
|
7723
|
+
for (let at = 0; at < count; at++) {
|
|
7724
|
+
const position = first + at;
|
|
7725
|
+
if (isDead(file.dead, position) && data.topLayerOf(file.firstNumber + position) < 0) continue;
|
|
7726
|
+
const start = view.getUint32(at * ID_OFFSET_BYTES, true);
|
|
7727
|
+
const end = view.getUint32((at + 1) * ID_OFFSET_BYTES, true);
|
|
7728
|
+
if (end <= start) loadFailed(`The ID offsets of "${file.file.path}" hold no ID at position ${position}`);
|
|
7729
|
+
slots += 1;
|
|
7730
|
+
docIdBytes += utf8BytesOfEncodedId(end - start);
|
|
7731
|
+
}
|
|
7732
|
+
}
|
|
7733
|
+
}
|
|
7734
|
+
return { slots, docIdBytes };
|
|
7735
|
+
}
|
|
7672
7736
|
function positionOfId(file, docId, idAt2, slot, view) {
|
|
7673
7737
|
let low = 0;
|
|
7674
7738
|
let high = file.count - 1;
|
|
@@ -7725,6 +7789,9 @@ function openColdVectorField(input) {
|
|
|
7725
7789
|
}
|
|
7726
7790
|
let residentBytes = 0;
|
|
7727
7791
|
for (const part of parts) residentBytes += part.levels.byteLength + part.nodeOffsets.byteLength;
|
|
7792
|
+
let encodedIdBytes = 0;
|
|
7793
|
+
for (const file of opened.files) encodedIdBytes += file.ids.length;
|
|
7794
|
+
let claimsOnceRead = null;
|
|
7728
7795
|
const orderSlot = new Uint8Array(4);
|
|
7729
7796
|
const orderView = new DataView(orderSlot.buffer);
|
|
7730
7797
|
function liveNumberOf(docId) {
|
|
@@ -7743,6 +7810,11 @@ function openColdVectorField(input) {
|
|
|
7743
7810
|
for (const part of parts) bytes += cachedFileBytes(part.file);
|
|
7744
7811
|
return bytes;
|
|
7745
7812
|
},
|
|
7813
|
+
claimsAtMost: { slots: data.total, docIdBytes: encodedIdBytes },
|
|
7814
|
+
claims() {
|
|
7815
|
+
claimsOnceRead ??= claimsOfLoad(data);
|
|
7816
|
+
return claimsOnceRead;
|
|
7817
|
+
},
|
|
7746
7818
|
numberOf: liveNumberOf,
|
|
7747
7819
|
vectorOfNumber: (number) => data.vectorAt(number),
|
|
7748
7820
|
reusedVectorOfNumber: (number) => data.reusedVectorAt(number),
|
|
@@ -10856,6 +10928,21 @@ async function writeIndexCheckpointSegments(input) {
|
|
|
10856
10928
|
});
|
|
10857
10929
|
}
|
|
10858
10930
|
|
|
10931
|
+
// src/partitioning/saved-partitions.ts
|
|
10932
|
+
function requireSavedPartitionCount(partitionCount, source, indexName) {
|
|
10933
|
+
if (partitionCount <= MAX_PARTITIONS) return;
|
|
10934
|
+
throw new NarsilError(
|
|
10935
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
10936
|
+
`The ${source} of index "${indexName}" holds partition ${partitionCount - 1}, and a saved index numbers its partitions from 0 to ${MAX_PARTITIONS - 1}`,
|
|
10937
|
+
{ indexName, partitionId: partitionCount - 1, maxPartitions: MAX_PARTITIONS }
|
|
10938
|
+
);
|
|
10939
|
+
}
|
|
10940
|
+
function highestPartitionIdOf(partitions) {
|
|
10941
|
+
let highest = partitions.length - 1;
|
|
10942
|
+
for (const partition of partitions) if (partition.partitionId > highest) highest = partition.partitionId;
|
|
10943
|
+
return highest;
|
|
10944
|
+
}
|
|
10945
|
+
|
|
10859
10946
|
// src/persistence/paged/read-ahead.ts
|
|
10860
10947
|
var FILE_PAGE_SIZES2 = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
|
|
10861
10948
|
function rangeOf(path, section) {
|
|
@@ -11294,21 +11381,11 @@ async function readCurrentManifest(store, indexName) {
|
|
|
11294
11381
|
);
|
|
11295
11382
|
}
|
|
11296
11383
|
function growPartitionsTo(manager, highestPartitionId) {
|
|
11297
|
-
|
|
11298
|
-
const before = manager.partitionCount;
|
|
11299
|
-
try {
|
|
11300
|
-
manager.addPartition();
|
|
11301
|
-
} catch {
|
|
11302
|
-
break;
|
|
11303
|
-
}
|
|
11304
|
-
if (manager.partitionCount === before) break;
|
|
11305
|
-
}
|
|
11384
|
+
manager.growToSavedPartitionCount(highestPartitionId + 1);
|
|
11306
11385
|
}
|
|
11307
11386
|
async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodesEveryDocument = false) {
|
|
11308
|
-
|
|
11309
|
-
|
|
11310
|
-
if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
|
|
11311
|
-
}
|
|
11387
|
+
const highestPartitionId = highestPartitionIdOf(manifest.partitions);
|
|
11388
|
+
requireSavedPartitionCount(highestPartitionId + 1, "segment manifest", indexName);
|
|
11312
11389
|
growPartitionsTo(deps.manager, highestPartitionId);
|
|
11313
11390
|
const { directory } = store;
|
|
11314
11391
|
const opensVectorsOnDemand = manifest.vectors.some(
|
|
@@ -11318,13 +11395,6 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
|
|
|
11318
11395
|
await startReadingOpeningSectionsAhead(directory, manifest);
|
|
11319
11396
|
}
|
|
11320
11397
|
for (const partition of manifest.partitions) {
|
|
11321
|
-
if (partition.partitionId >= deps.manager.partitionCount) {
|
|
11322
|
-
throw new NarsilError(
|
|
11323
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
11324
|
-
`Segment manifest references partition ${partition.partitionId} beyond the partition count ${deps.manager.partitionCount}`,
|
|
11325
|
-
{ indexName, partitionId: partition.partitionId, partitionCount: deps.manager.partitionCount }
|
|
11326
|
-
);
|
|
11327
|
-
}
|
|
11328
11398
|
await loadPartitionSegments(store, partition, deps.manager, {
|
|
11329
11399
|
schema: manifest.schema,
|
|
11330
11400
|
decodesEveryDocument,
|
|
@@ -13408,7 +13478,8 @@ function countSnapshotBundleDocuments(bundle) {
|
|
|
13408
13478
|
}
|
|
13409
13479
|
async function loadSnapshotBundle(bytes, deps) {
|
|
13410
13480
|
const bundle = await decodeSnapshotBundle(bytes);
|
|
13411
|
-
|
|
13481
|
+
requireSavedPartitionCount(bundle.partitions.length, "snapshot bundle", deps.manager.indexName);
|
|
13482
|
+
deps.manager.growToSavedPartitionCount(bundle.partitions.length);
|
|
13412
13483
|
if (bundle.partitions.length > 0) deps.manager.trimPartitions(bundle.partitions.length);
|
|
13413
13484
|
for (let i = 0; i < bundle.partitions.length; i += 1) {
|
|
13414
13485
|
deps.manager.deserializePartition(i, deserializePayloadV2(bundle.partitions[i]), deserializePartition);
|
|
@@ -13877,9 +13948,7 @@ async function readGraph(store, ref) {
|
|
|
13877
13948
|
what: `graph file "${ref.graphKey}"`,
|
|
13878
13949
|
allowsCompression: true
|
|
13879
13950
|
});
|
|
13880
|
-
|
|
13881
|
-
if (graph.graphs.some((numbered) => numbered.m < MIN_M)) return null;
|
|
13882
|
-
return graphPartsOfEarlierPayload(graph);
|
|
13951
|
+
return graphPartsOfEarlierPayload(decodeVectorGraphPayload(decode(payload)));
|
|
13883
13952
|
}
|
|
13884
13953
|
async function loadVectorField2(store, ref, vectorIndex, fileRefs) {
|
|
13885
13954
|
let liveVectors = 0;
|
|
@@ -13924,21 +13993,10 @@ async function loadVectorField2(store, ref, vectorIndex, fileRefs) {
|
|
|
13924
13993
|
restore.finish();
|
|
13925
13994
|
}
|
|
13926
13995
|
async function loadEarlierLayout(store, indexName, manifest, deps) {
|
|
13927
|
-
|
|
13928
|
-
|
|
13929
|
-
if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
|
|
13930
|
-
}
|
|
13996
|
+
const highestPartitionId = highestPartitionIdOf(manifest.partitions);
|
|
13997
|
+
requireSavedPartitionCount(highestPartitionId + 1, "earlier segment manifest", indexName);
|
|
13931
13998
|
growPartitionsTo(deps.manager, highestPartitionId);
|
|
13932
|
-
for (const partition of manifest.partitions)
|
|
13933
|
-
if (partition.partitionId >= deps.manager.partitionCount) {
|
|
13934
|
-
throw new NarsilError(
|
|
13935
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
13936
|
-
`Segment manifest references partition ${partition.partitionId} beyond the partition count ${deps.manager.partitionCount}`,
|
|
13937
|
-
{ indexName, partitionId: partition.partitionId }
|
|
13938
|
-
);
|
|
13939
|
-
}
|
|
13940
|
-
await loadPartition(store, indexName, partition, deps);
|
|
13941
|
-
}
|
|
13999
|
+
for (const partition of manifest.partitions) await loadPartition(store, indexName, partition, deps);
|
|
13942
14000
|
const vectorFileRefs = /* @__PURE__ */ new Map();
|
|
13943
14001
|
for (const vector of manifest.vectors) {
|
|
13944
14002
|
const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
|
|
@@ -16303,7 +16361,7 @@ function restoredPartitionConfig(raw, reject = rejectSnapshot) {
|
|
|
16303
16361
|
}
|
|
16304
16362
|
return {
|
|
16305
16363
|
...typeof maxDocsPerPartition === "number" ? { maxDocsPerPartition } : {},
|
|
16306
|
-
...typeof maxPartitions === "number" ? { maxPartitions } : {},
|
|
16364
|
+
...typeof maxPartitions === "number" ? { maxPartitions: Math.min(maxPartitions, MAX_PARTITIONS) } : {},
|
|
16307
16365
|
...typeof watermark === "number" ? { watermark } : {}
|
|
16308
16366
|
};
|
|
16309
16367
|
}
|
|
@@ -16411,6 +16469,59 @@ function restoredConfigFields(envelope, reject = rejectSnapshot) {
|
|
|
16411
16469
|
};
|
|
16412
16470
|
}
|
|
16413
16471
|
|
|
16472
|
+
// src/engine/snapshot/stored-schema.ts
|
|
16473
|
+
function isRecord4(value) {
|
|
16474
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
16475
|
+
}
|
|
16476
|
+
function conflictingStoredPath(path) {
|
|
16477
|
+
return new NarsilError(
|
|
16478
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
16479
|
+
`Field path "${path}" is declared both as a field and as an object of fields`,
|
|
16480
|
+
{ path }
|
|
16481
|
+
);
|
|
16482
|
+
}
|
|
16483
|
+
function defineOwn(level, key, value) {
|
|
16484
|
+
Object.defineProperty(level, key, { value, enumerable: true, writable: true, configurable: true });
|
|
16485
|
+
}
|
|
16486
|
+
function placeStoredField(nested, path, type) {
|
|
16487
|
+
const segments = path.split(".");
|
|
16488
|
+
let level = nested;
|
|
16489
|
+
for (let at = 0; at < segments.length; at++) {
|
|
16490
|
+
const segment = segments[at];
|
|
16491
|
+
const held = Object.hasOwn(level, segment) ? level[segment] : void 0;
|
|
16492
|
+
if (at === segments.length - 1) {
|
|
16493
|
+
if (held !== void 0) throw conflictingStoredPath(path);
|
|
16494
|
+
defineOwn(level, segment, type);
|
|
16495
|
+
return;
|
|
16496
|
+
}
|
|
16497
|
+
if (held === void 0) {
|
|
16498
|
+
const child = {};
|
|
16499
|
+
defineOwn(level, segment, child);
|
|
16500
|
+
level = child;
|
|
16501
|
+
} else if (isRecord4(held)) {
|
|
16502
|
+
level = held;
|
|
16503
|
+
} else {
|
|
16504
|
+
throw conflictingStoredPath(segments.slice(0, at + 1).join("."));
|
|
16505
|
+
}
|
|
16506
|
+
}
|
|
16507
|
+
}
|
|
16508
|
+
function placeStoredFields(nested, stored, prefix) {
|
|
16509
|
+
for (const [key, type] of Object.entries(stored)) {
|
|
16510
|
+
const path = prefix ? `${prefix}.${key}` : key;
|
|
16511
|
+
if (isRecord4(type)) {
|
|
16512
|
+
placeStoredFields(nested, type, path);
|
|
16513
|
+
} else {
|
|
16514
|
+
placeStoredField(nested, path, type);
|
|
16515
|
+
}
|
|
16516
|
+
}
|
|
16517
|
+
}
|
|
16518
|
+
function nestedSchemaOf(stored) {
|
|
16519
|
+
if (!isRecord4(stored)) return stored;
|
|
16520
|
+
const nested = {};
|
|
16521
|
+
placeStoredFields(nested, stored, "");
|
|
16522
|
+
return nested;
|
|
16523
|
+
}
|
|
16524
|
+
|
|
16414
16525
|
// src/persistence/archive/byte-reader.ts
|
|
16415
16526
|
async function* chunksOf(source) {
|
|
16416
16527
|
if (source instanceof Uint8Array) {
|
|
@@ -16632,7 +16743,7 @@ function refuse2(path, reason) {
|
|
|
16632
16743
|
field: path
|
|
16633
16744
|
});
|
|
16634
16745
|
}
|
|
16635
|
-
function
|
|
16746
|
+
function isRecord5(value) {
|
|
16636
16747
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
16637
16748
|
}
|
|
16638
16749
|
function startsWithTagPrefix(key) {
|
|
@@ -16699,7 +16810,7 @@ function untagTag(tag, value, path, depth) {
|
|
|
16699
16810
|
case NUMBER_TAG:
|
|
16700
16811
|
return numberOf(value, path);
|
|
16701
16812
|
case OBJECT_TAG:
|
|
16702
|
-
if (!
|
|
16813
|
+
if (!isRecord5(value)) refuse2(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
|
|
16703
16814
|
return untagMap(value, path, depth);
|
|
16704
16815
|
default:
|
|
16705
16816
|
return refuse2(path, `holds an object whose only key, "${tag}", names no tag`);
|
|
@@ -16723,9 +16834,6 @@ var DOCUMENT_EXPORT_VERSION = 1;
|
|
|
16723
16834
|
function invalidHeader(message) {
|
|
16724
16835
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
|
|
16725
16836
|
}
|
|
16726
|
-
function isRecord5(value) {
|
|
16727
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
16728
|
-
}
|
|
16729
16837
|
function configOfHeader(line) {
|
|
16730
16838
|
if (line === void 0) invalidHeader("is missing");
|
|
16731
16839
|
if (!("text" in line)) {
|
|
@@ -16739,14 +16847,14 @@ function configOfHeader(line) {
|
|
|
16739
16847
|
} catch {
|
|
16740
16848
|
invalidHeader("is not JSON");
|
|
16741
16849
|
}
|
|
16742
|
-
if (!
|
|
16850
|
+
if (!isRecord4(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
|
|
16743
16851
|
invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
|
|
16744
16852
|
}
|
|
16745
16853
|
const configuration = requireConfigurationShape(raw.index, "document export header");
|
|
16746
16854
|
const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
|
|
16747
16855
|
const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
|
|
16748
16856
|
return {
|
|
16749
|
-
schema: configuration.schema,
|
|
16857
|
+
schema: nestedSchemaOf(configuration.schema),
|
|
16750
16858
|
language: configuration.language,
|
|
16751
16859
|
...fields,
|
|
16752
16860
|
...embedding === void 0 ? {} : {
|
|
@@ -16832,7 +16940,7 @@ async function takeLine(state, entry) {
|
|
|
16832
16940
|
refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
|
|
16833
16941
|
return;
|
|
16834
16942
|
}
|
|
16835
|
-
if (!
|
|
16943
|
+
if (!isRecord4(document) || typeof document.id !== "string" || document.id.length === 0) {
|
|
16836
16944
|
refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
|
|
16837
16945
|
return;
|
|
16838
16946
|
}
|
|
@@ -17505,9 +17613,9 @@ function attachablePolicy(shared) {
|
|
|
17505
17613
|
}
|
|
17506
17614
|
};
|
|
17507
17615
|
}
|
|
17508
|
-
function createDetachedIndex(indexName, config, language, synonyms, executor) {
|
|
17616
|
+
function createDetachedIndex(indexName, config, language, synonyms, executor, partitionCount) {
|
|
17509
17617
|
const { policy, attach } = attachablePolicy(executor.vectorWorkerCopies);
|
|
17510
|
-
const entry = executor.createDetachedEntry(indexName, config, language, policy);
|
|
17618
|
+
const entry = executor.createDetachedEntry(indexName, config, language, policy, partitionCount);
|
|
17511
17619
|
entry.manager.synonyms = synonyms;
|
|
17512
17620
|
return { indexName, manager: entry.manager, entry, attach };
|
|
17513
17621
|
}
|
|
@@ -17518,7 +17626,7 @@ function prepareRestoredIndex(configuration, deps, reject) {
|
|
|
17518
17626
|
const embedding = restoredEmbedding(configuration.embedding, reject);
|
|
17519
17627
|
const languageName = configuration.language ?? "";
|
|
17520
17628
|
const language = getLanguage(languageName);
|
|
17521
|
-
const schema = validateSchema(configuration.schema);
|
|
17629
|
+
const schema = validateSchema(nestedSchemaOf(configuration.schema));
|
|
17522
17630
|
validatePatternValueLimit(configFields.patternValueLimit);
|
|
17523
17631
|
requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
|
|
17524
17632
|
validateVectorPromotion(configFields.vectorPromotion);
|
|
@@ -17568,17 +17676,17 @@ function restoredSynonyms(raw, config, language, reject) {
|
|
|
17568
17676
|
return prepareStoredSynonyms(raw, config, language, (message) => reject(`synonyms: ${message}`));
|
|
17569
17677
|
}
|
|
17570
17678
|
async function prepareDetachedIndex(indexName, prepared, partitionCount, deps) {
|
|
17571
|
-
|
|
17679
|
+
if (partitionCount !== null) requireSavedPartitionCount(partitionCount, "snapshot", indexName);
|
|
17680
|
+
const loadingSchemaCode = loadSchemaCodeFor(prepared.config.schema, prepared.config.tokenizer);
|
|
17572
17681
|
if (loadingSchemaCode !== null) await loadingSchemaCode;
|
|
17573
|
-
|
|
17682
|
+
return createDetachedIndex(
|
|
17574
17683
|
indexName,
|
|
17575
17684
|
prepared.config,
|
|
17576
17685
|
prepared.language,
|
|
17577
17686
|
prepared.synonyms?.lookup ?? null,
|
|
17578
|
-
deps.executor
|
|
17687
|
+
deps.executor,
|
|
17688
|
+
partitionCount ?? void 0
|
|
17579
17689
|
);
|
|
17580
|
-
while (detached.manager.partitionCount < partitionCount) detached.manager.addPartition();
|
|
17581
|
-
return detached;
|
|
17582
17690
|
}
|
|
17583
17691
|
function restoredRegistryEntry(prepared, manager, deps) {
|
|
17584
17692
|
return {
|
|
@@ -17656,30 +17764,42 @@ function persistedCheckpoint(indexName, partitionCount, deps) {
|
|
|
17656
17764
|
primaryTerm: SINGLE_NODE_PRIMARY_TERM
|
|
17657
17765
|
}));
|
|
17658
17766
|
}
|
|
17659
|
-
|
|
17767
|
+
function pagedSegmentsOf(manager) {
|
|
17768
|
+
const paged = [];
|
|
17769
|
+
if (manager === void 0) return paged;
|
|
17770
|
+
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
|
|
17771
|
+
const partition = manager.getPartition(partitionId);
|
|
17772
|
+
if (!isCompositePartition(partition)) continue;
|
|
17773
|
+
for (const segment of partition.frozenSegments()) if (segment.paged !== null) paged.push(segment.paged);
|
|
17774
|
+
}
|
|
17775
|
+
return paged;
|
|
17776
|
+
}
|
|
17777
|
+
function samePath(path) {
|
|
17778
|
+
return path;
|
|
17779
|
+
}
|
|
17780
|
+
async function commitToStorage(indexName, restored, previous, deps) {
|
|
17660
17781
|
const durable = deps.durability?.manager;
|
|
17661
17782
|
const staging = restored.staging;
|
|
17662
17783
|
if (durable === void 0 || staging === null) return;
|
|
17663
|
-
const
|
|
17784
|
+
const previousPartitions = previous?.partitionCount ?? 0;
|
|
17785
|
+
const previousCheckpoint = persistedCheckpoint(indexName, previousPartitions, deps);
|
|
17664
17786
|
if (previousPartitions > 0) await durable.unloadIndex(indexName);
|
|
17665
17787
|
const vectorIndexes = [...restored.detached.manager.getVectorIndexes().values()];
|
|
17666
17788
|
try {
|
|
17667
17789
|
for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
|
|
17668
17790
|
await staging.commit();
|
|
17669
17791
|
} catch (error) {
|
|
17670
|
-
if (previousPartitions > 0) durable.adoptRestoredCheckpoint?.(indexName,
|
|
17792
|
+
if (previousPartitions > 0) durable.adoptRestoredCheckpoint?.(indexName, previousCheckpoint);
|
|
17671
17793
|
throw error;
|
|
17672
17794
|
}
|
|
17795
|
+
for (const paged of pagedSegmentsOf(previous)) paged.close();
|
|
17796
|
+
const restoredSegments = pagedSegmentsOf(restored.detached.manager);
|
|
17797
|
+
for (const paged of restoredSegments) paged.suspend();
|
|
17673
17798
|
await staging.finish();
|
|
17674
17799
|
const relocate = await staging.relocation().catch(() => null);
|
|
17800
|
+
for (const paged of restoredSegments) paged.resume(relocate ?? samePath);
|
|
17675
17801
|
if (relocate === null) return;
|
|
17676
17802
|
for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
|
|
17677
|
-
const { manager } = restored.detached;
|
|
17678
|
-
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
|
|
17679
|
-
const partition = manager.getPartition(partitionId);
|
|
17680
|
-
if (!isCompositePartition(partition)) continue;
|
|
17681
|
-
for (const segment of partition.frozenSegments()) segment.paged?.relocate(relocate);
|
|
17682
|
-
}
|
|
17683
17803
|
}
|
|
17684
17804
|
function activate(indexName, restored, entry, deps) {
|
|
17685
17805
|
deps.executor.installDetachedIndex(restored.detached);
|
|
@@ -17709,7 +17829,7 @@ async function replaceExisting(indexName, restored, entry, deps) {
|
|
|
17709
17829
|
const vectorIndexes = previous === void 0 ? [] : vectorIndexesOf2(indexName, deps);
|
|
17710
17830
|
await withinLimit(indexName, release, holdVectorCopies(vectorIndexes, release), deps);
|
|
17711
17831
|
for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
|
|
17712
|
-
await commitToStorage(indexName, restored, previous
|
|
17832
|
+
await commitToStorage(indexName, restored, previous, deps);
|
|
17713
17833
|
if (previous === void 0) activate(indexName, restored, entry, deps);
|
|
17714
17834
|
else await previous.publication.write(() => activate(indexName, restored, entry, deps));
|
|
17715
17835
|
await deps.querySuggestions?.afterActivate(entry);
|
|
@@ -17725,7 +17845,7 @@ async function createFresh(indexName, restored, entry, deps) {
|
|
|
17725
17845
|
deps.indexRegistry.set(indexName, entry);
|
|
17726
17846
|
try {
|
|
17727
17847
|
await deps.indexState.registerReplacing(indexName, async () => {
|
|
17728
|
-
await commitToStorage(indexName, restored, 0, deps);
|
|
17848
|
+
await commitToStorage(indexName, restored, void 0, deps);
|
|
17729
17849
|
activate(indexName, restored, entry, deps);
|
|
17730
17850
|
await deps.querySuggestions?.afterActivate(entry);
|
|
17731
17851
|
});
|
|
@@ -17996,7 +18116,8 @@ async function stageBuiltIndex(indexName, prepared, built, staging, deps) {
|
|
|
17996
18116
|
return archive.manifest;
|
|
17997
18117
|
}
|
|
17998
18118
|
async function buildRestoredIndex(indexName, envelope, prepared, staging, deps) {
|
|
17999
|
-
const
|
|
18119
|
+
const savedPartitions = envelope.partitions?.length ?? 0;
|
|
18120
|
+
const built = await prepareDetachedIndex(indexName, prepared, savedPartitions > 0 ? savedPartitions : null, deps);
|
|
18000
18121
|
let manifest;
|
|
18001
18122
|
try {
|
|
18002
18123
|
insertSnapshotDocuments(envelope, built.manager);
|
|
@@ -18227,4 +18348,4 @@ function createRewriteSpill(storage, indexName, analysis) {
|
|
|
18227
18348
|
};
|
|
18228
18349
|
}
|
|
18229
18350
|
|
|
18230
|
-
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, abandonAnalysisChange, actionPartitionsOf, adoptSavedAnalysis, adoptSegmentsOnWorker, bindStorageHost, buildHeldAnalysisChange, bytesThatSegmentsHoldInMemory, changeIndexAnalysis, checkPagedContents, checkPagedSegmentFully, checkPagedSelections, checkpointForArchive, closeUnusedSegments, countsAsLatestHolder, createByteReader, createDurabilityFromTier, createHNSWIndex, createInvalidationFromConfig, createPagedFrozenSegment, createSnapshot, decodeIndexConfiguration, documentExportStream, dropHeldAnalysisChange, encodeIndexConfiguration, executeRebalance, heldAnalysisIsBuilt, holdRenewedSyncEpoch, holdsNamedSettings, indexConfigurationOf, isPagedSegmentFile, keysThatSegmentsStillRead, loadFilesByParts, mergePagedWindow, openColdVectorField, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, osqBitsOf, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, readDocumentExport, readSnapshotArchive, releaseHeldAnalysisChange, reloadIndexAndCopies, requireNoQuerySuggestions, requireSyncSetting, restoreFromSnapshot, resumeAnalysisChange, scheduleSyncRetention, stampAnalysisGeneration, startReadingOpeningSectionsAhead, startSyncRetentionSweep, swapPreparedPartitions, syncOf, syncStorageOf, targetOfPending, transitionWalLayout, updateIndexSync, updateSyncOf, waitBeforeColdLoad, writePagedSegment };
|
|
18351
|
+
export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, abandonAnalysisChange, actionPartitionsOf, adoptSavedAnalysis, adoptSegmentsOnWorker, bindStorageHost, buildHeldAnalysisChange, bytesThatSegmentsHoldInMemory, changeIndexAnalysis, checkPagedContents, checkPagedSegmentFully, checkPagedSelections, checkpointForArchive, closeUnusedSegments, countsAsLatestHolder, createByteReader, createDurabilityFromTier, createHNSWIndex, createInvalidationFromConfig, createPagedFrozenSegment, createSnapshot, decodeIndexConfiguration, documentExportStream, dropHeldAnalysisChange, encodeIndexConfiguration, executeRebalance, heldAnalysisIsBuilt, holdRenewedSyncEpoch, holdsNamedSettings, indexConfigurationOf, isPagedSegmentFile, keysThatSegmentsStillRead, loadFilesByParts, mergePagedWindow, openColdVectorField, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, osqBitsOf, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, readDocumentExport, readSnapshotArchive, releaseHeldAnalysisChange, reloadIndexAndCopies, requireNoQuerySuggestions, requireSavedPartitionCount, requireSyncSetting, restoreFromSnapshot, resumeAnalysisChange, scheduleSyncRetention, stampAnalysisGeneration, startReadingOpeningSectionsAhead, startSyncRetentionSweep, swapPreparedPartitions, syncOf, syncStorageOf, targetOfPending, transitionWalLayout, updateIndexSync, updateSyncOf, waitBeforeColdLoad, writePagedSegment };
|
|
@@ -1055,6 +1055,7 @@ async function loadIndexOntoWorkers(indexName, executors, source, manager, onDro
|
|
|
1055
1055
|
analysisStale,
|
|
1056
1056
|
copyLoad,
|
|
1057
1057
|
publication: held.publication,
|
|
1058
|
+
partitionCount: captured.length,
|
|
1058
1059
|
...synonyms !== null ? { synonyms } : {}
|
|
1059
1060
|
},
|
|
1060
1061
|
held.retain()
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-pljAqmrC.js';
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
2
|
+
import { C as ClusterNamespace } from './config-CIpLXXxm.js';
|
|
3
3
|
import { l as RecordedQuery } from './query-suggestions-CBYkrQ5T.js';
|
|
4
|
-
import { E as ErrorDetail, j as TaskStore } from './tasks-
|
|
4
|
+
import { E as ErrorDetail, j as TaskStore } from './tasks-CXtQ3Zpp.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Context handed to {@link OnRequestHook} for every inbound request, captured
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { encode, Merger } from './chunk-
|
|
2
|
-
import { DayStore, RestoredHistory, dayOf, sameBytes } from './chunk-
|
|
1
|
+
import { encode, Merger } from './chunk-2KWTQ2P7.mjs';
|
|
2
|
+
import { DayStore, RestoredHistory, dayOf, sameBytes } from './chunk-YXZDTKQS.mjs';
|
|
3
3
|
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
4
|
-
import { isGroup, invalidSetting, historyFilesOf, buildRecordKey, MIN_KEY_BYTES, resolveSetting, isPerson, acceptedPhrase, removeHistoryFolder, normaliseBan, banListFromInput, validateSetting, withState, packEnvelopeWithChecksum, crc32, openChecksummedEnvelope, SCRAMBLED_ID_BYTES, KEY_FINGERPRINT_LABEL, KEY_FINGERPRINT_BYTES, removeRestoreLeftovers, HISTORY_SWEEP_INTERVAL_MS, SCHEDULE_OFF, parseSchedule, nextMatchAfter, latestMatchAtOrBefore, DAY_MS, listHistoryFolder, historyFileOf, MAX_MANUAL_SUGGESTION_COUNT, MAX_SOURCE_INDEX_NAME_LENGTH, DEFAULT_PREVIEW_LIMIT, MAX_PREVIEW_LIMIT, requireFieldsInSchema, MAX_TIMER_DELAY_MS, BUILD_WRITE_BATCH, BYTES_PER_MIB, MS_PER_SECOND, BUILD_LIST_PAGE, unionOfBans } from './chunk-
|
|
5
|
-
import { GROUP_FORMAT, emitEngineEvent, emitWriteEventAfter, createEngineIndex, withOpenIndex, loadDigest, SUGGESTIONS_INDEX_INFIX } from './chunk-
|
|
4
|
+
import { isGroup, invalidSetting, historyFilesOf, buildRecordKey, MIN_KEY_BYTES, resolveSetting, isPerson, acceptedPhrase, removeHistoryFolder, normaliseBan, banListFromInput, validateSetting, withState, packEnvelopeWithChecksum, crc32, openChecksummedEnvelope, SCRAMBLED_ID_BYTES, KEY_FINGERPRINT_LABEL, KEY_FINGERPRINT_BYTES, removeRestoreLeftovers, HISTORY_SWEEP_INTERVAL_MS, SCHEDULE_OFF, parseSchedule, nextMatchAfter, latestMatchAtOrBefore, DAY_MS, listHistoryFolder, historyFileOf, MAX_MANUAL_SUGGESTION_COUNT, MAX_SOURCE_INDEX_NAME_LENGTH, DEFAULT_PREVIEW_LIMIT, MAX_PREVIEW_LIMIT, requireFieldsInSchema, MAX_TIMER_DELAY_MS, BUILD_WRITE_BATCH, BYTES_PER_MIB, MS_PER_SECOND, BUILD_LIST_PAGE, unionOfBans } from './chunk-JCWC75NH.mjs';
|
|
5
|
+
import { GROUP_FORMAT, emitEngineEvent, emitWriteEventAfter, createEngineIndex, withOpenIndex, loadDigest, SUGGESTIONS_INDEX_INFIX } from './chunk-2C3BXZGA.mjs';
|
|
6
6
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
7
|
-
import { spawnNodeWorker, resolveWorkerEntry } from './chunk-
|
|
7
|
+
import { spawnNodeWorker, resolveWorkerEntry } from './chunk-DG6WIEB3.mjs';
|
|
8
8
|
import { NarsilError, ErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
|
|
9
9
|
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
10
10
|
|