@delali/narsil 0.3.1-next.140 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
- package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
- package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
- package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
- package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
- package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
- package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
- package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
- package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
- package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
- package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
- package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
- package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
- package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
- package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
- package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
- package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
- package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
- package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
- package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
- package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
- package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
- package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
- package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
- package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
- package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
- package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
- package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
- package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
- package/dist/client.d.ts +7 -7
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
- package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +123 -42
- package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
- package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
- package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +11 -11
- package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
- package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +9 -9
- package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +8 -7
- package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
- package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
- package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +3 -2
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
- package/openapi.json +13 -0
- package/package.json +12 -8
- package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
- package/dist/chunk-BDIDUW2B.mjs +0 -387
- package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
|
@@ -1,9 +1,12 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
1
|
+
import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-NRUXBYPH.mjs';
|
|
2
|
+
import { SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-W2COJ5AH.mjs';
|
|
3
|
+
import { VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-ICW45TTU.mjs';
|
|
4
|
+
import { readSegmentFile, STABLE_HEAD_ATTEMPTS } from './chunk-7AUASRE7.mjs';
|
|
5
|
+
import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, manifestKey, segmentsPrefix, vectorFileKey, graphPartKey } from './chunk-UUZCQQG2.mjs';
|
|
6
|
+
import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks } from './chunk-IO5F7ZBE.mjs';
|
|
7
|
+
import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-WJGO5PGV.mjs';
|
|
8
|
+
import { DEFAULT_BM25_B, DEFAULT_BM25_K1 } from './chunk-ZPWS2SKJ.mjs';
|
|
9
|
+
import { flattenSchema, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-KGOXHLLZ.mjs';
|
|
7
10
|
import { MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-U476IXKQ.mjs';
|
|
8
11
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
9
12
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
@@ -62,6 +65,9 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
|
|
|
62
65
|
if (config.trackPositions !== void 0) {
|
|
63
66
|
metadata.trackPositions = config.trackPositions;
|
|
64
67
|
}
|
|
68
|
+
if (config.textStorage !== void 0) {
|
|
69
|
+
metadata.textStorage = config.textStorage;
|
|
70
|
+
}
|
|
65
71
|
if (config.strict !== void 0) {
|
|
66
72
|
metadata.strict = config.strict;
|
|
67
73
|
}
|
|
@@ -112,6 +118,7 @@ function metadataConfigOf(entry) {
|
|
|
112
118
|
...entry.config.partitions !== void 0 ? { partitionLimits: entry.config.partitions } : {},
|
|
113
119
|
...entry.config.defaultScoring !== void 0 ? { defaultScoring: entry.config.defaultScoring } : {},
|
|
114
120
|
...entry.config.trackPositions !== void 0 ? { trackPositions: entry.config.trackPositions } : {},
|
|
121
|
+
...entry.config.textStorage !== void 0 ? { textStorage: entry.config.textStorage } : {},
|
|
115
122
|
...entry.config.strict !== void 0 ? { strict: entry.config.strict } : {},
|
|
116
123
|
...entry.config.required !== void 0 ? { required: entry.config.required } : {},
|
|
117
124
|
...entry.config.vectorPromotion !== void 0 ? { vectorPromotion: entry.config.vectorPromotion } : {},
|
|
@@ -388,6 +395,9 @@ function metadataToWire(meta) {
|
|
|
388
395
|
if (meta.trackPositions !== void 0) {
|
|
389
396
|
wire.track_positions = meta.trackPositions;
|
|
390
397
|
}
|
|
398
|
+
if (meta.textStorage !== void 0) {
|
|
399
|
+
wire.text_storage = meta.textStorage;
|
|
400
|
+
}
|
|
391
401
|
if (meta.strict !== void 0) {
|
|
392
402
|
wire.strict = meta.strict;
|
|
393
403
|
}
|
|
@@ -499,6 +509,14 @@ function wireToMetadata(raw) {
|
|
|
499
509
|
if (typeof raw.track_positions === "boolean") {
|
|
500
510
|
meta.trackPositions = raw.track_positions;
|
|
501
511
|
}
|
|
512
|
+
if (raw.text_storage !== void 0) {
|
|
513
|
+
if (raw.text_storage !== "memory" && raw.text_storage !== "disk") {
|
|
514
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "text_storage must be 'memory' or 'disk'", {
|
|
515
|
+
textStorage: raw.text_storage
|
|
516
|
+
});
|
|
517
|
+
}
|
|
518
|
+
meta.textStorage = raw.text_storage;
|
|
519
|
+
}
|
|
502
520
|
if (typeof raw.strict === "boolean") {
|
|
503
521
|
meta.strict = raw.strict;
|
|
504
522
|
}
|
|
@@ -760,7 +778,20 @@ function readFileRef(raw, expectedKey, what) {
|
|
|
760
778
|
invalidManifest(`gives the ${what} "${expectedKey}" an invalid length`, { key: expectedKey, bytes });
|
|
761
779
|
}
|
|
762
780
|
if (!isUint32(raw.checksum)) invalidManifest(`gives the ${what} "${expectedKey}" an invalid checksum`);
|
|
763
|
-
return { key: expectedKey, bytes, checksum: raw.checksum };
|
|
781
|
+
if (raw.headOffset === void 0) return { key: expectedKey, bytes, checksum: raw.checksum };
|
|
782
|
+
const { headOffset, headLength, headChecksum } = raw;
|
|
783
|
+
if (!isSequenceNumber(headOffset) || !isUint32(headLength) || !isUint32(headChecksum)) {
|
|
784
|
+
invalidManifest(`gives the ${what} "${expectedKey}" an invalid head`, { key: expectedKey });
|
|
785
|
+
}
|
|
786
|
+
if (headOffset + headLength > bytes - HEADER_SIZE) {
|
|
787
|
+
invalidManifest(`ends the head of the ${what} "${expectedKey}" outside its payload`, {
|
|
788
|
+
key: expectedKey,
|
|
789
|
+
headOffset,
|
|
790
|
+
headLength,
|
|
791
|
+
bytes
|
|
792
|
+
});
|
|
793
|
+
}
|
|
794
|
+
return { key: expectedKey, bytes, checksum: raw.checksum, headOffset, headLength, headChecksum };
|
|
764
795
|
}
|
|
765
796
|
|
|
766
797
|
// src/persistence/durability/segment/manifest-vectors-decode.ts
|
|
@@ -951,100 +982,313 @@ function readNextSegmentId(value, segments, partitionId) {
|
|
|
951
982
|
}
|
|
952
983
|
return value;
|
|
953
984
|
}
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
async function readGraphParts(store, ref) {
|
|
972
|
-
if (ref.graphParts.length === 0) return null;
|
|
973
|
-
const parts = [];
|
|
974
|
-
for (const part of ref.graphParts) {
|
|
975
|
-
const { payload } = await openListedFile(store, part, "graph part", true);
|
|
976
|
-
const decoded = decodeVectorGraphPartPayload(decode(payload));
|
|
977
|
-
if (decoded.fieldName !== ref.fieldPath) {
|
|
978
|
-
throw new NarsilError(
|
|
979
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
980
|
-
`The graph part "${part.key}" names field "${decoded.fieldName}" where the manifest lists "${ref.fieldPath}"`,
|
|
981
|
-
{ key: part.key }
|
|
982
|
-
);
|
|
983
|
-
}
|
|
984
|
-
parts.push(decoded);
|
|
985
|
-
}
|
|
986
|
-
return assembleGraphParts(parts);
|
|
985
|
+
|
|
986
|
+
// src/serialization/crc32-combine.ts
|
|
987
|
+
var IEEE_POLYNOMIAL = 3988292384;
|
|
988
|
+
var CRC_BITS = 32;
|
|
989
|
+
var zeroByteOperators = [];
|
|
990
|
+
function timesVector(operator, vector) {
|
|
991
|
+
let sum = 0;
|
|
992
|
+
let rest = vector >>> 0;
|
|
993
|
+
for (let row = 0; rest !== 0; row++, rest >>>= 1) {
|
|
994
|
+
if (rest & 1) sum ^= operator[row];
|
|
995
|
+
}
|
|
996
|
+
return sum >>> 0;
|
|
997
|
+
}
|
|
998
|
+
function squared(operator) {
|
|
999
|
+
const result = new Uint32Array(CRC_BITS);
|
|
1000
|
+
for (let row = 0; row < CRC_BITS; row++) result[row] = timesVector(operator, operator[row]);
|
|
1001
|
+
return result;
|
|
987
1002
|
}
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
throw new NarsilError(
|
|
1000
|
-
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1001
|
-
`The vector file "${file.key}" names field "${payload.fieldName}" where the manifest lists "${fieldPath}"`,
|
|
1002
|
-
{ key: file.key }
|
|
1003
|
-
);
|
|
1003
|
+
function firstOperator() {
|
|
1004
|
+
let oneZeroBit = new Uint32Array(CRC_BITS);
|
|
1005
|
+
oneZeroBit[0] = IEEE_POLYNOMIAL;
|
|
1006
|
+
for (let row = 1; row < CRC_BITS; row++) oneZeroBit[row] = 1 << row - 1;
|
|
1007
|
+
for (let doubling = 0; doubling < 3; doubling++) oneZeroBit = squared(oneZeroBit);
|
|
1008
|
+
return oneZeroBit;
|
|
1009
|
+
}
|
|
1010
|
+
function operatorForZeroBytesPower(power) {
|
|
1011
|
+
if (zeroByteOperators.length === 0) zeroByteOperators.push(firstOperator());
|
|
1012
|
+
while (zeroByteOperators.length <= power) {
|
|
1013
|
+
zeroByteOperators.push(squared(zeroByteOperators[zeroByteOperators.length - 1]));
|
|
1004
1014
|
}
|
|
1005
|
-
return
|
|
1015
|
+
return zeroByteOperators[power];
|
|
1006
1016
|
}
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
for (const file of ref.files) {
|
|
1014
|
-
const { payload, payloadLength } = await readVectorFile(store, file, ref.fieldPath);
|
|
1015
|
-
restore.addFile({
|
|
1016
|
-
id: file.id,
|
|
1017
|
-
key: file.key,
|
|
1018
|
-
payload,
|
|
1019
|
-
dead: file.dead,
|
|
1020
|
-
location: store.pathOf === void 0 ? null : {
|
|
1021
|
-
path: await store.pathOf(file.key),
|
|
1022
|
-
vectorsOffset: HEADER_SIZE + payloadLength - payload.vectors.byteLength
|
|
1023
|
-
}
|
|
1024
|
-
});
|
|
1017
|
+
function crc32Combine(first, second, secondByteLength) {
|
|
1018
|
+
let crc = first >>> 0;
|
|
1019
|
+
let rest = secondByteLength;
|
|
1020
|
+
for (let power = 0; rest > 0; power++) {
|
|
1021
|
+
if (rest % 2 === 1) crc = timesVector(operatorForZeroBytesPower(power), crc);
|
|
1022
|
+
rest = Math.floor(rest / 2);
|
|
1025
1023
|
}
|
|
1026
|
-
|
|
1024
|
+
return (crc ^ second) >>> 0;
|
|
1027
1025
|
}
|
|
1028
|
-
var EMPTY_BIN = new Uint8Array(0);
|
|
1029
|
-
var EMPTY_BIN_BYTES = 2;
|
|
1030
1026
|
var BIN8 = 196;
|
|
1031
1027
|
var BIN16 = 197;
|
|
1032
1028
|
var BIN32 = 198;
|
|
1029
|
+
var ARRAY16 = 220;
|
|
1030
|
+
var ARRAY32 = 221;
|
|
1031
|
+
var FIXMAP = 128;
|
|
1032
|
+
var FIXARRAY = 144;
|
|
1033
|
+
var FIXED_LIST_MAX = 15;
|
|
1034
|
+
var FIXSTR = 160;
|
|
1035
|
+
var FIXSTR_MAX_LENGTH = 31;
|
|
1036
|
+
var ASCII_MAX = 127;
|
|
1037
|
+
var UTF8_MAX_BYTES_PER_UNIT = 3;
|
|
1038
|
+
var STRING_HEADER_MAX_BYTES = 5;
|
|
1033
1039
|
var BIN8_MAX_BYTES = 255;
|
|
1034
|
-
var
|
|
1040
|
+
var UINT16_MAX = 65535;
|
|
1035
1041
|
function binHeader(byteLength) {
|
|
1036
1042
|
if (byteLength <= BIN8_MAX_BYTES) return Uint8Array.of(BIN8, byteLength);
|
|
1037
|
-
if (byteLength <=
|
|
1043
|
+
if (byteLength <= UINT16_MAX) return Uint8Array.of(BIN16, byteLength >> 8, byteLength & 255);
|
|
1038
1044
|
const header = new Uint8Array(5);
|
|
1039
1045
|
header[0] = BIN32;
|
|
1040
1046
|
new DataView(header.buffer).setUint32(1, byteLength, false);
|
|
1041
1047
|
return header;
|
|
1042
1048
|
}
|
|
1049
|
+
function arrayHeader(count) {
|
|
1050
|
+
if (count <= FIXED_LIST_MAX) return Uint8Array.of(FIXARRAY | count);
|
|
1051
|
+
if (count <= UINT16_MAX) return Uint8Array.of(ARRAY16, count >> 8, count & 255);
|
|
1052
|
+
const header = new Uint8Array(5);
|
|
1053
|
+
header[0] = ARRAY32;
|
|
1054
|
+
new DataView(header.buffer).setUint32(1, count, false);
|
|
1055
|
+
return header;
|
|
1056
|
+
}
|
|
1057
|
+
function createPayloadBuilder() {
|
|
1058
|
+
const chunks = [];
|
|
1059
|
+
let offset = 0;
|
|
1060
|
+
function push(chunk) {
|
|
1061
|
+
chunks.push(chunk);
|
|
1062
|
+
offset += chunk.byteLength;
|
|
1063
|
+
}
|
|
1064
|
+
return {
|
|
1065
|
+
chunks,
|
|
1066
|
+
get offset() {
|
|
1067
|
+
return offset;
|
|
1068
|
+
},
|
|
1069
|
+
push,
|
|
1070
|
+
entry(key, value) {
|
|
1071
|
+
push(encode(key));
|
|
1072
|
+
push(encode(value));
|
|
1073
|
+
},
|
|
1074
|
+
rawEntry(key, encoded) {
|
|
1075
|
+
push(encode(key));
|
|
1076
|
+
const at = offset;
|
|
1077
|
+
push(encoded);
|
|
1078
|
+
return at;
|
|
1079
|
+
},
|
|
1080
|
+
bytesEntry(key, bytes) {
|
|
1081
|
+
push(encode(key));
|
|
1082
|
+
push(binHeader(bytes.byteLength));
|
|
1083
|
+
const at = offset;
|
|
1084
|
+
push(bytes);
|
|
1085
|
+
return at;
|
|
1086
|
+
}
|
|
1087
|
+
};
|
|
1088
|
+
}
|
|
1089
|
+
function pageChecksums(bytes, pageBytes) {
|
|
1090
|
+
const pages = Math.ceil(bytes.byteLength / pageBytes);
|
|
1091
|
+
const checksums = new Uint8Array(pages * 4);
|
|
1092
|
+
const view = new DataView(checksums.buffer);
|
|
1093
|
+
for (let page = 0; page < pages; page++) {
|
|
1094
|
+
const from = page * pageBytes;
|
|
1095
|
+
view.setUint32(page * 4, crc32(bytes.subarray(from, Math.min(from + pageBytes, bytes.byteLength))), true);
|
|
1096
|
+
}
|
|
1097
|
+
return checksums;
|
|
1098
|
+
}
|
|
1099
|
+
function sectionOf(name, offset, bytes, pageBytes) {
|
|
1100
|
+
return { name, offset, length: bytes.byteLength, pageBytes, checksums: pageChecksums(bytes, pageBytes) };
|
|
1101
|
+
}
|
|
1102
|
+
function uint32Bytes(values) {
|
|
1103
|
+
const bytes = new Uint8Array(values.length * 4);
|
|
1104
|
+
const view = new DataView(bytes.buffer);
|
|
1105
|
+
for (let i = 0; i < values.length; i++) view.setUint32(i * 4, values[i], true);
|
|
1106
|
+
return bytes;
|
|
1107
|
+
}
|
|
1108
|
+
function writtenAsShortAscii(docId, list, at) {
|
|
1109
|
+
if (docId.length > FIXSTR_MAX_LENGTH) return false;
|
|
1110
|
+
for (let unit = 0; unit < docId.length; unit++) {
|
|
1111
|
+
const code = docId.charCodeAt(unit);
|
|
1112
|
+
if (code > ASCII_MAX) return false;
|
|
1113
|
+
list[at + 1 + unit] = code;
|
|
1114
|
+
}
|
|
1115
|
+
list[at] = FIXSTR | docId.length;
|
|
1116
|
+
return true;
|
|
1117
|
+
}
|
|
1118
|
+
function encodedIdList(docIds) {
|
|
1119
|
+
const header = arrayHeader(docIds.length);
|
|
1120
|
+
let capacity = header.byteLength;
|
|
1121
|
+
for (const docId of docIds) capacity += docId.length * UTF8_MAX_BYTES_PER_UNIT + STRING_HEADER_MAX_BYTES;
|
|
1122
|
+
const list = new Uint8Array(capacity);
|
|
1123
|
+
list.set(header);
|
|
1124
|
+
const offsets = new Uint32Array(docIds.length + 1);
|
|
1125
|
+
let cursor = header.byteLength;
|
|
1126
|
+
for (let i = 0; i < docIds.length; i++) {
|
|
1127
|
+
offsets[i] = cursor;
|
|
1128
|
+
if (writtenAsShortAscii(docIds[i], list, cursor)) {
|
|
1129
|
+
cursor += docIds[i].length + 1;
|
|
1130
|
+
continue;
|
|
1131
|
+
}
|
|
1132
|
+
const encoded = encode(docIds[i]);
|
|
1133
|
+
list.set(encoded, cursor);
|
|
1134
|
+
cursor += encoded.byteLength;
|
|
1135
|
+
}
|
|
1136
|
+
offsets[docIds.length] = cursor;
|
|
1137
|
+
return { list: list.subarray(0, cursor), offsets };
|
|
1138
|
+
}
|
|
1139
|
+
function positionsInIdOrder(docIds) {
|
|
1140
|
+
const positions = Array.from(docIds.keys());
|
|
1141
|
+
positions.sort((left, right) => compareCodePoints(docIds[left], docIds[right]));
|
|
1142
|
+
return Uint32Array.from(positions);
|
|
1143
|
+
}
|
|
1144
|
+
function magnitudeBytes(magnitudes) {
|
|
1145
|
+
const bytes = new Uint8Array(magnitudes.length * 8);
|
|
1146
|
+
const view = new DataView(bytes.buffer);
|
|
1147
|
+
for (let i = 0; i < magnitudes.length; i++) view.setFloat64(i * 8, magnitudes[i], true);
|
|
1148
|
+
return bytes;
|
|
1149
|
+
}
|
|
1150
|
+
function checksumAcrossPages(checksums, byteLength, pageBytes) {
|
|
1151
|
+
const view = new DataView(checksums.buffer, checksums.byteOffset, checksums.byteLength);
|
|
1152
|
+
let whole = 0;
|
|
1153
|
+
for (let page = 0; page * 4 < checksums.byteLength; page++) {
|
|
1154
|
+
const pageLength = Math.min(pageBytes, byteLength - page * pageBytes);
|
|
1155
|
+
whole = crc32Combine(whole, view.getUint32(page * 4, true), pageLength);
|
|
1156
|
+
}
|
|
1157
|
+
return whole;
|
|
1158
|
+
}
|
|
1159
|
+
function encodedHeadAndVectorsPlace(sections, headOffsetAfterHeader, vectors, checksums, vectorsKeyAndHeaderBytes) {
|
|
1160
|
+
let headLength = 0;
|
|
1161
|
+
let head = new Uint8Array(0);
|
|
1162
|
+
let vectorsSection = {
|
|
1163
|
+
name: "vectors",
|
|
1164
|
+
offset: 0,
|
|
1165
|
+
length: vectors.byteLength,
|
|
1166
|
+
pageBytes: SEGMENT_PAGE_BYTES,
|
|
1167
|
+
checksums
|
|
1168
|
+
};
|
|
1169
|
+
for (let attempt = 0; attempt < STABLE_HEAD_ATTEMPTS; attempt++) {
|
|
1170
|
+
vectorsSection = {
|
|
1171
|
+
...vectorsSection,
|
|
1172
|
+
offset: headOffsetAfterHeader(headLength) + headLength + vectorsKeyAndHeaderBytes
|
|
1173
|
+
};
|
|
1174
|
+
head = encode({ sections: [...sections, vectorsSection] });
|
|
1175
|
+
if (head.byteLength === headLength) break;
|
|
1176
|
+
headLength = head.byteLength;
|
|
1177
|
+
}
|
|
1178
|
+
return { head, vectorsSection };
|
|
1179
|
+
}
|
|
1180
|
+
function pagedVectorFileChunks(payload, heldMagnitudes) {
|
|
1181
|
+
const builder = createPayloadBuilder();
|
|
1182
|
+
const { codes } = payload;
|
|
1183
|
+
builder.push(Uint8Array.of(FIXMAP | 10));
|
|
1184
|
+
builder.entry("v", payload.v);
|
|
1185
|
+
builder.entry("fieldName", payload.fieldName);
|
|
1186
|
+
builder.entry("dimension", payload.dimension);
|
|
1187
|
+
const { list, offsets } = encodedIdList(payload.docIds);
|
|
1188
|
+
const docIdsAt = builder.rawEntry("docIds", list);
|
|
1189
|
+
const sections = [sectionOf("docIds", docIdsAt, list, SEGMENT_PAGE_BYTES)];
|
|
1190
|
+
builder.push(encode("codes"));
|
|
1191
|
+
if (codes === null) {
|
|
1192
|
+
builder.push(encode(null));
|
|
1193
|
+
} else {
|
|
1194
|
+
builder.push(Uint8Array.of(FIXMAP | 3));
|
|
1195
|
+
builder.entry("bits", codes.bits);
|
|
1196
|
+
builder.entry("centroid", codes.centroid);
|
|
1197
|
+
const recordsAt = builder.bytesEntry("records", codes.records);
|
|
1198
|
+
sections.push(sectionOf("records", recordsAt, codes.records, RECORD_PAGE_BYTES));
|
|
1199
|
+
}
|
|
1200
|
+
const magnitudes = magnitudeBytes(heldMagnitudes);
|
|
1201
|
+
sections.push(sectionOf("magnitudes", builder.bytesEntry("magnitudes", magnitudes), magnitudes, SEGMENT_PAGE_BYTES));
|
|
1202
|
+
const idOffsets = uint32Bytes(offsets);
|
|
1203
|
+
sections.push(sectionOf("idOffsets", builder.bytesEntry("idOffsets", idOffsets), idOffsets, SEGMENT_PAGE_BYTES));
|
|
1204
|
+
const idOrder = uint32Bytes(positionsInIdOrder(payload.docIds));
|
|
1205
|
+
sections.push(sectionOf("idOrder", builder.bytesEntry("idOrder", idOrder), idOrder, SEGMENT_PAGE_BYTES));
|
|
1206
|
+
const headKey = encode("head");
|
|
1207
|
+
const vectorsKey = encode("vectors");
|
|
1208
|
+
const vectorsHeader = binHeader(payload.vectors.byteLength);
|
|
1209
|
+
const beforeHead = builder.offset + headKey.byteLength;
|
|
1210
|
+
const vectorChecksums = pageChecksums(payload.vectors, SEGMENT_PAGE_BYTES);
|
|
1211
|
+
const { head } = encodedHeadAndVectorsPlace(
|
|
1212
|
+
sections,
|
|
1213
|
+
(headLength) => beforeHead + binHeader(headLength).byteLength,
|
|
1214
|
+
payload.vectors,
|
|
1215
|
+
vectorChecksums,
|
|
1216
|
+
vectorsKey.byteLength + vectorsHeader.byteLength
|
|
1217
|
+
);
|
|
1218
|
+
const headOffset = builder.bytesEntry("head", head);
|
|
1219
|
+
builder.push(vectorsKey);
|
|
1220
|
+
builder.push(vectorsHeader);
|
|
1221
|
+
const beforeVectors = [...builder.chunks];
|
|
1222
|
+
builder.push(payload.vectors);
|
|
1223
|
+
return {
|
|
1224
|
+
chunks: builder.chunks,
|
|
1225
|
+
headOffset,
|
|
1226
|
+
headLength: head.byteLength,
|
|
1227
|
+
headChecksum: crc32(head),
|
|
1228
|
+
beforeVectors,
|
|
1229
|
+
vectors: {
|
|
1230
|
+
byteLength: payload.vectors.byteLength,
|
|
1231
|
+
checksum: checksumAcrossPages(vectorChecksums, payload.vectors.byteLength, SEGMENT_PAGE_BYTES)
|
|
1232
|
+
}
|
|
1233
|
+
};
|
|
1234
|
+
}
|
|
1235
|
+
function nodeOffsetsOf(graph) {
|
|
1236
|
+
const offsets = new Uint32Array(graph.levels.length + 1);
|
|
1237
|
+
const view = new DataView(graph.neighbours.buffer, graph.neighbours.byteOffset, graph.neighbours.byteLength);
|
|
1238
|
+
let cursor = 0;
|
|
1239
|
+
for (let at = 0; at < graph.levels.length; at++) {
|
|
1240
|
+
offsets[at] = cursor;
|
|
1241
|
+
for (let layer = 0; layer < graph.levels[at]; layer++) cursor += (view.getUint32(cursor, true) + 1) * 4;
|
|
1242
|
+
}
|
|
1243
|
+
offsets[graph.levels.length] = cursor;
|
|
1244
|
+
return uint32Bytes(offsets);
|
|
1245
|
+
}
|
|
1246
|
+
function pagedGraphPartChunks(part) {
|
|
1247
|
+
const builder = createPayloadBuilder();
|
|
1248
|
+
builder.push(Uint8Array.of(FIXMAP | 4));
|
|
1249
|
+
builder.entry("v", part.v);
|
|
1250
|
+
builder.entry("fieldName", part.fieldName);
|
|
1251
|
+
builder.push(encode("graphs"));
|
|
1252
|
+
builder.push(arrayHeader(part.graphs.length));
|
|
1253
|
+
const layouts = [];
|
|
1254
|
+
for (const graph of part.graphs) {
|
|
1255
|
+
const { levels, neighbours, ...header } = graph;
|
|
1256
|
+
const headerEntries = Object.entries(header).filter(([, value]) => value !== void 0);
|
|
1257
|
+
builder.push(Uint8Array.of(FIXMAP | headerEntries.length + 2));
|
|
1258
|
+
for (const [key, value] of headerEntries) builder.entry(key, value);
|
|
1259
|
+
const levelsAt = builder.bytesEntry("levels", levels);
|
|
1260
|
+
const neighboursAt = builder.bytesEntry("neighbours", neighbours);
|
|
1261
|
+
layouts.push({
|
|
1262
|
+
...header,
|
|
1263
|
+
levels: sectionOf("levels", levelsAt, levels, SEGMENT_PAGE_BYTES),
|
|
1264
|
+
nodeOffsets: nodeOffsetsOf(graph),
|
|
1265
|
+
neighbours: sectionOf("neighbours", neighboursAt, neighbours, RECORD_PAGE_BYTES)
|
|
1266
|
+
});
|
|
1267
|
+
}
|
|
1268
|
+
const head = encode({ graphs: layouts });
|
|
1269
|
+
const headOffset = builder.bytesEntry("head", head);
|
|
1270
|
+
return { chunks: builder.chunks, headOffset, headLength: head.byteLength, headChecksum: crc32(head) };
|
|
1271
|
+
}
|
|
1272
|
+
var EMPTY_BIN = new Uint8Array(0);
|
|
1273
|
+
var EMPTY_BIN_BYTES = 2;
|
|
1274
|
+
var BIN82 = 196;
|
|
1275
|
+
var BIN162 = 197;
|
|
1276
|
+
var BIN322 = 198;
|
|
1277
|
+
var BIN8_MAX_BYTES2 = 255;
|
|
1278
|
+
var BIN16_MAX_BYTES = 65535;
|
|
1279
|
+
function binHeader2(byteLength) {
|
|
1280
|
+
if (byteLength <= BIN8_MAX_BYTES2) return Uint8Array.of(BIN82, byteLength);
|
|
1281
|
+
if (byteLength <= BIN16_MAX_BYTES) return Uint8Array.of(BIN162, byteLength >> 8, byteLength & 255);
|
|
1282
|
+
const header = new Uint8Array(5);
|
|
1283
|
+
header[0] = BIN322;
|
|
1284
|
+
new DataView(header.buffer).setUint32(1, byteLength, false);
|
|
1285
|
+
return header;
|
|
1286
|
+
}
|
|
1043
1287
|
function chunksEndingInBytes(withTheBytes, withEmptyBytesInTheirPlace, bytes) {
|
|
1044
1288
|
const head = encode(withEmptyBytesInTheirPlace);
|
|
1045
|
-
const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] ===
|
|
1289
|
+
const endsWithTheEmptyBin = head[head.length - EMPTY_BIN_BYTES] === BIN82 && head[head.length - 1] === 0;
|
|
1046
1290
|
if (!endsWithTheEmptyBin) return [encode(withTheBytes)];
|
|
1047
|
-
return [head.subarray(0, head.length - EMPTY_BIN_BYTES),
|
|
1291
|
+
return [head.subarray(0, head.length - EMPTY_BIN_BYTES), binHeader2(bytes.byteLength), bytes];
|
|
1048
1292
|
}
|
|
1049
1293
|
function vectorPartChunks(part) {
|
|
1050
1294
|
return chunksEndingInBytes(part, { ...part, vectors: EMPTY_BIN }, part.vectors);
|
|
@@ -1060,20 +1304,27 @@ function keptFileRef(input, prior, kept) {
|
|
|
1060
1304
|
{ key: kept.key }
|
|
1061
1305
|
);
|
|
1062
1306
|
}
|
|
1063
|
-
return { ...kept, bytes: listed.bytes, checksum: listed.checksum };
|
|
1307
|
+
return { ...kept, bytes: listed.bytes, checksum: listed.checksum, ...headFieldsOf(listed) };
|
|
1308
|
+
}
|
|
1309
|
+
function headFieldsOf(ref) {
|
|
1310
|
+
return ref.headOffset === void 0 ? {} : { headOffset: ref.headOffset, headLength: ref.headLength, headChecksum: ref.headChecksum };
|
|
1311
|
+
}
|
|
1312
|
+
function headRefOf(paged) {
|
|
1313
|
+
return paged === null ? {} : { headOffset: paged.headOffset, headLength: paged.headLength, headChecksum: paged.headChecksum };
|
|
1064
1314
|
}
|
|
1065
1315
|
async function writeGraphParts(input, fieldPath, plan, generation) {
|
|
1066
1316
|
const parts = plan.graphParts ?? [];
|
|
1067
1317
|
const refs = [];
|
|
1068
1318
|
for (let part = 0; part < parts.length; part++) {
|
|
1069
1319
|
const key = graphPartKey(input.indexName, fieldPath, generation, part);
|
|
1320
|
+
const paged = plan.storesHeads ? pagedGraphPartChunks(parts[part]) : null;
|
|
1070
1321
|
const packed = await packChecksummedEnvelope(
|
|
1071
|
-
[encode(parts[part])],
|
|
1322
|
+
paged === null ? [encode(parts[part])] : paged.chunks,
|
|
1072
1323
|
GRAPH_PART_ENVELOPE_VERSION,
|
|
1073
1324
|
`graph part "${key}"`
|
|
1074
1325
|
);
|
|
1075
1326
|
await input.store.write(key, packed.chunks);
|
|
1076
|
-
refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum });
|
|
1327
|
+
refs.push({ key, bytes: packed.byteLength, checksum: packed.checksum, ...headRefOf(paged) });
|
|
1077
1328
|
}
|
|
1078
1329
|
return refs;
|
|
1079
1330
|
}
|
|
@@ -1087,17 +1338,28 @@ async function writeField(input, fieldPath, plan, outcome) {
|
|
|
1087
1338
|
const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
|
|
1088
1339
|
const written = [];
|
|
1089
1340
|
for (let index = 0; index < plan.newFiles; index++) {
|
|
1090
|
-
const { payload, ordinals, dead } = plan.readNewFile(index);
|
|
1341
|
+
const { payload, ordinals, dead, magnitudes } = plan.readNewFile(index);
|
|
1091
1342
|
const id = nextFileId;
|
|
1092
1343
|
nextFileId += 1;
|
|
1093
1344
|
const key = vectorFileKey(input.indexName, fieldPath, id);
|
|
1094
|
-
const
|
|
1095
|
-
|
|
1345
|
+
const paged = plan.storesHeads && magnitudes !== null ? pagedVectorFileChunks(payload, magnitudes) : null;
|
|
1346
|
+
const what = `vector file "${key}"`;
|
|
1347
|
+
const packed = paged === null ? await packChecksummedEnvelope(vectorPartChunks(payload), VECTOR_FILE_ENVELOPE_VERSION, what) : packEnvelopeWithChecksum(
|
|
1348
|
+
paged.chunks,
|
|
1096
1349
|
VECTOR_FILE_ENVELOPE_VERSION,
|
|
1097
|
-
|
|
1350
|
+
what,
|
|
1351
|
+
crc32Combine(await checksumOfChunks(paged.beforeVectors), paged.vectors.checksum, paged.vectors.byteLength)
|
|
1098
1352
|
);
|
|
1099
1353
|
await input.store.write(key, packed.chunks);
|
|
1100
|
-
files.push({
|
|
1354
|
+
files.push({
|
|
1355
|
+
id,
|
|
1356
|
+
key,
|
|
1357
|
+
count: payload.docIds.length,
|
|
1358
|
+
dead,
|
|
1359
|
+
bytes: packed.byteLength,
|
|
1360
|
+
checksum: packed.checksum,
|
|
1361
|
+
...headRefOf(paged)
|
|
1362
|
+
});
|
|
1101
1363
|
written.push({ id, key, ordinals });
|
|
1102
1364
|
outcome.layouts.push({
|
|
1103
1365
|
fieldPath,
|
|
@@ -1174,8 +1436,41 @@ function removeWhatTheSegmentReplaces(manager, segment, contents) {
|
|
|
1174
1436
|
function yieldToEventLoop() {
|
|
1175
1437
|
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
1176
1438
|
}
|
|
1439
|
+
function removeFromPartition(manager, partitionId, docIds) {
|
|
1440
|
+
const { composite, docPartitionMap, insertOptions } = manager.diskSegmentAccess(partitionId);
|
|
1441
|
+
composite.beginBatch();
|
|
1442
|
+
try {
|
|
1443
|
+
for (const docId of docIds) {
|
|
1444
|
+
if (!composite.has(docId)) continue;
|
|
1445
|
+
composite.remove(docId, manager.config.schema, manager.language, insertOptions);
|
|
1446
|
+
docPartitionMap.delete(docId);
|
|
1447
|
+
}
|
|
1448
|
+
} finally {
|
|
1449
|
+
composite.endBatch();
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
async function attachPagedSegment(directory, ref, partitionId, manager, options) {
|
|
1453
|
+
const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-IYPLWHGC.mjs');
|
|
1454
|
+
if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
|
|
1455
|
+
const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
|
|
1456
|
+
requireAgreedSchema(options.schema, snapshot.schema, ref.key);
|
|
1457
|
+
const segment = createPagedFrozenSegment(snapshot);
|
|
1458
|
+
removeFromPartition(manager, partitionId, snapshot.tombstones);
|
|
1459
|
+
options.writeRecords?.loadPagedSegment(manager.indexName, partitionId, segment);
|
|
1460
|
+
if (snapshot.documentCount > 0) manager.attachFrozenSegment(partitionId, segment);
|
|
1461
|
+
else segment.paged?.close();
|
|
1462
|
+
}
|
|
1177
1463
|
async function loadPartitionSegments(store, partition, manager, options) {
|
|
1464
|
+
const { directory } = store;
|
|
1178
1465
|
for (const ref of partition.segments) {
|
|
1466
|
+
if (manager.config.textStorage === "disk" && directory !== void 0) {
|
|
1467
|
+
const { isPagedSegmentFile } = await import('./module-IYPLWHGC.mjs');
|
|
1468
|
+
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
1469
|
+
await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
|
|
1470
|
+
await yieldToEventLoop();
|
|
1471
|
+
continue;
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1179
1474
|
const contents = await readSegmentFile(store, ref);
|
|
1180
1475
|
requireAgreedSchema(options.schema, contents.schema, ref.key);
|
|
1181
1476
|
if (options.decodesEveryDocument) decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
@@ -1220,6 +1515,14 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
|
|
|
1220
1515
|
if (partition.partitionId > highestPartitionId) highestPartitionId = partition.partitionId;
|
|
1221
1516
|
}
|
|
1222
1517
|
growPartitionsTo(deps.manager, highestPartitionId);
|
|
1518
|
+
const { directory } = store;
|
|
1519
|
+
const opensVectorsOnDemand = manifest.vectors.some(
|
|
1520
|
+
(field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
|
|
1521
|
+
);
|
|
1522
|
+
if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
|
|
1523
|
+
const { startReadingOpeningSectionsAhead } = await import('./module-IYPLWHGC.mjs');
|
|
1524
|
+
await startReadingOpeningSectionsAhead(directory, manifest);
|
|
1525
|
+
}
|
|
1223
1526
|
for (const partition of manifest.partitions) {
|
|
1224
1527
|
if (partition.partitionId >= deps.manager.partitionCount) {
|
|
1225
1528
|
throw new NarsilError(
|
|
@@ -1237,7 +1540,12 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
|
|
|
1237
1540
|
for (const vector of manifest.vectors) {
|
|
1238
1541
|
const vectorIndex = deps.vectorIndexes.get(vector.fieldPath);
|
|
1239
1542
|
if (vectorIndex === void 0) continue;
|
|
1240
|
-
await loadVectorField(
|
|
1543
|
+
await loadVectorField(
|
|
1544
|
+
store,
|
|
1545
|
+
vector,
|
|
1546
|
+
vectorIndex,
|
|
1547
|
+
decodesEveryDocument ? null : (docId) => deps.manager.partitionIdOf(docId)
|
|
1548
|
+
);
|
|
1241
1549
|
}
|
|
1242
1550
|
return manifest.checkpoint;
|
|
1243
1551
|
}
|
|
@@ -1247,4 +1555,4 @@ async function reclaimUnlistedKeys(store, indexName, referenced) {
|
|
|
1247
1555
|
}
|
|
1248
1556
|
}
|
|
1249
1557
|
|
|
1250
|
-
export { SINGLE_NODE_PRIMARY_TERM,
|
|
1558
|
+
export { SINGLE_NODE_PRIMARY_TERM, createSeqOwner, decodeManifestPayload, decodeSegmentManifest, deserializePayloadV1, dropStoredVectorValues, durableMetadataConfigOf, growPartitionsTo, indexMetadataOf, invalidManifest, isId, isRecord, isSequenceNumber, isUint32, listedKeysOf, loadPartitionSegments, loadSegmentedCheckpoint, metadataConfigOf, metadataFromConfig, openManifestPayload, patternIndexesFromWire, readCurrentManifest, readMetadataEnvelope, readSegmentManifest, reclaimUnlistedKeys, sanitizeSurfaceForms, unpackEnvelopeBytes, unpackIndexSnapshotEnvelope, writeLiveVectors, writeMetadataEnvelope };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { sendRelayed, parseJson, validateQuery, rejectInvalid, refuseRegex, respondJson, withServerRerankWindow, respondError, loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedBodyBudget, createSharedRequestGate, registerServerRoutes, closeWhenTheProcessExits, statusLine, sendError, REQUEST_THREAD_MOVE_AFTER_MS, MAX_LONG_QUERIES_IN_FLIGHT, createDocumentReadHandlers, SERVER_CAPABILITIES, versionReport, serverRoutes, validateSuggest, SEARCH_CLOCK_SAMPLE_UNITS, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, encodeJson } from './chunk-SBMAWTRG.mjs';
|
|
2
1
|
import { withRerankQueryText } from './chunk-KZ3NX2P7.mjs';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
2
|
+
import { sendRelayed, parseJson, validateQuery, rejectInvalid, refuseRegex, respondJson, withServerRerankWindow, respondError, loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedBodyBudget, createSharedRequestGate, registerServerRoutes, closeWhenTheProcessExits, statusLine, sendError, REQUEST_THREAD_MOVE_AFTER_MS, MAX_LONG_QUERIES_IN_FLIGHT, createDocumentReadHandlers, SERVER_CAPABILITIES, versionReport, serverRoutes, validateSuggest, SEARCH_CLOCK_SAMPLE_UNITS, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, encodeJson } from './chunk-SBMAWTRG.mjs';
|
|
3
|
+
import { executeListDocuments, executeSuggest, executeQuery, lazyQueryBinding, executePreflight } from './chunk-VEFN6X6Q.mjs';
|
|
4
|
+
import { holdsQuotedText, resolveProjection, projectionKeepsField, setNestedValue } from './chunk-ZPWS2SKJ.mjs';
|
|
5
|
+
import { extractVectorFieldsFromSchema } from './chunk-KGOXHLLZ.mjs';
|
|
6
6
|
import { ServerErrorCodes, ErrorCodes, NarsilError, isNarsilError } from './chunk-HNHCNTVG.mjs';
|
|
7
7
|
|
|
8
8
|
// src/server/request-threads/messages.ts
|