@delali/narsil 0.3.1-next.140 → 0.3.1-next.143
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
- package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
- package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
- package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
- package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
- package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
- package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
- package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
- package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
- package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
- package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
- package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
- package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
- package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
- package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
- package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
- package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
- package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
- package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
- package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
- package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
- package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
- package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
- package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
- package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
- package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
- package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
- package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
- package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
- package/dist/client.d.ts +7 -7
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
- package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +123 -42
- package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
- package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
- package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +11 -11
- package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
- package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +9 -9
- package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +8 -7
- package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
- package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
- package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +3 -2
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
- package/openapi.json +13 -0
- package/package.json +12 -8
- package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
- package/dist/chunk-BDIDUW2B.mjs +0 -387
- package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
import { readVectorFile, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-NRUXBYPH.mjs';
|
|
2
|
+
import { loadFailed, decodeSections, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-W2COJ5AH.mjs';
|
|
3
|
+
import { vectorsPerFile } from './chunk-Z4HFMYNX.mjs';
|
|
4
|
+
import { isDead, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-ICW45TTU.mjs';
|
|
5
|
+
import './chunk-TNBIK2LQ.mjs';
|
|
6
|
+
import './chunk-37CD3OKS.mjs';
|
|
7
|
+
import './chunk-DJHHF2L5.mjs';
|
|
8
|
+
import { PARTITION_COPY_VECTOR_RUN_BYTES } from './chunk-JDDXV6UF.mjs';
|
|
9
|
+
import { segmentsPrefix, encodeSegmentManifest, directoryCheckpointStore, vectorFileKey } from './chunk-UUZCQQG2.mjs';
|
|
10
|
+
import { HEADER_SIZE, crc32Init, crc32Final, engineVersionParts, writeHeader, RESERVED_SIZE, crc32Update } from './chunk-IO5F7ZBE.mjs';
|
|
11
|
+
import './chunk-VEFN6X6Q.mjs';
|
|
12
|
+
import './chunk-5WYVUPGD.mjs';
|
|
13
|
+
import './chunk-HJSN7UYG.mjs';
|
|
14
|
+
import './chunk-UIKN2OTQ.mjs';
|
|
15
|
+
import './chunk-AV4IJKD2.mjs';
|
|
16
|
+
import './chunk-EEBCDP36.mjs';
|
|
17
|
+
import './chunk-WJGO5PGV.mjs';
|
|
18
|
+
import { isCompositePartition } from './chunk-ZPWS2SKJ.mjs';
|
|
19
|
+
import { generateId } from './chunk-N37VMC62.mjs';
|
|
20
|
+
import './chunk-ZATRDAYU.mjs';
|
|
21
|
+
import './chunk-EBIHRL7B.mjs';
|
|
22
|
+
import './chunk-KOKM4PQN.mjs';
|
|
23
|
+
import './chunk-QSK35UC2.mjs';
|
|
24
|
+
import './chunk-74XB2SXI.mjs';
|
|
25
|
+
import './chunk-RKAVHNRW.mjs';
|
|
26
|
+
import './chunk-WQBCBKY3.mjs';
|
|
27
|
+
import './chunk-JUKYYXSP.mjs';
|
|
28
|
+
import { parseVectorDimension } from './chunk-DQNKVOJ2.mjs';
|
|
29
|
+
import './chunk-KGOXHLLZ.mjs';
|
|
30
|
+
import './chunk-U476IXKQ.mjs';
|
|
31
|
+
import './chunk-FQBFZAWG.mjs';
|
|
32
|
+
import './chunk-VJS4JFBQ.mjs';
|
|
33
|
+
import './chunk-GES7U2PM.mjs';
|
|
34
|
+
import './chunk-ZSWHPKGM.mjs';
|
|
35
|
+
import './chunk-O4LJQI4G.mjs';
|
|
36
|
+
import './chunk-X5CV2E2B.mjs';
|
|
37
|
+
import './chunk-WNBRW4Q2.mjs';
|
|
38
|
+
import './chunk-TGP5M5R7.mjs';
|
|
39
|
+
import './chunk-U2BKYEC7.mjs';
|
|
40
|
+
import './chunk-NPNK57CA.mjs';
|
|
41
|
+
import './chunk-4UMZ7KQL.mjs';
|
|
42
|
+
import './chunk-R6A773WK.mjs';
|
|
43
|
+
import './chunk-YTJCGKM4.mjs';
|
|
44
|
+
import './chunk-QHPYS427.mjs';
|
|
45
|
+
import './chunk-GLZHNSGB.mjs';
|
|
46
|
+
import './chunk-22YPZVMG.mjs';
|
|
47
|
+
import './chunk-RFBLQLLZ.mjs';
|
|
48
|
+
import './chunk-L5ORED7V.mjs';
|
|
49
|
+
import './chunk-HNHCNTVG.mjs';
|
|
50
|
+
import { decode, encode } from '@msgpack/msgpack';
|
|
51
|
+
|
|
52
|
+
var FLOAT32_BYTES = 4;
|
|
53
|
+
var BIN8 = 196;
|
|
54
|
+
var BIN16 = 197;
|
|
55
|
+
var BIN32 = 198;
|
|
56
|
+
var UINT8_MAX = 255;
|
|
57
|
+
var UINT16_MAX = 65535;
|
|
58
|
+
var FIXMAP_OF_SIX = 134;
|
|
59
|
+
var VECTOR_FILE_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
|
|
60
|
+
function binHeader(length) {
|
|
61
|
+
if (length <= UINT8_MAX) return Uint8Array.of(BIN8, length);
|
|
62
|
+
if (length <= UINT16_MAX) return Uint8Array.of(BIN16, length >> 8, length & 255);
|
|
63
|
+
const header = new Uint8Array(5);
|
|
64
|
+
header[0] = BIN32;
|
|
65
|
+
new DataView(header.buffer).setUint32(1, length, false);
|
|
66
|
+
return header;
|
|
67
|
+
}
|
|
68
|
+
async function docIdsOf(checkpoint, directory, ref, fieldPath) {
|
|
69
|
+
const { headOffset, headLength } = ref;
|
|
70
|
+
if (headOffset === void 0 || headLength === void 0) {
|
|
71
|
+
const { payload } = await readVectorFile(directoryCheckpointStore(directory), ref, fieldPath);
|
|
72
|
+
return payload.docIds;
|
|
73
|
+
}
|
|
74
|
+
const head = await checkpoint.readRange(ref, HEADER_SIZE + headOffset, headLength);
|
|
75
|
+
const raw = decode(head);
|
|
76
|
+
const sections = decodeSections(raw.sections, ref.bytes - HEADER_SIZE, `head of "${ref.key}"`, VECTOR_FILE_PAGE_SIZES);
|
|
77
|
+
const section = sections.find((candidate) => candidate.name === "docIds");
|
|
78
|
+
if (section === void 0) loadFailed(`The head of "${ref.key}" lists no section "docIds"`);
|
|
79
|
+
const docIds = decode(await checkpoint.readRange(ref, HEADER_SIZE + section.offset, section.length));
|
|
80
|
+
if (!Array.isArray(docIds) || docIds.length !== ref.count) {
|
|
81
|
+
loadFailed(`The vector file "${ref.key}" lists the wrong number of IDs`);
|
|
82
|
+
}
|
|
83
|
+
return docIds;
|
|
84
|
+
}
|
|
85
|
+
async function writePartitionVectorFile(checkpoint, directory, target, slots) {
|
|
86
|
+
const { dimension, fieldPath, temporaryKey } = target;
|
|
87
|
+
const width = dimension * FLOAT32_BYTES;
|
|
88
|
+
const handle = await directory.appendHandle(temporaryKey);
|
|
89
|
+
let state = crc32Init();
|
|
90
|
+
let payloadLength = 0;
|
|
91
|
+
async function appendPayload(bytes) {
|
|
92
|
+
state = crc32Update(state, bytes);
|
|
93
|
+
payloadLength += bytes.byteLength;
|
|
94
|
+
await handle.append(bytes);
|
|
95
|
+
}
|
|
96
|
+
try {
|
|
97
|
+
await handle.append(new Uint8Array(HEADER_SIZE));
|
|
98
|
+
await appendPayload(Uint8Array.of(FIXMAP_OF_SIX));
|
|
99
|
+
for (const [key, value] of [
|
|
100
|
+
["v", VECTOR_FILE_PAYLOAD_VERSION],
|
|
101
|
+
["fieldName", fieldPath],
|
|
102
|
+
["dimension", dimension],
|
|
103
|
+
["docIds", slots.map((slot) => slot.docId)],
|
|
104
|
+
["codes", null]
|
|
105
|
+
]) {
|
|
106
|
+
await appendPayload(encode(key));
|
|
107
|
+
await appendPayload(encode(value));
|
|
108
|
+
}
|
|
109
|
+
await appendPayload(encode("vectors"));
|
|
110
|
+
await appendPayload(binHeader(slots.length * width));
|
|
111
|
+
let at = 0;
|
|
112
|
+
while (at < slots.length) {
|
|
113
|
+
const first = slots[at];
|
|
114
|
+
let end = at + 1;
|
|
115
|
+
const limit = Math.max(1, Math.floor(PARTITION_COPY_VECTOR_RUN_BYTES / width));
|
|
116
|
+
while (end < slots.length && end - at < limit && slots[end].ref === first.ref && slots[end].position === first.position + (end - at)) {
|
|
117
|
+
end += 1;
|
|
118
|
+
}
|
|
119
|
+
const offset = first.ref.bytes - (first.ref.count - first.position) * width;
|
|
120
|
+
await appendPayload(await checkpoint.readRange(first.ref, offset, (end - at) * width));
|
|
121
|
+
at = end;
|
|
122
|
+
}
|
|
123
|
+
await handle.sync();
|
|
124
|
+
} finally {
|
|
125
|
+
await handle.close();
|
|
126
|
+
}
|
|
127
|
+
const checksum = crc32Final(state);
|
|
128
|
+
const [major, minor, patch] = engineVersionParts();
|
|
129
|
+
const header = writeHeader({
|
|
130
|
+
magic: "NRSL",
|
|
131
|
+
envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
|
|
132
|
+
engineVersionMajor: major,
|
|
133
|
+
engineVersionMinor: minor,
|
|
134
|
+
engineVersionPatch: patch,
|
|
135
|
+
payloadLength,
|
|
136
|
+
flags: {
|
|
137
|
+
compressionEnabled: false,
|
|
138
|
+
compressionAlgorithm: "none",
|
|
139
|
+
checksumPresent: true,
|
|
140
|
+
encryptionEnabled: false
|
|
141
|
+
},
|
|
142
|
+
checksum,
|
|
143
|
+
reserved: new Uint8Array(RESERVED_SIZE)
|
|
144
|
+
});
|
|
145
|
+
const patched = await directory.markerHandle(temporaryKey);
|
|
146
|
+
try {
|
|
147
|
+
await patched.writeSlot(0, header);
|
|
148
|
+
} finally {
|
|
149
|
+
await patched.close();
|
|
150
|
+
}
|
|
151
|
+
return {
|
|
152
|
+
temporaryKey,
|
|
153
|
+
ref: {
|
|
154
|
+
id: target.id,
|
|
155
|
+
key: target.key,
|
|
156
|
+
count: slots.length,
|
|
157
|
+
dead: null,
|
|
158
|
+
bytes: HEADER_SIZE + payloadLength,
|
|
159
|
+
checksum
|
|
160
|
+
}
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
async function partitionField(checkpoint, directory, scope, field, written) {
|
|
164
|
+
const fieldType = scope.manifest.schema[field.fieldPath];
|
|
165
|
+
const dimension = fieldType === void 0 ? null : parseVectorDimension(fieldType);
|
|
166
|
+
if (dimension === null) loadFailed(`The manifest gives vector field "${field.fieldPath}" no dimension`);
|
|
167
|
+
const capacity = vectorsPerFile(dimension, null);
|
|
168
|
+
const files = [];
|
|
169
|
+
let pending = [];
|
|
170
|
+
async function flush() {
|
|
171
|
+
if (pending.length === 0) return;
|
|
172
|
+
const id = files.length;
|
|
173
|
+
const key = vectorFileKey(scope.indexName, field.fieldPath, id);
|
|
174
|
+
const temporaryKey = `${scope.temporaryPrefix}${written.size}`;
|
|
175
|
+
const file = await writePartitionVectorFile(
|
|
176
|
+
checkpoint,
|
|
177
|
+
directory,
|
|
178
|
+
{ key, id, temporaryKey, fieldPath: field.fieldPath, dimension },
|
|
179
|
+
pending
|
|
180
|
+
);
|
|
181
|
+
files.push(file.ref);
|
|
182
|
+
written.set(key, temporaryKey);
|
|
183
|
+
pending = [];
|
|
184
|
+
}
|
|
185
|
+
for (const ref of field.files) {
|
|
186
|
+
const docIds = await docIdsOf(checkpoint, directory, ref, field.fieldPath);
|
|
187
|
+
for (let position = 0; position < docIds.length; position++) {
|
|
188
|
+
if (isDead(ref.dead, position) || !scope.holds(docIds[position])) continue;
|
|
189
|
+
pending.push({ docId: docIds[position], ref, position });
|
|
190
|
+
if (pending.length === capacity) await flush();
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
await flush();
|
|
194
|
+
return { fieldPath: field.fieldPath, nextFileId: files.length, files, graphGeneration: 0, graphParts: [] };
|
|
195
|
+
}
|
|
196
|
+
async function openPagedPartitionArchive(checkpoint, directory, indexName, partitionId, manager) {
|
|
197
|
+
const { manifest } = checkpoint;
|
|
198
|
+
const temporaryPrefix = `${segmentsPrefix(indexName)}partition-copy-${partitionId}-${generateId()}-`;
|
|
199
|
+
const written = /* @__PURE__ */ new Map();
|
|
200
|
+
const partition = manager.getPartition(partitionId);
|
|
201
|
+
async function removeWritten() {
|
|
202
|
+
for (const temporaryKey of written.values()) await directory.remove(temporaryKey);
|
|
203
|
+
}
|
|
204
|
+
try {
|
|
205
|
+
const vectors = [];
|
|
206
|
+
for (const field of manifest.vectors) {
|
|
207
|
+
vectors.push(
|
|
208
|
+
await partitionField(
|
|
209
|
+
checkpoint,
|
|
210
|
+
directory,
|
|
211
|
+
{ indexName, manifest, temporaryPrefix, holds: (docId) => partition.has(docId) },
|
|
212
|
+
field,
|
|
213
|
+
written
|
|
214
|
+
)
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
const narrowed = {
|
|
218
|
+
...manifest,
|
|
219
|
+
partitions: manifest.partitions.filter((entry) => entry.partitionId === partitionId),
|
|
220
|
+
checkpoint: manifest.checkpoint.filter((entry) => entry.partitionId === partitionId),
|
|
221
|
+
vectors
|
|
222
|
+
};
|
|
223
|
+
const encoded = await encodeSegmentManifest(narrowed);
|
|
224
|
+
let manifestLength = 0;
|
|
225
|
+
for (const chunk of encoded.chunks) manifestLength += chunk.byteLength;
|
|
226
|
+
const manifestBytes = new Uint8Array(manifestLength);
|
|
227
|
+
let cursor = 0;
|
|
228
|
+
for (const chunk of encoded.chunks) {
|
|
229
|
+
manifestBytes.set(chunk, cursor);
|
|
230
|
+
cursor += chunk.byteLength;
|
|
231
|
+
}
|
|
232
|
+
const { readRange } = directory;
|
|
233
|
+
return {
|
|
234
|
+
manifestBytes,
|
|
235
|
+
manifest: narrowed,
|
|
236
|
+
lastSeqNo: narrowed.checkpoint[0]?.lastSeqNo ?? 0,
|
|
237
|
+
directory,
|
|
238
|
+
readFile: (ref) => checkpoint.readFile(ref),
|
|
239
|
+
readRange: (ref, offset, length) => {
|
|
240
|
+
const temporaryKey = written.get(ref.key);
|
|
241
|
+
if (temporaryKey === void 0 || readRange === void 0) return checkpoint.readRange(ref, offset, length);
|
|
242
|
+
return readRange.call(directory, temporaryKey, offset, length);
|
|
243
|
+
},
|
|
244
|
+
release: () => {
|
|
245
|
+
void removeWritten().catch(() => void 0).finally(() => checkpoint.release());
|
|
246
|
+
}
|
|
247
|
+
};
|
|
248
|
+
} catch (error) {
|
|
249
|
+
await removeWritten().catch(() => void 0);
|
|
250
|
+
throw error;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
async function removeReceivedFilesNothingReads(directory, manager, partitionId, currentPrefix) {
|
|
254
|
+
const partition = manager.getPartition(partitionId);
|
|
255
|
+
const read = /* @__PURE__ */ new Set();
|
|
256
|
+
if (isCompositePartition(partition)) {
|
|
257
|
+
for (const segment of partition.frozenSegments()) {
|
|
258
|
+
if (segment.paged !== null) read.add(segment.paged.key);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
const everyReceipt = `${manager.indexName}/received/p${partitionId}/`;
|
|
262
|
+
for (const key of await directory.list(everyReceipt)) {
|
|
263
|
+
if (read.has(key)) continue;
|
|
264
|
+
if (key.startsWith(currentPrefix) && [...read].some((held) => held.startsWith(currentPrefix))) continue;
|
|
265
|
+
await directory.remove(key);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
export { openPagedPartitionArchive, removeReceivedFilesNothingReads };
|
package/dist/partial-words.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { partialWordSearchModule } from './chunk-
|
|
2
|
-
import './chunk-
|
|
1
|
+
import { partialWordSearchModule } from './chunk-DQNKVOJ2.mjs';
|
|
2
|
+
import './chunk-KGOXHLLZ.mjs';
|
|
3
3
|
import './chunk-U476IXKQ.mjs';
|
|
4
4
|
import './chunk-FQBFZAWG.mjs';
|
|
5
5
|
import './chunk-VJS4JFBQ.mjs';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { c as BM25FieldParams, P as PartitionConfig, e as ScoringMode, V as VectorIndexConfig } from '../../schema-
|
|
1
|
+
import { c as BM25FieldParams, P as PartitionConfig, e as ScoringMode, V as VectorIndexConfig } from '../../schema-B-Runv0n.js';
|
|
2
2
|
import '../../adapters-BQFQ4PRz.js';
|
|
3
3
|
import '../../synonyms-CpOAHPa-.js';
|
|
4
4
|
|
|
@@ -153,6 +153,9 @@ interface FileRef {
|
|
|
153
153
|
key: string;
|
|
154
154
|
bytes: number;
|
|
155
155
|
checksum: number;
|
|
156
|
+
headOffset?: number;
|
|
157
|
+
headLength?: number;
|
|
158
|
+
headChecksum?: number;
|
|
156
159
|
}
|
|
157
160
|
interface SegmentRef extends FileRef {
|
|
158
161
|
id: number;
|
|
@@ -210,6 +213,7 @@ interface IndexMetadata {
|
|
|
210
213
|
partitionLimits?: PartitionConfig;
|
|
211
214
|
defaultScoring?: ScoringMode;
|
|
212
215
|
trackPositions?: boolean;
|
|
216
|
+
textStorage?: 'memory' | 'disk';
|
|
213
217
|
strict?: boolean;
|
|
214
218
|
required?: string[];
|
|
215
219
|
vectorPromotion?: VectorIndexConfig;
|
|
@@ -1,30 +1,32 @@
|
|
|
1
|
-
import { createDurableDirectory } from '../../chunk-
|
|
1
|
+
import { createDurableDirectory } from '../../chunk-AST3SHFA.mjs';
|
|
2
2
|
import '../../chunk-W55X2BYN.mjs';
|
|
3
3
|
import '../../chunk-HLPRHOSO.mjs';
|
|
4
|
-
import {
|
|
5
|
-
import '../../chunk-
|
|
6
|
-
import '../../chunk-
|
|
4
|
+
import { writeCheckpointSegments } from '../../chunk-LA4WNDHL.mjs';
|
|
5
|
+
import '../../chunk-JDDXV6UF.mjs';
|
|
6
|
+
import { CHECKPOINT_WORKER_HEARTBEAT_MS } from '../../chunk-7AUASRE7.mjs';
|
|
7
|
+
import { directoryCheckpointStore } from '../../chunk-UUZCQQG2.mjs';
|
|
8
|
+
import '../../chunk-KEIYVZUX.mjs';
|
|
7
9
|
import '../../chunk-YH5OIL7B.mjs';
|
|
8
|
-
import '../../chunk-
|
|
9
|
-
import '../../chunk-
|
|
10
|
-
import '../../chunk-
|
|
10
|
+
import '../../chunk-IO5F7ZBE.mjs';
|
|
11
|
+
import '../../chunk-VEFN6X6Q.mjs';
|
|
12
|
+
import '../../chunk-5WYVUPGD.mjs';
|
|
13
|
+
import '../../chunk-HJSN7UYG.mjs';
|
|
11
14
|
import '../../chunk-UIKN2OTQ.mjs';
|
|
12
15
|
import '../../chunk-AV4IJKD2.mjs';
|
|
13
|
-
import '../../chunk-
|
|
14
|
-
import '../../chunk-
|
|
16
|
+
import '../../chunk-EEBCDP36.mjs';
|
|
17
|
+
import '../../chunk-WJGO5PGV.mjs';
|
|
18
|
+
import '../../chunk-ZPWS2SKJ.mjs';
|
|
15
19
|
import '../../chunk-N37VMC62.mjs';
|
|
16
20
|
import '../../chunk-ZATRDAYU.mjs';
|
|
17
21
|
import '../../chunk-EBIHRL7B.mjs';
|
|
18
|
-
import '../../chunk-HJSN7UYG.mjs';
|
|
19
|
-
import '../../chunk-EEBCDP36.mjs';
|
|
20
22
|
import '../../chunk-KOKM4PQN.mjs';
|
|
21
23
|
import '../../chunk-QSK35UC2.mjs';
|
|
22
24
|
import '../../chunk-74XB2SXI.mjs';
|
|
23
25
|
import '../../chunk-RKAVHNRW.mjs';
|
|
24
26
|
import '../../chunk-WQBCBKY3.mjs';
|
|
25
27
|
import '../../chunk-JUKYYXSP.mjs';
|
|
26
|
-
import '../../chunk-
|
|
27
|
-
import '../../chunk-
|
|
28
|
+
import '../../chunk-DQNKVOJ2.mjs';
|
|
29
|
+
import '../../chunk-KGOXHLLZ.mjs';
|
|
28
30
|
import '../../chunk-U476IXKQ.mjs';
|
|
29
31
|
import '../../chunk-FQBFZAWG.mjs';
|
|
30
32
|
import '../../chunk-VJS4JFBQ.mjs';
|
|
@@ -39,8 +41,8 @@ import '../../chunk-NPNK57CA.mjs';
|
|
|
39
41
|
import '../../chunk-4UMZ7KQL.mjs';
|
|
40
42
|
import '../../chunk-R6A773WK.mjs';
|
|
41
43
|
import '../../chunk-YTJCGKM4.mjs';
|
|
42
|
-
import '../../chunk-
|
|
43
|
-
import '../../chunk-
|
|
44
|
+
import '../../chunk-QHPYS427.mjs';
|
|
45
|
+
import '../../chunk-GLZHNSGB.mjs';
|
|
44
46
|
import '../../chunk-22YPZVMG.mjs';
|
|
45
47
|
import '../../chunk-RFBLQLLZ.mjs';
|
|
46
48
|
import '../../chunk-L5ORED7V.mjs';
|
package/dist/react.d.ts
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import { N as NarsilError, L as ListParams, f as ListResult, I as IndexInfo, k as IndexStats, Q as QueryParams, j as PreflightResult, m as QueryResult, g as SuggestParams, h as SuggestResult } from './config-
|
|
2
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './config-
|
|
3
|
-
import { f as TaskRecord, e as TaskProgress, c as ImportResult, d as TaskListQuery, T as TaskListPage } from './tasks-
|
|
4
|
-
export { E as ErrorDetail, I as ImportError, g as TaskStatus, h as TaskType } from './tasks-
|
|
1
|
+
import { N as NarsilError, L as ListParams, f as ListResult, I as IndexInfo, k as IndexStats, Q as QueryParams, j as PreflightResult, m as QueryResult, g as SuggestParams, h as SuggestResult } from './config-BhZxIeb1.js';
|
|
2
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './config-BhZxIeb1.js';
|
|
3
|
+
import { f as TaskRecord, e as TaskProgress, c as ImportResult, d as TaskListQuery, T as TaskListPage } from './tasks-CjRJRGTb.js';
|
|
4
|
+
export { E as ErrorDetail, I as ImportError, g as TaskStatus, h as TaskType } from './tasks-CjRJRGTb.js';
|
|
5
5
|
import { ReactNode, ReactElement } from 'react';
|
|
6
|
-
import { N as Narsil } from './engine-
|
|
7
|
-
import { N as NarsilClient, I as ImportSource } from './client-
|
|
8
|
-
import { A as AnyDocument } from './schema-
|
|
6
|
+
import { N as Narsil } from './engine-CtzbHOmG.js';
|
|
7
|
+
import { N as NarsilClient, I as ImportSource } from './client-B12YFALP.js';
|
|
8
|
+
import { A as AnyDocument } from './schema-B-Runv0n.js';
|
|
9
9
|
import './adapters-BQFQ4PRz.js';
|
|
10
10
|
import './synonyms-CpOAHPa-.js';
|
|
11
|
-
import './index-
|
|
12
|
-
import './config-
|
|
11
|
+
import './index-DsEurhSl.js';
|
|
12
|
+
import './config-CIG5yDyg.js';
|
|
13
13
|
import './types-DvncmvuQ.js';
|
|
14
14
|
|
|
15
15
|
/**
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { abandonChangeOf } from './chunk-
|
|
1
|
+
import { abandonChangeOf } from './chunk-46NZOPHS.mjs';
|
|
2
2
|
|
|
3
3
|
// src/narsil/restore-queue.ts
|
|
4
4
|
function restoreDepsOf(core, host) {
|
|
@@ -24,7 +24,7 @@ function restoreDepsOf(core, host) {
|
|
|
24
24
|
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
25
25
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
26
26
|
const restored = previous.then(async () => {
|
|
27
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
27
|
+
const { restoreFromSnapshot } = await import('./snapshot-CGM65VCD.mjs');
|
|
28
28
|
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
29
29
|
});
|
|
30
30
|
const settled = restored.then(
|
|
@@ -254,6 +254,13 @@ interface IndexConfig {
|
|
|
254
254
|
* because the engine's highlighter analyses the stored text again.
|
|
255
255
|
*/
|
|
256
256
|
trackPositions?: boolean;
|
|
257
|
+
/**
|
|
258
|
+
* Setting this to `'disk'` lets the index answer from segment files that the
|
|
259
|
+
* engine reads by position, so its keyword index and documents can outgrow
|
|
260
|
+
* the node's memory once a checkpoint covers them. The default is `'memory'`,
|
|
261
|
+
* and `'disk'` raises `CONFIG_INVALID` without filesystem durability.
|
|
262
|
+
*/
|
|
263
|
+
textStorage?: 'memory' | 'disk';
|
|
257
264
|
/** These settings control when vector fields move to an HNSW graph, and how that graph is built. */
|
|
258
265
|
vectorPromotion?: VectorIndexConfig;
|
|
259
266
|
/**
|
package/dist/server.d.ts
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import { G as GroupResult, Q as QueryParams, n as GroupConfig, m as QueryResult, L as ListParams, g as SuggestParams, k as IndexStats, f as ListResult, j as PreflightResult, h as SuggestResult } from './config-
|
|
2
|
-
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './config-
|
|
3
|
-
import { E as ErrorEnvelope, c as ServerOptions, N as NarsilServer } from './capabilities-
|
|
4
|
-
export { A as ASYNC_IMPORT_CAPABILITY, B as BM25_TUNING_CAPABILITY, C as CHANGE_ANALYSIS_CAPABILITY, d as CorsOptions, D as DOCUMENT_EXPORT_CAPABILITY, I as INDEX_LIFECYCLE_CAPABILITY, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY, e as RequestContext, f as RequestDenial, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, g as ServerLimits, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-
|
|
5
|
-
import { N as Narsil } from './engine-
|
|
6
|
-
import { j as CreateIndexRequest, H as HttpIndexConfig, B as BatchBody, a as ChangeAnalysisBody, D as DocumentBody, k as InsertBody, l as MultiGetBody, R as RebalanceBody, m as ReplicationFactorBody, d as TaskListQuery, V as VectorFieldBody, S as SynonymSetBody, n as BatchResponse, C as CapabilitiesResponse, o as ClusterAllocationResponse, p as ClusterTopologyResponse, q as DocumentCountResponse, r as DocumentExistsResponse, s as DocumentIdResponse, t as DocumentPutResponse, u as DocumentRemovedResponse, v as DocumentResponse, c as ImportResult, w as IndexClearedResponse, x as IndexCreatedResponse, y as IndexDroppedResponse, z as IndexListResponse, A as IndexPartitionsResponse, F as IndexStateResponse, L as LivenessResponse, M as MemoryStatsResponse, G as MultiGetResponse, O as OkResponse, J as ReadinessResponse, K as ReplicationFactorResponse, T as TaskListPage, f as TaskRecord, N as VectorMaintenanceResponse, P as VersionResponse, e as TaskProgress, i as TaskStore } from './tasks-
|
|
7
|
-
export { Q as BatchFailure, U as ClusterMemberReport, W as ClusterNodeReport, b as CreateIndexEmbedding, E as ErrorDetail, I as ImportError, X as PartitionAllocationReport, g as TaskStatus, h as TaskType } from './tasks-
|
|
8
|
-
import { e as ClusterHealth } from './config-
|
|
9
|
-
import { S as SchemaDefinition, P as PartitionConfig, V as VectorIndexConfig, B as BM25Params, F as FieldType } from './schema-
|
|
1
|
+
import { G as GroupResult, Q as QueryParams, n as GroupConfig, m as QueryResult, L as ListParams, g as SuggestParams, k as IndexStats, f as ListResult, j as PreflightResult, h as SuggestResult } from './config-BhZxIeb1.js';
|
|
2
|
+
export { C as ClientErrorCode, a as ClientErrorCodes, E as ErrorCode, b as ErrorCodes, N as NarsilError, c as NarsilErrorCode, S as ServerErrorCode, d as ServerErrorCodes, i as isNarsilError } from './config-BhZxIeb1.js';
|
|
3
|
+
import { E as ErrorEnvelope, c as ServerOptions, N as NarsilServer } from './capabilities-BCdUwK_L.js';
|
|
4
|
+
export { A as ASYNC_IMPORT_CAPABILITY, B as BM25_TUNING_CAPABILITY, C as CHANGE_ANALYSIS_CAPABILITY, d as CorsOptions, D as DOCUMENT_EXPORT_CAPABILITY, I as INDEX_LIFECYCLE_CAPABILITY, O as OnRequestHook, R as REBUILD_ANALYSIS_CAPABILITY, e as RequestContext, f as RequestDenial, S as SERVER_CAPABILITIES, a as SYNONYMS_CAPABILITY, g as ServerLimits, T as TASK_CANCEL_CAPABILITY, b as TASK_FILTER_CAPABILITY } from './capabilities-BCdUwK_L.js';
|
|
5
|
+
import { N as Narsil } from './engine-CtzbHOmG.js';
|
|
6
|
+
import { j as CreateIndexRequest, H as HttpIndexConfig, B as BatchBody, a as ChangeAnalysisBody, D as DocumentBody, k as InsertBody, l as MultiGetBody, R as RebalanceBody, m as ReplicationFactorBody, d as TaskListQuery, V as VectorFieldBody, S as SynonymSetBody, n as BatchResponse, C as CapabilitiesResponse, o as ClusterAllocationResponse, p as ClusterTopologyResponse, q as DocumentCountResponse, r as DocumentExistsResponse, s as DocumentIdResponse, t as DocumentPutResponse, u as DocumentRemovedResponse, v as DocumentResponse, c as ImportResult, w as IndexClearedResponse, x as IndexCreatedResponse, y as IndexDroppedResponse, z as IndexListResponse, A as IndexPartitionsResponse, F as IndexStateResponse, L as LivenessResponse, M as MemoryStatsResponse, G as MultiGetResponse, O as OkResponse, J as ReadinessResponse, K as ReplicationFactorResponse, T as TaskListPage, f as TaskRecord, N as VectorMaintenanceResponse, P as VersionResponse, e as TaskProgress, i as TaskStore } from './tasks-CjRJRGTb.js';
|
|
7
|
+
export { Q as BatchFailure, U as ClusterMemberReport, W as ClusterNodeReport, b as CreateIndexEmbedding, E as ErrorDetail, I as ImportError, X as PartitionAllocationReport, g as TaskStatus, h as TaskType } from './tasks-CjRJRGTb.js';
|
|
8
|
+
import { e as ClusterHealth } from './config-CIG5yDyg.js';
|
|
9
|
+
import { S as SchemaDefinition, P as PartitionConfig, V as VectorIndexConfig, B as BM25Params, F as FieldType } from './schema-B-Runv0n.js';
|
|
10
10
|
import './adapters-BQFQ4PRz.js';
|
|
11
11
|
import './synonyms-CpOAHPa-.js';
|
|
12
|
-
import './index-
|
|
12
|
+
import './index-DsEurhSl.js';
|
|
13
13
|
import './types-DvncmvuQ.js';
|
|
14
14
|
|
|
15
15
|
/**
|
package/dist/server.mjs
CHANGED
|
@@ -2,22 +2,22 @@ import { answerRequest } from './chunk-ZMBFWQXN.mjs';
|
|
|
2
2
|
import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, PROMETHEUS_TEXT_CONTENT_TYPE, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, clearGateSlot, clearBodyBudgetSlot, createCloseAtExitWords, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, isRemoveOnlyBatch, validateBatch, serializeBatchResult, parseWriteJson, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, withServerRerankWindow, respondAnswer, parseTaskListQuery, parseJsonOptional, validateChangeAnalysis, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, parseRefusalOf, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, SERVER_CAPABILITIES, toHttpError, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-SBMAWTRG.mjs';
|
|
3
3
|
export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, CHANGE_ANALYSIS_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-SBMAWTRG.mjs';
|
|
4
4
|
import { rerankOnHost } from './chunk-HJJWY6PT.mjs';
|
|
5
|
-
import { writeBlockLookupOf, indexReadsOf, isRecord, validateIndexName } from './chunk-
|
|
6
|
-
import './chunk-TCWQJL7O.mjs';
|
|
5
|
+
import { writeBlockLookupOf, indexReadsOf, isRecord, validateIndexName } from './chunk-BRUUBXCB.mjs';
|
|
7
6
|
import { readMemoryFigures } from './chunk-DD2673TX.mjs';
|
|
7
|
+
import './chunk-L76W3IZJ.mjs';
|
|
8
8
|
import './chunk-IWVEG5CX.mjs';
|
|
9
9
|
import './chunk-DJHHF2L5.mjs';
|
|
10
|
-
import './chunk-
|
|
11
|
-
import { rerankWindowCapOf, engineCoreOf, providedDocId, resolveVectorText } from './chunk-
|
|
10
|
+
import './chunk-JDDXV6UF.mjs';
|
|
11
|
+
import { rerankWindowCapOf, engineCoreOf, providedDocId, resolveVectorText } from './chunk-5WYVUPGD.mjs';
|
|
12
|
+
import { MAX_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
|
|
12
13
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
14
|
+
import './chunk-EEBCDP36.mjs';
|
|
13
15
|
import './chunk-N37VMC62.mjs';
|
|
14
16
|
import './chunk-ZATRDAYU.mjs';
|
|
15
17
|
import './chunk-EBIHRL7B.mjs';
|
|
16
|
-
import { MAX_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
|
|
17
|
-
import './chunk-EEBCDP36.mjs';
|
|
18
18
|
import './chunk-74XB2SXI.mjs';
|
|
19
19
|
import './chunk-WQBCBKY3.mjs';
|
|
20
|
-
import './chunk-
|
|
20
|
+
import './chunk-KGOXHLLZ.mjs';
|
|
21
21
|
import './chunk-U476IXKQ.mjs';
|
|
22
22
|
import './chunk-FQBFZAWG.mjs';
|
|
23
23
|
import './chunk-VJS4JFBQ.mjs';
|
|
@@ -787,6 +787,7 @@ function mapHttpIndexConfig(http) {
|
|
|
787
787
|
if (http.defaultScoring !== void 0) config.defaultScoring = http.defaultScoring;
|
|
788
788
|
if (http.bm25 !== void 0) config.bm25 = http.bm25;
|
|
789
789
|
if (http.trackPositions !== void 0) config.trackPositions = http.trackPositions;
|
|
790
|
+
if (http.textStorage !== void 0) config.textStorage = http.textStorage;
|
|
790
791
|
if (http.surfaceForms !== void 0) config.surfaceForms = http.surfaceForms === true;
|
|
791
792
|
if (http.vectorPromotion !== void 0) config.vectorPromotion = http.vectorPromotion;
|
|
792
793
|
if (http.strict !== void 0) config.strict = http.strict;
|
|
@@ -1,38 +1,42 @@
|
|
|
1
|
-
import { documentsOfPayloadV2 } from './chunk-
|
|
2
|
-
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-
|
|
3
|
-
import { indexMetadataOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope,
|
|
1
|
+
import { documentsOfPayloadV2 } from './chunk-NJJCPFXN.mjs';
|
|
2
|
+
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive, writeArchiveFile } from './chunk-KZL72VTC.mjs';
|
|
3
|
+
import { indexMetadataOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, deserializePayloadV1, SINGLE_NODE_PRIMARY_TERM } from './chunk-YHUTF6EA.mjs';
|
|
4
|
+
import { readVectorFile, readGraphParts } from './chunk-NRUXBYPH.mjs';
|
|
4
5
|
import { createByteReader } from './chunk-CZQ33DLW.mjs';
|
|
5
|
-
import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-
|
|
6
|
-
import './chunk-
|
|
6
|
+
import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-L76W3IZJ.mjs';
|
|
7
|
+
import './chunk-W2COJ5AH.mjs';
|
|
8
|
+
import './chunk-Z4HFMYNX.mjs';
|
|
7
9
|
import { decodeVectorIndexParts, bytesToVectors } from './chunk-ICW45TTU.mjs';
|
|
8
|
-
import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, archiveFootprint } from './chunk-
|
|
9
|
-
import './chunk-
|
|
10
|
+
import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, storedBytesOfSegments, archiveFootprint } from './chunk-VJEGOELZ.mjs';
|
|
11
|
+
import './chunk-TNBIK2LQ.mjs';
|
|
10
12
|
import './chunk-37CD3OKS.mjs';
|
|
11
13
|
import './chunk-DJHHF2L5.mjs';
|
|
12
|
-
import
|
|
13
|
-
import './chunk-
|
|
14
|
-
import './chunk-
|
|
14
|
+
import './chunk-LA4WNDHL.mjs';
|
|
15
|
+
import './chunk-JDDXV6UF.mjs';
|
|
16
|
+
import { readSegmentFile } from './chunk-7AUASRE7.mjs';
|
|
17
|
+
import { manifestKey, encodeSegmentManifest, manifestFileRefs } from './chunk-UUZCQQG2.mjs';
|
|
18
|
+
import './chunk-KEIYVZUX.mjs';
|
|
15
19
|
import './chunk-YH5OIL7B.mjs';
|
|
16
|
-
import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, storedArrayPositionGap, decodeIndexConfiguration, prepareStoredSynonyms, arrayPositionsAreStale, insertDocumentVectors } from './chunk-
|
|
17
|
-
import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-
|
|
18
|
-
import './chunk-
|
|
20
|
+
import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, storedArrayPositionGap, decodeIndexConfiguration, prepareStoredSynonyms, arrayPositionsAreStale, insertDocumentVectors } from './chunk-IO5F7ZBE.mjs';
|
|
21
|
+
import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-VEFN6X6Q.mjs';
|
|
22
|
+
import './chunk-5WYVUPGD.mjs';
|
|
23
|
+
import './chunk-HJSN7UYG.mjs';
|
|
19
24
|
import './chunk-UIKN2OTQ.mjs';
|
|
20
25
|
import './chunk-AV4IJKD2.mjs';
|
|
21
|
-
import
|
|
22
|
-
import {
|
|
26
|
+
import './chunk-EEBCDP36.mjs';
|
|
27
|
+
import { decodeEveryStoredDocument } from './chunk-WJGO5PGV.mjs';
|
|
28
|
+
import { ARRAY_POSITION_GAP, isCompositePartition } from './chunk-ZPWS2SKJ.mjs';
|
|
23
29
|
import './chunk-N37VMC62.mjs';
|
|
24
30
|
import './chunk-ZATRDAYU.mjs';
|
|
25
31
|
import './chunk-EBIHRL7B.mjs';
|
|
26
|
-
import './chunk-HJSN7UYG.mjs';
|
|
27
|
-
import './chunk-EEBCDP36.mjs';
|
|
28
32
|
import './chunk-KOKM4PQN.mjs';
|
|
29
33
|
import './chunk-QSK35UC2.mjs';
|
|
30
34
|
import './chunk-74XB2SXI.mjs';
|
|
31
35
|
import './chunk-RKAVHNRW.mjs';
|
|
32
36
|
import './chunk-WQBCBKY3.mjs';
|
|
33
37
|
import './chunk-JUKYYXSP.mjs';
|
|
34
|
-
import { parseVectorDimension } from './chunk-
|
|
35
|
-
import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, flattenSchema } from './chunk-
|
|
38
|
+
import { parseVectorDimension } from './chunk-DQNKVOJ2.mjs';
|
|
39
|
+
import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, flattenSchema } from './chunk-KGOXHLLZ.mjs';
|
|
36
40
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
37
41
|
import './chunk-FQBFZAWG.mjs';
|
|
38
42
|
import './chunk-VJS4JFBQ.mjs';
|
|
@@ -48,8 +52,8 @@ import './chunk-4UMZ7KQL.mjs';
|
|
|
48
52
|
import './chunk-R6A773WK.mjs';
|
|
49
53
|
import './chunk-YTJCGKM4.mjs';
|
|
50
54
|
import './chunk-RIIUZJQ3.mjs';
|
|
51
|
-
import './chunk-
|
|
52
|
-
import { loadVectorCodeFor } from './chunk-
|
|
55
|
+
import './chunk-QHPYS427.mjs';
|
|
56
|
+
import { loadVectorCodeFor } from './chunk-GLZHNSGB.mjs';
|
|
53
57
|
import './chunk-22YPZVMG.mjs';
|
|
54
58
|
import './chunk-RFBLQLLZ.mjs';
|
|
55
59
|
import './chunk-L5ORED7V.mjs';
|
|
@@ -79,6 +83,14 @@ function requireSchemaOfManifest(key, schema, manifest) {
|
|
|
79
83
|
async function validateStagedCheckpoint(store, manifest) {
|
|
80
84
|
for (const partition of manifest.partitions) {
|
|
81
85
|
for (const ref of partition.segments) {
|
|
86
|
+
const { directory } = store;
|
|
87
|
+
if (directory !== void 0) {
|
|
88
|
+
const { checkPagedSegmentFully, isPagedSegmentFile } = await import('./module-IYPLWHGC.mjs');
|
|
89
|
+
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
90
|
+
requireSchemaOfManifest(ref.key, await checkPagedSegmentFully(directory, ref), manifest);
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
82
94
|
const contents = await readSegmentFile(store, ref);
|
|
83
95
|
requireSchemaOfManifest(ref.key, contents.schema, manifest);
|
|
84
96
|
decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
@@ -134,6 +146,7 @@ function prepareRestoredIndex(configuration, deps, reject) {
|
|
|
134
146
|
requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
|
|
135
147
|
validateVectorPromotion(configFields.vectorPromotion);
|
|
136
148
|
validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
|
|
149
|
+
validateTextStorage(configFields.textStorage, deps.filesystemDurability);
|
|
137
150
|
if (configFields.required !== void 0 && configFields.required.length > 0) {
|
|
138
151
|
validateRequiredFieldsInSchema(configFields.required, schema);
|
|
139
152
|
}
|
|
@@ -279,7 +292,14 @@ async function commitToStorage(indexName, restored, previousPartitions, deps) {
|
|
|
279
292
|
}
|
|
280
293
|
await staging.finish();
|
|
281
294
|
const relocate = await staging.relocation().catch(() => null);
|
|
282
|
-
if (relocate
|
|
295
|
+
if (relocate === null) return;
|
|
296
|
+
for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
|
|
297
|
+
const { manager } = restored.detached;
|
|
298
|
+
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
|
|
299
|
+
const partition = manager.getPartition(partitionId);
|
|
300
|
+
if (!isCompositePartition(partition)) continue;
|
|
301
|
+
for (const segment of partition.frozenSegments()) segment.paged?.relocate(relocate);
|
|
302
|
+
}
|
|
283
303
|
}
|
|
284
304
|
function activate(indexName, restored, entry, deps) {
|
|
285
305
|
deps.executor.installDetachedIndex(restored.detached);
|
|
@@ -402,6 +422,9 @@ function partitionCountOf(manifest) {
|
|
|
402
422
|
function missingPart(what) {
|
|
403
423
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive holds no ${what}`);
|
|
404
424
|
}
|
|
425
|
+
function keepsTextOnDisk(prepared, staging) {
|
|
426
|
+
return prepared.config.textStorage === "disk" && staging.directory !== void 0;
|
|
427
|
+
}
|
|
405
428
|
async function stageArchive(indexName, reader, staging, deps) {
|
|
406
429
|
const staged = { prepared: null, manifest: null };
|
|
407
430
|
await readSnapshotArchive(reader, {
|
|
@@ -412,17 +435,29 @@ async function stageArchive(indexName, reader, staging, deps) {
|
|
|
412
435
|
if (staged.prepared === null) missingPart("configuration");
|
|
413
436
|
requireSameSchema(staged.prepared, manifest2);
|
|
414
437
|
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
415
|
-
|
|
438
|
+
const segmentBytes = keepsTextOnDisk(staged.prepared, staging) ? 0 : storedBytesOfSegments(manifest2);
|
|
439
|
+
requireRestoreMemory(
|
|
440
|
+
indexName,
|
|
441
|
+
archiveFootprint(staged.prepared, manifest2, replacesIndex, segmentBytes),
|
|
442
|
+
replacesIndex
|
|
443
|
+
);
|
|
416
444
|
staged.manifest = renamed(manifest2, sourceName, indexName);
|
|
417
445
|
},
|
|
418
|
-
file: async (entryName, _ref,
|
|
446
|
+
file: async (entryName, _ref, chunks) => {
|
|
419
447
|
if (staged.manifest === null) missingPart("manifest");
|
|
420
|
-
await staging
|
|
448
|
+
await writeArchiveFile(staging, `${indexName}/${entryName}`, chunks);
|
|
421
449
|
}
|
|
422
450
|
});
|
|
423
451
|
const { prepared, manifest } = staged;
|
|
424
452
|
if (prepared === null) missingPart("configuration");
|
|
425
453
|
if (manifest === null) missingPart("manifest");
|
|
454
|
+
const { directory } = staging;
|
|
455
|
+
if (keepsTextOnDisk(prepared, staging) && directory !== void 0) {
|
|
456
|
+
const { bytesThatSegmentsHoldInMemory } = await import('./module-IYPLWHGC.mjs');
|
|
457
|
+
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
458
|
+
const segmentBytes = await bytesThatSegmentsHoldInMemory(directory, manifest);
|
|
459
|
+
requireRestoreMemory(indexName, archiveFootprint(prepared, manifest, replacesIndex, segmentBytes), replacesIndex);
|
|
460
|
+
}
|
|
426
461
|
await validateStagedCheckpoint(staging, manifest);
|
|
427
462
|
return { prepared, manifest };
|
|
428
463
|
}
|