@delali/narsil 0.3.1-next.131 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
- package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
- package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
- package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
- package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
- package/dist/browser/import-TLB2A5LJ.mjs +9 -0
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
- package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
- package/dist/chunk-46NZOPHS.mjs +21 -0
- package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
- package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
- package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
- package/dist/chunk-L76W3IZJ.mjs +193 -0
- package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-SRJBFG2T.mjs +881 -0
- package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
- package/dist/chunk-VJEGOELZ.mjs +177 -0
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
- package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
- package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
- package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +11 -1
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
- package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -8
- package/dist/distribution.mjs +982 -211
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
- package/dist/import-OOBUW3RR.mjs +44 -0
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +5 -5
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +22 -22
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
- package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +12 -10
- package/dist/server.mjs +54 -21
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
- package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
- package/openapi.json +114 -6
- package/package.json +12 -8
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { abandonChangeOf } from './chunk-XNWPP5NS.mjs';
|
|
2
|
+
|
|
1
3
|
// src/narsil/restore-queue.ts
|
|
2
4
|
function restoreDepsOf(core, host) {
|
|
3
5
|
return {
|
|
@@ -9,7 +11,11 @@ function restoreDepsOf(core, host) {
|
|
|
9
11
|
embeddingAdapters: core.embeddingAdapters,
|
|
10
12
|
markAnalysisStale: core.analysisRebuild.markStale,
|
|
11
13
|
clearAnalysisStale: core.analysisRebuild.clearStale,
|
|
12
|
-
abandonAnalysisRebuild:
|
|
14
|
+
abandonAnalysisRebuild: async (indexName) => {
|
|
15
|
+
await core.analysisRebuild.abandon(indexName);
|
|
16
|
+
await abandonChangeOf(core, indexName);
|
|
17
|
+
},
|
|
18
|
+
abandonAnalysisChange: (indexName) => abandonChangeOf(core, indexName),
|
|
13
19
|
indexState: core.indexState,
|
|
14
20
|
copies: core.orchestrator,
|
|
15
21
|
forgetWatermark: core.watermarkNotifier.forget
|
|
@@ -18,7 +24,7 @@ function restoreDepsOf(core, host) {
|
|
|
18
24
|
function queueRestore(core, host, restoresInFlight, indexName, data) {
|
|
19
25
|
const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
|
|
20
26
|
const restored = previous.then(async () => {
|
|
21
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
27
|
+
const { restoreFromSnapshot } = await import('./snapshot-QXBKHJAG.mjs');
|
|
22
28
|
await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
|
|
23
29
|
});
|
|
24
30
|
const settled = restored.then(
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { MIN_RERANK_TIMEOUT_MS, MAX_RERANK_TIMEOUT_MS, MIN_RERANK_WINDOW, MAX_RERANK_WINDOW, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS, DEFAULT_RERANK_WINDOW } from './chunk-JPTQZWJQ.mjs';
|
|
2
|
-
import { DEFAULT_LIMIT, DEFAULT_OFFSET } from './chunk-
|
|
3
|
-
import { NarsilError, ErrorCodes, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, RESULT_WINDOW } from './chunk-
|
|
2
|
+
import { DEFAULT_LIMIT, DEFAULT_OFFSET } from './chunk-MZI3TEOW.mjs';
|
|
3
|
+
import { NarsilError, ErrorCodes, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, RESULT_WINDOW } from './chunk-35B42UFM.mjs';
|
|
4
4
|
|
|
5
5
|
// src/search/hybrid-depth.ts
|
|
6
6
|
function fixedHybridDepthOf(hybrid) {
|
|
@@ -1,13 +1,18 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import './chunk-
|
|
4
|
-
import
|
|
5
|
-
import './chunk-
|
|
6
|
-
import
|
|
7
|
-
import {
|
|
1
|
+
import { indexMetadataOf, writeCheckpointSegments, writeLiveVectors, commitCheckpointManifest, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-KTOK275N.mjs';
|
|
2
|
+
import { createByteReader, restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-ZEV26AQ6.mjs';
|
|
3
|
+
import { DEFAULT_COMPACTION_THRESHOLD, manifestKey, manifestFileRefs, encodeSegmentManifest, readSegmentFile } from './chunk-MVM33ZUL.mjs';
|
|
4
|
+
import './chunk-TCLEOZ4P.mjs';
|
|
5
|
+
import { vectorsPerFile } from './chunk-6RHTAVZX.mjs';
|
|
6
|
+
import './chunk-HKX2G6WH.mjs';
|
|
7
|
+
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-LVY2LI35.mjs';
|
|
8
|
+
import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, archiveFootprint, storedBytesOfSegments } from './chunk-R4WUDZQV.mjs';
|
|
9
|
+
import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, ARRAY_POSITION_GAP, raiseEarlierReleaseGraphSetting, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateTextStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, prepareStoredSynonyms, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, isCompositePartition, arrayPositionsAreStale, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-5DB2VDGI.mjs';
|
|
8
10
|
import './chunk-JPTQZWJQ.mjs';
|
|
9
|
-
import './chunk-
|
|
10
|
-
import
|
|
11
|
+
import './chunk-26A5JFT2.mjs';
|
|
12
|
+
import './chunk-OIZS5KIX.mjs';
|
|
13
|
+
import { MAX_FILE_BYTES } from './chunk-D33A5VMU.mjs';
|
|
14
|
+
import './chunk-MZI3TEOW.mjs';
|
|
15
|
+
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
11
16
|
import { decode } from '@msgpack/msgpack';
|
|
12
17
|
|
|
13
18
|
// src/persistence/archive/memory-store.ts
|
|
@@ -67,10 +72,11 @@ function partitionVectorPlan(vectorIndex, docIds) {
|
|
|
67
72
|
codes: null,
|
|
68
73
|
vectors: vectorsToBytes(packed)
|
|
69
74
|
};
|
|
70
|
-
files.push({ payload, ordinals: new Int32Array(slice.length), dead: null });
|
|
75
|
+
files.push({ payload, ordinals: new Int32Array(slice.length), dead: null, magnitudes: null });
|
|
71
76
|
}
|
|
72
77
|
return {
|
|
73
78
|
unchanged: false,
|
|
79
|
+
storesHeads: false,
|
|
74
80
|
kept: [],
|
|
75
81
|
keptFiles: [],
|
|
76
82
|
newFiles: files.length,
|
|
@@ -162,6 +168,7 @@ async function buildArchiveFromMemory(input) {
|
|
|
162
168
|
|
|
163
169
|
// src/persistence/archive/constants.ts
|
|
164
170
|
var MAX_PAX_HEADER_BYTES = 65536;
|
|
171
|
+
var ARCHIVE_FILE_CHUNK_BYTES = 4194304;
|
|
165
172
|
|
|
166
173
|
// src/persistence/archive/tar.ts
|
|
167
174
|
var TAR_BLOCK_BYTES = 512;
|
|
@@ -346,17 +353,32 @@ async function readBlock(reader) {
|
|
|
346
353
|
if (block.byteLength < TAR_BLOCK_BYTES) invalidArchive("the archive ends inside an entry header");
|
|
347
354
|
return block;
|
|
348
355
|
}
|
|
356
|
+
async function skipPadding(reader, size) {
|
|
357
|
+
const remainder = size % TAR_BLOCK_BYTES;
|
|
358
|
+
if (remainder === 0) return;
|
|
359
|
+
const padding = await reader.readExactly(TAR_BLOCK_BYTES - remainder);
|
|
360
|
+
if (padding === null || padding.byteLength < TAR_BLOCK_BYTES - remainder) {
|
|
361
|
+
invalidArchive("the archive ends inside the padding of an entry");
|
|
362
|
+
}
|
|
363
|
+
}
|
|
349
364
|
async function readPadded(reader, size) {
|
|
350
365
|
const body = await reader.readExactly(size) ?? new Uint8Array(0);
|
|
351
366
|
if (body.byteLength < size) invalidArchive("the archive ends inside an entry");
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
367
|
+
await skipPadding(reader, size);
|
|
368
|
+
return body;
|
|
369
|
+
}
|
|
370
|
+
function entryBody(reader, size) {
|
|
371
|
+
let remaining = size;
|
|
372
|
+
async function* chunks() {
|
|
373
|
+
while (remaining > 0) {
|
|
374
|
+
const length = Math.min(ARCHIVE_FILE_CHUNK_BYTES, remaining);
|
|
375
|
+
const chunk = await reader.readExactly(length);
|
|
376
|
+
if (chunk === null || chunk.byteLength < length) invalidArchive("the archive ends inside an entry");
|
|
377
|
+
remaining -= length;
|
|
378
|
+
yield chunk;
|
|
357
379
|
}
|
|
358
380
|
}
|
|
359
|
-
return
|
|
381
|
+
return chunks();
|
|
360
382
|
}
|
|
361
383
|
async function nextEntryHeader(reader) {
|
|
362
384
|
const block = await readBlock(reader);
|
|
@@ -416,12 +438,16 @@ async function readSnapshotArchive(reader, handlers) {
|
|
|
416
438
|
if (header.size !== ref.bytes) {
|
|
417
439
|
invalidArchive(`the entry "${expected}" holds ${header.size} bytes where the manifest lists ${ref.bytes}`);
|
|
418
440
|
}
|
|
419
|
-
const
|
|
420
|
-
if (headerChecksumOf(
|
|
441
|
+
const start = await reader.peek(Math.min(HEADER_SIZE, header.size));
|
|
442
|
+
if (headerChecksumOf(start) !== ref.checksum) {
|
|
421
443
|
invalidArchive(`the entry "${expected}" carries a checksum other than the manifest lists`, { entry: expected });
|
|
422
444
|
}
|
|
423
445
|
seen.add(expected);
|
|
424
|
-
|
|
446
|
+
const entry = entryBody(reader, header.size);
|
|
447
|
+
await handlers.file(expected, ref, entry);
|
|
448
|
+
let unread = await entry.next();
|
|
449
|
+
while (unread.done !== true) unread = await entry.next();
|
|
450
|
+
await skipPadding(reader, header.size);
|
|
425
451
|
}
|
|
426
452
|
const trailing = await nextEntryHeader(reader);
|
|
427
453
|
if (trailing !== null) {
|
|
@@ -437,12 +463,20 @@ function entryChunks(name, bytes) {
|
|
|
437
463
|
(chunk) => chunk.byteLength > 0
|
|
438
464
|
);
|
|
439
465
|
}
|
|
466
|
+
async function* fileEntryChunks(source, name, ref) {
|
|
467
|
+
yield* tarEntryHeaders(name, ref.bytes).filter((chunk) => chunk.byteLength > 0);
|
|
468
|
+
for (let offset = 0; offset < ref.bytes; offset += ARCHIVE_FILE_CHUNK_BYTES) {
|
|
469
|
+
yield await source.readRange(ref, offset, Math.min(ARCHIVE_FILE_CHUNK_BYTES, ref.bytes - offset));
|
|
470
|
+
}
|
|
471
|
+
const padding = tarPadding(ref.bytes);
|
|
472
|
+
if (padding.byteLength > 0) yield padding;
|
|
473
|
+
}
|
|
440
474
|
async function* archiveChunks(source) {
|
|
441
475
|
yield* entryChunks(ARCHIVE_CONFIG_ENTRY, source.config);
|
|
442
476
|
yield* entryChunks(ARCHIVE_MANIFEST_ENTRY, source.manifestBytes);
|
|
443
477
|
const prefix = `${source.sourceIndexName}/`;
|
|
444
478
|
for (const ref of manifestFileRefs(source.manifest)) {
|
|
445
|
-
yield*
|
|
479
|
+
yield* fileEntryChunks(source, ref.key.slice(prefix.length), ref);
|
|
446
480
|
}
|
|
447
481
|
yield tarEnd();
|
|
448
482
|
}
|
|
@@ -476,6 +510,27 @@ function archiveStream(source) {
|
|
|
476
510
|
});
|
|
477
511
|
}
|
|
478
512
|
|
|
513
|
+
// src/persistence/archive/file-entry.ts
|
|
514
|
+
async function writeArchiveFile(store, key, chunks) {
|
|
515
|
+
const { directory } = store;
|
|
516
|
+
if (directory === void 0) {
|
|
517
|
+
const held = [];
|
|
518
|
+
for await (const chunk of chunks) held.push(chunk);
|
|
519
|
+
await store.write(key, held);
|
|
520
|
+
return;
|
|
521
|
+
}
|
|
522
|
+
const receiving = `${key}.receiving`;
|
|
523
|
+
await directory.remove(receiving);
|
|
524
|
+
const handle = await directory.appendHandle(receiving);
|
|
525
|
+
try {
|
|
526
|
+
for await (const chunk of chunks) await handle.append(chunk);
|
|
527
|
+
await handle.sync();
|
|
528
|
+
} finally {
|
|
529
|
+
await handle.close();
|
|
530
|
+
}
|
|
531
|
+
await directory.move(receiving, key);
|
|
532
|
+
}
|
|
533
|
+
|
|
479
534
|
// src/persistence/archive/validate.ts
|
|
480
535
|
function requireDimension(key, found, expected) {
|
|
481
536
|
if (expected === found) return;
|
|
@@ -499,6 +554,14 @@ function requireSchemaOfManifest(key, schema, manifest) {
|
|
|
499
554
|
async function validateStagedCheckpoint(store, manifest) {
|
|
500
555
|
for (const partition of manifest.partitions) {
|
|
501
556
|
for (const ref of partition.segments) {
|
|
557
|
+
const { directory } = store;
|
|
558
|
+
if (directory !== void 0) {
|
|
559
|
+
const { checkPagedSegmentFully, isPagedSegmentFile } = await import('./module-BUFNZMEU.mjs');
|
|
560
|
+
if (await isPagedSegmentFile(directory, ref.key)) {
|
|
561
|
+
requireSchemaOfManifest(ref.key, await checkPagedSegmentFully(directory, ref), manifest);
|
|
562
|
+
continue;
|
|
563
|
+
}
|
|
564
|
+
}
|
|
502
565
|
const contents = await readSegmentFile(store, ref);
|
|
503
566
|
requireSchemaOfManifest(ref.key, contents.schema, manifest);
|
|
504
567
|
decodeEveryStoredDocument(contents.arrays.documentTable);
|
|
@@ -515,161 +578,6 @@ async function validateStagedCheckpoint(store, manifest) {
|
|
|
515
578
|
}
|
|
516
579
|
}
|
|
517
580
|
|
|
518
|
-
// src/engine/snapshot/constants.ts
|
|
519
|
-
var RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE = 0.2;
|
|
520
|
-
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 7.86;
|
|
521
|
-
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR = 264;
|
|
522
|
-
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.24;
|
|
523
|
-
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 738;
|
|
524
|
-
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.2;
|
|
525
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE = 4.06;
|
|
526
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 14.5;
|
|
527
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR = 1482;
|
|
528
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 9.29;
|
|
529
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2141;
|
|
530
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 24.15;
|
|
531
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE = 0.16;
|
|
532
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 6.16;
|
|
533
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR = 204;
|
|
534
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.17;
|
|
535
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 695;
|
|
536
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.04;
|
|
537
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE = 2.14;
|
|
538
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 5.05;
|
|
539
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR = 1080;
|
|
540
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 0.72;
|
|
541
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 1744;
|
|
542
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 20.05;
|
|
543
|
-
var RESTORE_ESTIMATE_MARGIN = 1.25;
|
|
544
|
-
|
|
545
|
-
// src/engine/snapshot/memory-budget.ts
|
|
546
|
-
var MEGABYTE_BYTES = 1048576;
|
|
547
|
-
var ARCHIVE_RATES = {
|
|
548
|
-
worst: {
|
|
549
|
-
heap: {
|
|
550
|
-
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
|
|
551
|
-
perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
|
|
552
|
-
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
553
|
-
},
|
|
554
|
-
external: {
|
|
555
|
-
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
556
|
-
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
|
|
557
|
-
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
558
|
-
}
|
|
559
|
-
},
|
|
560
|
-
floor: {
|
|
561
|
-
heap: {
|
|
562
|
-
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
|
|
563
|
-
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
|
|
564
|
-
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
565
|
-
},
|
|
566
|
-
external: {
|
|
567
|
-
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
568
|
-
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
|
|
569
|
-
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
570
|
-
}
|
|
571
|
-
}
|
|
572
|
-
};
|
|
573
|
-
var EARLIER_SNAPSHOT_RATES = {
|
|
574
|
-
worst: {
|
|
575
|
-
heap: {
|
|
576
|
-
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
577
|
-
perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
578
|
-
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
579
|
-
},
|
|
580
|
-
external: {
|
|
581
|
-
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
582
|
-
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
583
|
-
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
584
|
-
}
|
|
585
|
-
},
|
|
586
|
-
floor: {
|
|
587
|
-
heap: {
|
|
588
|
-
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
589
|
-
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
590
|
-
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
591
|
-
},
|
|
592
|
-
external: {
|
|
593
|
-
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
594
|
-
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
595
|
-
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
596
|
-
}
|
|
597
|
-
}
|
|
598
|
-
};
|
|
599
|
-
function bytesAt(rate, storedBytes, vectors) {
|
|
600
|
-
let bytes = storedBytes * rate.perStoredByte;
|
|
601
|
-
for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
|
|
602
|
-
return bytes;
|
|
603
|
-
}
|
|
604
|
-
function footprintAt(rates, replacesIndex, storedBytes, vectors) {
|
|
605
|
-
const chosen = replacesIndex ? rates.worst : rates.floor;
|
|
606
|
-
const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
|
|
607
|
-
return {
|
|
608
|
-
heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
|
|
609
|
-
externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
|
|
610
|
-
};
|
|
611
|
-
}
|
|
612
|
-
function deadCountOf(dead) {
|
|
613
|
-
if (dead === null) return 0;
|
|
614
|
-
let count = 0;
|
|
615
|
-
for (const byte of dead) {
|
|
616
|
-
for (let bits = byte; bits !== 0; bits &= bits - 1) count += 1;
|
|
617
|
-
}
|
|
618
|
-
return count;
|
|
619
|
-
}
|
|
620
|
-
function archiveFootprint(prepared, manifest, replacesIndex) {
|
|
621
|
-
let segmentBytes = 0;
|
|
622
|
-
for (const partition of manifest.partitions) {
|
|
623
|
-
for (const segment of partition.segments) segmentBytes += segment.bytes;
|
|
624
|
-
}
|
|
625
|
-
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
626
|
-
const vectors = [];
|
|
627
|
-
for (const field of manifest.vectors) {
|
|
628
|
-
const dimension = dimensions.get(field.fieldPath) ?? 0;
|
|
629
|
-
for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
|
|
630
|
-
}
|
|
631
|
-
return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
|
|
632
|
-
}
|
|
633
|
-
function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
|
|
634
|
-
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
635
|
-
const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
|
|
636
|
-
return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
|
|
637
|
-
}
|
|
638
|
-
function inMegabytes(bytes) {
|
|
639
|
-
return Math.ceil(bytes / MEGABYTE_BYTES);
|
|
640
|
-
}
|
|
641
|
-
function shortfallOf(heapBytes, processBytes, countInUse) {
|
|
642
|
-
const heap = readHeapStatistics();
|
|
643
|
-
if (heap !== null) {
|
|
644
|
-
const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
|
|
645
|
-
const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
|
|
646
|
-
if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
|
|
647
|
-
}
|
|
648
|
-
const resident = readResidentMemory();
|
|
649
|
-
if (resident !== null) {
|
|
650
|
-
const availableBytes = countInUse ? resident.limitBytes - resident.residentBytes : resident.limitBytes;
|
|
651
|
-
if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
|
|
652
|
-
}
|
|
653
|
-
return null;
|
|
654
|
-
}
|
|
655
|
-
function refuse(indexName, replacesIndex, shortfall) {
|
|
656
|
-
const { budget, neededBytes, availableBytes } = shortfall;
|
|
657
|
-
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
658
|
-
const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
|
|
659
|
-
throw new NarsilError(
|
|
660
|
-
ErrorCodes.RESTORE_MEMORY_EXCEEDED,
|
|
661
|
-
`The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
|
|
662
|
-
{ indexName, budget, neededBytes, availableBytes }
|
|
663
|
-
);
|
|
664
|
-
}
|
|
665
|
-
function requireRestoreMemory(indexName, footprint, replacesIndex) {
|
|
666
|
-
const processBytes = footprint.heapBytes + footprint.externalBytes;
|
|
667
|
-
if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
|
|
668
|
-
const collected = collectAndReturnPages();
|
|
669
|
-
const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
|
|
670
|
-
if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
|
|
671
|
-
}
|
|
672
|
-
|
|
673
581
|
// src/workers/detached-index.ts
|
|
674
582
|
function attachablePolicy(shared) {
|
|
675
583
|
let attached = false;
|
|
@@ -709,6 +617,7 @@ function prepareRestoredIndex(configuration, deps, reject) {
|
|
|
709
617
|
requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
|
|
710
618
|
validateVectorPromotion(configFields.vectorPromotion);
|
|
711
619
|
validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
|
|
620
|
+
validateTextStorage(configFields.textStorage, deps.filesystemDurability);
|
|
712
621
|
if (configFields.required !== void 0 && configFields.required.length > 0) {
|
|
713
622
|
validateRequiredFieldsInSchema(configFields.required, schema);
|
|
714
623
|
}
|
|
@@ -735,12 +644,14 @@ function prepareRestoredIndex(configuration, deps, reject) {
|
|
|
735
644
|
}
|
|
736
645
|
const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
|
|
737
646
|
const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
|
|
647
|
+
const analysisGeneration = Number.isSafeInteger(configuration.analysisGeneration) ? Math.max(0, configuration.analysisGeneration) : 0;
|
|
738
648
|
return {
|
|
739
649
|
config,
|
|
740
650
|
language,
|
|
741
651
|
embeddingAdapter,
|
|
742
652
|
adapterName,
|
|
743
653
|
storedRevision,
|
|
654
|
+
analysisGeneration,
|
|
744
655
|
storedArrayPositionGap: storedArrayPositionGap(configuration.arrayPositionGap),
|
|
745
656
|
synonyms
|
|
746
657
|
};
|
|
@@ -775,7 +686,10 @@ function restoredRegistryEntry(prepared, manager, deps) {
|
|
|
775
686
|
partitionCount: manager.partitionCount,
|
|
776
687
|
analysisRevision: prepared.storedRevision,
|
|
777
688
|
arrayPositionGap: prepared.storedArrayPositionGap,
|
|
778
|
-
synonyms: prepared.synonyms?.encoded ?? null
|
|
689
|
+
synonyms: prepared.synonyms?.encoded ?? null,
|
|
690
|
+
analysisGeneration: prepared.analysisGeneration,
|
|
691
|
+
pendingAnalysis: null,
|
|
692
|
+
durableAnalysis: null
|
|
779
693
|
};
|
|
780
694
|
}
|
|
781
695
|
|
|
@@ -849,7 +763,14 @@ async function commitToStorage(indexName, restored, previousPartitions, deps) {
|
|
|
849
763
|
}
|
|
850
764
|
await staging.finish();
|
|
851
765
|
const relocate = await staging.relocation().catch(() => null);
|
|
852
|
-
if (relocate
|
|
766
|
+
if (relocate === null) return;
|
|
767
|
+
for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
|
|
768
|
+
const { manager } = restored.detached;
|
|
769
|
+
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
|
|
770
|
+
const partition = manager.getPartition(partitionId);
|
|
771
|
+
if (!isCompositePartition(partition)) continue;
|
|
772
|
+
for (const segment of partition.frozenSegments()) segment.paged?.relocate(relocate);
|
|
773
|
+
}
|
|
853
774
|
}
|
|
854
775
|
function activate(indexName, restored, entry, deps) {
|
|
855
776
|
deps.executor.installDetachedIndex(restored.detached);
|
|
@@ -871,6 +792,7 @@ function activate(indexName, restored, entry, deps) {
|
|
|
871
792
|
async function replaceExisting(indexName, restored, entry, deps) {
|
|
872
793
|
const release = { reloads: false, held: [], settled: Promise.resolve(), timedOut: false };
|
|
873
794
|
try {
|
|
795
|
+
await deps.abandonAnalysisChange(indexName);
|
|
874
796
|
await withinLimit(indexName, release, releaseCopies(indexName, release, deps), deps);
|
|
875
797
|
await deps.indexState.replace(indexName, async (wasOpen) => {
|
|
876
798
|
await deps.abandonAnalysisRebuild(indexName);
|
|
@@ -971,6 +893,9 @@ function partitionCountOf(manifest) {
|
|
|
971
893
|
function missingPart(what) {
|
|
972
894
|
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive holds no ${what}`);
|
|
973
895
|
}
|
|
896
|
+
function keepsTextOnDisk(prepared, staging) {
|
|
897
|
+
return prepared.config.textStorage === "disk" && staging.directory !== void 0;
|
|
898
|
+
}
|
|
974
899
|
async function stageArchive(indexName, reader, staging, deps) {
|
|
975
900
|
const staged = { prepared: null, manifest: null };
|
|
976
901
|
await readSnapshotArchive(reader, {
|
|
@@ -981,17 +906,29 @@ async function stageArchive(indexName, reader, staging, deps) {
|
|
|
981
906
|
if (staged.prepared === null) missingPart("configuration");
|
|
982
907
|
requireSameSchema(staged.prepared, manifest2);
|
|
983
908
|
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
984
|
-
|
|
909
|
+
const segmentBytes = keepsTextOnDisk(staged.prepared, staging) ? 0 : storedBytesOfSegments(manifest2);
|
|
910
|
+
requireRestoreMemory(
|
|
911
|
+
indexName,
|
|
912
|
+
archiveFootprint(staged.prepared, manifest2, replacesIndex, segmentBytes),
|
|
913
|
+
replacesIndex
|
|
914
|
+
);
|
|
985
915
|
staged.manifest = renamed(manifest2, sourceName, indexName);
|
|
986
916
|
},
|
|
987
|
-
file: async (entryName, _ref,
|
|
917
|
+
file: async (entryName, _ref, chunks) => {
|
|
988
918
|
if (staged.manifest === null) missingPart("manifest");
|
|
989
|
-
await staging
|
|
919
|
+
await writeArchiveFile(staging, `${indexName}/${entryName}`, chunks);
|
|
990
920
|
}
|
|
991
921
|
});
|
|
992
922
|
const { prepared, manifest } = staged;
|
|
993
923
|
if (prepared === null) missingPart("configuration");
|
|
994
924
|
if (manifest === null) missingPart("manifest");
|
|
925
|
+
const { directory } = staging;
|
|
926
|
+
if (keepsTextOnDisk(prepared, staging) && directory !== void 0) {
|
|
927
|
+
const { bytesThatSegmentsHoldInMemory } = await import('./module-BUFNZMEU.mjs');
|
|
928
|
+
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
929
|
+
const segmentBytes = await bytesThatSegmentsHoldInMemory(directory, manifest);
|
|
930
|
+
requireRestoreMemory(indexName, archiveFootprint(prepared, manifest, replacesIndex, segmentBytes), replacesIndex);
|
|
931
|
+
}
|
|
995
932
|
await validateStagedCheckpoint(staging, manifest);
|
|
996
933
|
return { prepared, manifest };
|
|
997
934
|
}
|
|
@@ -1174,9 +1111,24 @@ async function checkpointForArchive(deps) {
|
|
|
1174
1111
|
});
|
|
1175
1112
|
}
|
|
1176
1113
|
async function createSnapshot(deps) {
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1114
|
+
for (let attempt = 0; attempt < MAX_SNAPSHOT_ANALYSIS_ATTEMPTS; attempt += 1) {
|
|
1115
|
+
const checkpoint = await checkpointForArchive(deps);
|
|
1116
|
+
try {
|
|
1117
|
+
if ((checkpoint.manifest.analysisGeneration ?? 0) === deps.entry.analysisGeneration) {
|
|
1118
|
+
const config = await encodeIndexConfiguration(deps.entry);
|
|
1119
|
+
return archiveStream({ ...checkpoint, config, sourceIndexName: deps.indexName });
|
|
1120
|
+
}
|
|
1121
|
+
} catch (error) {
|
|
1122
|
+
checkpoint.release();
|
|
1123
|
+
throw error;
|
|
1124
|
+
}
|
|
1125
|
+
checkpoint.release();
|
|
1126
|
+
}
|
|
1127
|
+
throw new NarsilError(
|
|
1128
|
+
ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
|
|
1129
|
+
`The analysis of index "${deps.indexName}" changed while the engine wrote its snapshot, so try the snapshot again once the change has a durable checkpoint`,
|
|
1130
|
+
{ indexName: deps.indexName }
|
|
1131
|
+
);
|
|
1180
1132
|
}
|
|
1181
1133
|
function requireSnapshotSource(data) {
|
|
1182
1134
|
if (data instanceof Uint8Array) return data;
|
|
@@ -1,98 +1,20 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import
|
|
3
|
-
import './chunk-
|
|
4
|
-
import {
|
|
5
|
-
import './chunk-
|
|
1
|
+
import { DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-VFYMPCOV.mjs';
|
|
2
|
+
import './chunk-ZEV26AQ6.mjs';
|
|
3
|
+
import { encodeExactJson, encodeDocumentJson } from './chunk-CCOAOO73.mjs';
|
|
4
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-TCLEOZ4P.mjs';
|
|
5
|
+
import './chunk-WCFCHIBB.mjs';
|
|
6
|
+
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-LVY2LI35.mjs';
|
|
7
|
+
import { decodeStoredDocument, openChecksummedEnvelope } from './chunk-5DB2VDGI.mjs';
|
|
6
8
|
import './chunk-JPTQZWJQ.mjs';
|
|
7
|
-
import
|
|
8
|
-
import
|
|
9
|
+
import './chunk-26A5JFT2.mjs';
|
|
10
|
+
import './chunk-D33A5VMU.mjs';
|
|
11
|
+
import { EXPORT_CHUNK_BYTES } from './chunk-MZI3TEOW.mjs';
|
|
12
|
+
import './chunk-35B42UFM.mjs';
|
|
9
13
|
import { decode } from '@msgpack/msgpack';
|
|
10
14
|
|
|
11
|
-
// src/engine/export/json-line.ts
|
|
12
|
-
function refuse(path, kind, docId) {
|
|
13
|
-
throw new NarsilError(
|
|
14
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
15
|
-
`Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
|
|
16
|
-
{ docId, field: path }
|
|
17
|
-
);
|
|
18
|
-
}
|
|
19
|
-
function encodeNumber(value, path, target) {
|
|
20
|
-
if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
|
|
21
|
-
if (!target.tagged) refuse(path, String(value), target.docId);
|
|
22
|
-
return `{"${NUMBER_TAG}":"${String(value)}"}`;
|
|
23
|
-
}
|
|
24
|
-
function encodeDate(value, path, target) {
|
|
25
|
-
const milliseconds = value.getTime();
|
|
26
|
-
if (!target.tagged || Number.isNaN(milliseconds)) refuse(path, "a date", target.docId);
|
|
27
|
-
const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
|
|
28
|
-
const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
|
|
29
|
-
return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
|
|
30
|
-
}
|
|
31
|
-
function encodeBytes(value, path, target) {
|
|
32
|
-
if (!target.tagged) refuse(path, "bytes", target.docId);
|
|
33
|
-
return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
|
|
34
|
-
}
|
|
35
|
-
function encodeList(value, path, target) {
|
|
36
|
-
target.parts.push("[");
|
|
37
|
-
for (let at = 0; at < value.length; at++) {
|
|
38
|
-
if (at > 0) target.parts.push(",");
|
|
39
|
-
encodeValue(value[at], `${path}[${at}]`, target);
|
|
40
|
-
}
|
|
41
|
-
target.parts.push("]");
|
|
42
|
-
}
|
|
43
|
-
function encodeMap(value, path, target) {
|
|
44
|
-
const proto = Object.getPrototypeOf(value);
|
|
45
|
-
if (proto !== Object.prototype && proto !== null) refuse(path, "a value other than a plain object", target.docId);
|
|
46
|
-
const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
|
|
47
|
-
const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
|
|
48
|
-
target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
|
|
49
|
-
for (let at = 0; at < entries.length; at++) {
|
|
50
|
-
const [key, entry] = entries[at];
|
|
51
|
-
if (at > 0) target.parts.push(",");
|
|
52
|
-
target.parts.push(JSON.stringify(key), ":");
|
|
53
|
-
encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
|
|
54
|
-
}
|
|
55
|
-
target.parts.push(wrapped ? "}}" : "}");
|
|
56
|
-
}
|
|
57
|
-
function encodeValue(value, path, target) {
|
|
58
|
-
if (value === null) {
|
|
59
|
-
target.parts.push("null");
|
|
60
|
-
return;
|
|
61
|
-
}
|
|
62
|
-
switch (typeof value) {
|
|
63
|
-
case "string":
|
|
64
|
-
target.parts.push(JSON.stringify(value));
|
|
65
|
-
return;
|
|
66
|
-
case "boolean":
|
|
67
|
-
target.parts.push(value ? "true" : "false");
|
|
68
|
-
return;
|
|
69
|
-
case "number":
|
|
70
|
-
target.parts.push(encodeNumber(value, path, target));
|
|
71
|
-
return;
|
|
72
|
-
case "object":
|
|
73
|
-
break;
|
|
74
|
-
default:
|
|
75
|
-
refuse(path, `a ${typeof value}`, target.docId);
|
|
76
|
-
}
|
|
77
|
-
if (Array.isArray(value)) encodeList(value, path, target);
|
|
78
|
-
else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
|
|
79
|
-
else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
|
|
80
|
-
else encodeMap(value, path, target);
|
|
81
|
-
}
|
|
82
|
-
function encodeExactJson(value, docId) {
|
|
83
|
-
const target = { docId, tagged: false, parts: [] };
|
|
84
|
-
encodeValue(value, "", target);
|
|
85
|
-
return target.parts.join("");
|
|
86
|
-
}
|
|
87
|
-
function encodeDocumentJson(document, docId) {
|
|
88
|
-
const target = { docId, tagged: true, parts: [] };
|
|
89
|
-
encodeValue(document, "", target);
|
|
90
|
-
return target.parts.join("");
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
// src/engine/export/write.ts
|
|
94
15
|
var VECTOR_FILE_ENVELOPE_VERSION = 2;
|
|
95
16
|
var FLOAT32_BYTES = 4;
|
|
17
|
+
var ENVELOPE_VERSION_OFFSET = 4;
|
|
96
18
|
async function locateVectors(checkpoint) {
|
|
97
19
|
const located = /* @__PURE__ */ new Map();
|
|
98
20
|
for (const field of checkpoint.manifest.vectors) {
|
|
@@ -140,7 +62,59 @@ async function winnersOf(checkpoint, partition) {
|
|
|
140
62
|
}
|
|
141
63
|
return winners;
|
|
142
64
|
}
|
|
65
|
+
async function wholeSegmentScan(checkpoint, ref) {
|
|
66
|
+
const contents = await readSegment(checkpoint, ref);
|
|
67
|
+
const table = contents.arrays.documentTable;
|
|
68
|
+
const ids = /* @__PURE__ */ new Set();
|
|
69
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) ids.add(idAt(contents, ordinal));
|
|
70
|
+
return {
|
|
71
|
+
count: contents.arrays.documentCount,
|
|
72
|
+
schema: contents.schema,
|
|
73
|
+
tombstones: new Set(contents.tombstones),
|
|
74
|
+
idAt: (ordinal) => idAt(contents, ordinal),
|
|
75
|
+
holds: (docId) => ids.has(docId),
|
|
76
|
+
storedDocumentAt: (ordinal) => decodeStoredDocument(
|
|
77
|
+
table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
|
|
78
|
+
table.discardedFields
|
|
79
|
+
),
|
|
80
|
+
close: () => void 0
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
async function wholeFileVectorReads(checkpoint) {
|
|
84
|
+
const located = await locateVectors(checkpoint);
|
|
85
|
+
return {
|
|
86
|
+
async vectorOf(fieldPath, docId) {
|
|
87
|
+
const location = located.get(fieldPath)?.get(docId);
|
|
88
|
+
return location === void 0 ? null : readVector(checkpoint, location);
|
|
89
|
+
},
|
|
90
|
+
close: () => located.clear()
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
async function* pagedDocumentLines(checkpoint) {
|
|
94
|
+
const { openFileVectorReads, pagedPartitionLines } = await import('./module-BUFNZMEU.mjs');
|
|
95
|
+
const vectors = await openFileVectorReads(checkpoint, () => wholeFileVectorReads(checkpoint));
|
|
96
|
+
try {
|
|
97
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
98
|
+
yield* pagedPartitionLines(checkpoint, partition, vectors, (ref) => wholeSegmentScan(checkpoint, ref));
|
|
99
|
+
}
|
|
100
|
+
} finally {
|
|
101
|
+
vectors.close();
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
async function anyPartitionIsPaged(checkpoint) {
|
|
105
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
106
|
+
for (const ref of partition.segments) {
|
|
107
|
+
const [version] = await checkpoint.readRange(ref, ENVELOPE_VERSION_OFFSET, 1);
|
|
108
|
+
if (version !== KEYWORD_SEGMENT_ENVELOPE_VERSION) return true;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return false;
|
|
112
|
+
}
|
|
143
113
|
async function* documentLines(checkpoint) {
|
|
114
|
+
if (await anyPartitionIsPaged(checkpoint)) {
|
|
115
|
+
yield* pagedDocumentLines(checkpoint);
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
144
118
|
const vectors = await locateVectors(checkpoint);
|
|
145
119
|
for (const partition of checkpoint.manifest.partitions) {
|
|
146
120
|
const winners = await winnersOf(checkpoint, partition);
|