@delali/narsil 0.3.1-next.130 → 0.3.1-next.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-JN23AZQP.mjs +43 -0
- package/dist/analysis-change-entry-5JSIMEKG.mjs +1 -0
- package/dist/browser/analysis-change-57MIRJWR.mjs +499 -0
- package/dist/browser/analysis-change-entry-ICGRR3IN.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-3EYDURFD.mjs} +39 -27
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-4MFH226Q.mjs} +1 -1
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-5LYWZJE2.mjs} +2 -2
- package/dist/browser/chunk-67KSBUYR.mjs +173 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-6BSB4N6B.mjs} +2 -2
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-CGVBGHBX.mjs} +81 -6
- package/dist/browser/chunk-E5SFTOQR.mjs +21 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-QHNNULPW.mjs +265 -0
- package/dist/browser/chunk-XVMOQQ34.mjs +384 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-ZAIZQOTM.mjs} +256 -777
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-MTIAN6VT.mjs} +145 -50
- package/dist/browser/import-Q7MXD6BS.mjs +8 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-5J3PEUEB.mjs} +7 -6
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-DISFECE4.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-YBJ5MF2M.mjs} +36 -167
- package/dist/browser/{write-YIC2MKRY.mjs → write-RTWI5MXL.mjs} +8 -6
- package/dist/{capabilities-Bl8G9taj.d.ts → capabilities-DXAmysqJ.d.ts} +11 -11
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-4GCN34JV.mjs} +2 -2
- package/dist/{chunk-SOHMMNCH.mjs → chunk-6VHWXAOI.mjs} +4 -4
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/{chunk-QQTXT65X.mjs → chunk-B5I3IL4H.mjs} +12 -7
- package/dist/chunk-BDIDUW2B.mjs +387 -0
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-CCCGEPJM.mjs} +2 -2
- package/dist/{chunk-2JWNJSNE.mjs → chunk-CPCGEIOM.mjs} +11 -8
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/{chunk-HCGJKDZV.mjs → chunk-FFVLPCLH.mjs} +1 -1
- package/dist/{chunk-WVRDRLIC.mjs → chunk-FKARMUWH.mjs} +6 -6
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/chunk-HMDT3I63.mjs +21 -0
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-IBATGS3J.mjs} +87 -61
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-IP4HJOGC.mjs} +90 -16
- package/dist/{chunk-EUCX5K74.mjs → chunk-J3EHATEZ.mjs} +3 -3
- package/dist/{chunk-X5CNUKD5.mjs → chunk-K3AQOZVM.mjs} +37 -20
- package/dist/chunk-LK3MOTXF.mjs +174 -0
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-LLDFIKGP.mjs} +4 -4
- package/dist/{chunk-K77FJXKS.mjs → chunk-MTQTX4BP.mjs} +81 -8
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-PNNVTKMT.mjs} +2 -2
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-XABMCBHA.mjs → chunk-SBMAWTRG.mjs} +684 -99
- package/dist/chunk-TCWQJL7O.mjs +188 -0
- package/dist/{chunk-2MKX2SCO.mjs → chunk-U4X4AWKL.mjs} +7 -7
- package/dist/{chunk-PU6SKH7O.mjs → chunk-UBXJQJII.mjs} +2 -2
- package/dist/chunk-UEV53BBO.mjs +480 -0
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/{chunk-F6AAXI7T.mjs → chunk-VSUYTPME.mjs} +3 -3
- package/dist/{chunk-TGHNXN7W.mjs → chunk-W2XDK66Z.mjs} +180 -541
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-6M6SDH3S.mjs → chunk-XMQNOD5V.mjs} +8 -8
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-JT3YZGCE.mjs → chunk-XXKWEG53.mjs} +3 -188
- package/dist/{chunk-YBWSV5DV.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-Cx-TiqPE.d.ts → client-DApBeLbk.d.ts} +18 -3
- package/dist/client.d.ts +7 -8
- package/dist/client.mjs +11 -1
- package/dist/{events-DErop1LM.d.ts → config-B0H-vp4U.d.ts} +424 -2
- package/dist/{config-CzB9YvF_.d.ts → config-Brysl9NO.d.ts} +3 -3
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-J5UDLPL3.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -9
- package/dist/distribution.mjs +882 -192
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-AYOTAPNQ.mjs} +165 -70
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-B_W9sdZK.d.ts → engine-CuLMbH1-.d.ts} +64 -7
- package/dist/import-QRZAQ5RH.mjs +43 -0
- package/dist/{index-CKI1fDEl.d.ts → index-HYnqS1SE.d.ts} +2 -2
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +6 -7
- package/dist/index.mjs +20 -20
- package/dist/{long-query-thread-WVSMIBYP.mjs → long-query-thread-U7K5A72G.mjs} +14 -14
- package/dist/{module-2E4XYSGT.mjs → module-L2OZO5PQ.mjs} +11 -11
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +12 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -23
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +11 -8
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-RYSRRFS4.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-CS-clLun.d.ts → schema-BTXbHCqA.d.ts} +4 -1
- package/dist/server.d.ts +200 -11
- package/dist/server.mjs +66 -23
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-BTRZ3MZK.mjs} +59 -190
- package/dist/synonyms.mjs +6 -4
- package/dist/tasks-DjGjK70Y.d.ts +720 -0
- package/dist/{thread-DNK5XWXG.mjs → thread-USB3GOPY.mjs} +13 -13
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-ZKR7SNRT.mjs → write-GHTA56OW.mjs} +25 -22
- package/openapi.json +5310 -0
- package/package.json +16 -8
- package/dist/config-hOWGBeC6.d.ts +0 -414
- package/dist/tasks-BfF-VRhm.d.ts +0 -302
|
@@ -1,13 +1,16 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import './chunk-
|
|
4
|
-
import
|
|
1
|
+
import { createByteReader, restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-QHNNULPW.mjs';
|
|
2
|
+
import { vectorsPerFile } from './chunk-3EYDURFD.mjs';
|
|
3
|
+
import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, archiveFootprint } from './chunk-67KSBUYR.mjs';
|
|
4
|
+
import './chunk-OIZS5KIX.mjs';
|
|
5
|
+
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-CGVBGHBX.mjs';
|
|
6
|
+
import './chunk-6BSB4N6B.mjs';
|
|
7
|
+
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-5LYWZJE2.mjs';
|
|
8
|
+
import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, ARRAY_POSITION_GAP, raiseEarlierReleaseGraphSetting, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, prepareStoredSynonyms, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, arrayPositionsAreStale, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-ZAIZQOTM.mjs';
|
|
5
9
|
import './chunk-26A5JFT2.mjs';
|
|
6
|
-
import {
|
|
7
|
-
import { MAX_FILE_BYTES } from './chunk-AIIJDWMG.mjs';
|
|
10
|
+
import { MAX_FILE_BYTES } from './chunk-4MFH226Q.mjs';
|
|
8
11
|
import './chunk-JPTQZWJQ.mjs';
|
|
9
|
-
import './chunk-
|
|
10
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
12
|
+
import './chunk-MZI3TEOW.mjs';
|
|
13
|
+
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
11
14
|
import { decode } from '@msgpack/msgpack';
|
|
12
15
|
|
|
13
16
|
// src/persistence/archive/memory-store.ts
|
|
@@ -515,161 +518,6 @@ async function validateStagedCheckpoint(store, manifest) {
|
|
|
515
518
|
}
|
|
516
519
|
}
|
|
517
520
|
|
|
518
|
-
// src/engine/snapshot/constants.ts
|
|
519
|
-
var RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE = 0.2;
|
|
520
|
-
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 7.86;
|
|
521
|
-
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR = 264;
|
|
522
|
-
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.24;
|
|
523
|
-
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 738;
|
|
524
|
-
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.2;
|
|
525
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE = 4.06;
|
|
526
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 14.5;
|
|
527
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR = 1482;
|
|
528
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 9.29;
|
|
529
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2141;
|
|
530
|
-
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 24.15;
|
|
531
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE = 0.16;
|
|
532
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 6.16;
|
|
533
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR = 204;
|
|
534
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.17;
|
|
535
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 695;
|
|
536
|
-
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.04;
|
|
537
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE = 2.14;
|
|
538
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 5.05;
|
|
539
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR = 1080;
|
|
540
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 0.72;
|
|
541
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 1744;
|
|
542
|
-
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 20.05;
|
|
543
|
-
var RESTORE_ESTIMATE_MARGIN = 1.25;
|
|
544
|
-
|
|
545
|
-
// src/engine/snapshot/memory-budget.ts
|
|
546
|
-
var MEGABYTE_BYTES = 1048576;
|
|
547
|
-
var ARCHIVE_RATES = {
|
|
548
|
-
worst: {
|
|
549
|
-
heap: {
|
|
550
|
-
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
|
|
551
|
-
perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
|
|
552
|
-
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
553
|
-
},
|
|
554
|
-
external: {
|
|
555
|
-
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
556
|
-
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
|
|
557
|
-
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
558
|
-
}
|
|
559
|
-
},
|
|
560
|
-
floor: {
|
|
561
|
-
heap: {
|
|
562
|
-
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
|
|
563
|
-
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
|
|
564
|
-
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
565
|
-
},
|
|
566
|
-
external: {
|
|
567
|
-
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
568
|
-
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
|
|
569
|
-
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
570
|
-
}
|
|
571
|
-
}
|
|
572
|
-
};
|
|
573
|
-
var EARLIER_SNAPSHOT_RATES = {
|
|
574
|
-
worst: {
|
|
575
|
-
heap: {
|
|
576
|
-
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
577
|
-
perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
578
|
-
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
579
|
-
},
|
|
580
|
-
external: {
|
|
581
|
-
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
582
|
-
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
583
|
-
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
584
|
-
}
|
|
585
|
-
},
|
|
586
|
-
floor: {
|
|
587
|
-
heap: {
|
|
588
|
-
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
589
|
-
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
590
|
-
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
591
|
-
},
|
|
592
|
-
external: {
|
|
593
|
-
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
594
|
-
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
595
|
-
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
596
|
-
}
|
|
597
|
-
}
|
|
598
|
-
};
|
|
599
|
-
function bytesAt(rate, storedBytes, vectors) {
|
|
600
|
-
let bytes = storedBytes * rate.perStoredByte;
|
|
601
|
-
for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
|
|
602
|
-
return bytes;
|
|
603
|
-
}
|
|
604
|
-
function footprintAt(rates, replacesIndex, storedBytes, vectors) {
|
|
605
|
-
const chosen = replacesIndex ? rates.worst : rates.floor;
|
|
606
|
-
const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
|
|
607
|
-
return {
|
|
608
|
-
heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
|
|
609
|
-
externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
|
|
610
|
-
};
|
|
611
|
-
}
|
|
612
|
-
function deadCountOf(dead) {
|
|
613
|
-
if (dead === null) return 0;
|
|
614
|
-
let count = 0;
|
|
615
|
-
for (const byte of dead) {
|
|
616
|
-
for (let bits = byte; bits !== 0; bits &= bits - 1) count += 1;
|
|
617
|
-
}
|
|
618
|
-
return count;
|
|
619
|
-
}
|
|
620
|
-
function archiveFootprint(prepared, manifest, replacesIndex) {
|
|
621
|
-
let segmentBytes = 0;
|
|
622
|
-
for (const partition of manifest.partitions) {
|
|
623
|
-
for (const segment of partition.segments) segmentBytes += segment.bytes;
|
|
624
|
-
}
|
|
625
|
-
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
626
|
-
const vectors = [];
|
|
627
|
-
for (const field of manifest.vectors) {
|
|
628
|
-
const dimension = dimensions.get(field.fieldPath) ?? 0;
|
|
629
|
-
for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
|
|
630
|
-
}
|
|
631
|
-
return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
|
|
632
|
-
}
|
|
633
|
-
function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
|
|
634
|
-
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
635
|
-
const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
|
|
636
|
-
return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
|
|
637
|
-
}
|
|
638
|
-
function inMegabytes(bytes) {
|
|
639
|
-
return Math.ceil(bytes / MEGABYTE_BYTES);
|
|
640
|
-
}
|
|
641
|
-
function shortfallOf(heapBytes, processBytes, countInUse) {
|
|
642
|
-
const heap = readHeapStatistics();
|
|
643
|
-
if (heap !== null) {
|
|
644
|
-
const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
|
|
645
|
-
const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
|
|
646
|
-
if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
|
|
647
|
-
}
|
|
648
|
-
const resident = readResidentMemory();
|
|
649
|
-
if (resident !== null) {
|
|
650
|
-
const availableBytes = countInUse ? resident.limitBytes - resident.residentBytes : resident.limitBytes;
|
|
651
|
-
if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
|
|
652
|
-
}
|
|
653
|
-
return null;
|
|
654
|
-
}
|
|
655
|
-
function refuse(indexName, replacesIndex, shortfall) {
|
|
656
|
-
const { budget, neededBytes, availableBytes } = shortfall;
|
|
657
|
-
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
658
|
-
const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
|
|
659
|
-
throw new NarsilError(
|
|
660
|
-
ErrorCodes.RESTORE_MEMORY_EXCEEDED,
|
|
661
|
-
`The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
|
|
662
|
-
{ indexName, budget, neededBytes, availableBytes }
|
|
663
|
-
);
|
|
664
|
-
}
|
|
665
|
-
function requireRestoreMemory(indexName, footprint, replacesIndex) {
|
|
666
|
-
const processBytes = footprint.heapBytes + footprint.externalBytes;
|
|
667
|
-
if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
|
|
668
|
-
const collected = collectAndReturnPages();
|
|
669
|
-
const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
|
|
670
|
-
if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
|
|
671
|
-
}
|
|
672
|
-
|
|
673
521
|
// src/workers/detached-index.ts
|
|
674
522
|
function attachablePolicy(shared) {
|
|
675
523
|
let attached = false;
|
|
@@ -735,12 +583,14 @@ function prepareRestoredIndex(configuration, deps, reject) {
|
|
|
735
583
|
}
|
|
736
584
|
const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
|
|
737
585
|
const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
|
|
586
|
+
const analysisGeneration = Number.isSafeInteger(configuration.analysisGeneration) ? Math.max(0, configuration.analysisGeneration) : 0;
|
|
738
587
|
return {
|
|
739
588
|
config,
|
|
740
589
|
language,
|
|
741
590
|
embeddingAdapter,
|
|
742
591
|
adapterName,
|
|
743
592
|
storedRevision,
|
|
593
|
+
analysisGeneration,
|
|
744
594
|
storedArrayPositionGap: storedArrayPositionGap(configuration.arrayPositionGap),
|
|
745
595
|
synonyms
|
|
746
596
|
};
|
|
@@ -775,7 +625,10 @@ function restoredRegistryEntry(prepared, manager, deps) {
|
|
|
775
625
|
partitionCount: manager.partitionCount,
|
|
776
626
|
analysisRevision: prepared.storedRevision,
|
|
777
627
|
arrayPositionGap: prepared.storedArrayPositionGap,
|
|
778
|
-
synonyms: prepared.synonyms?.encoded ?? null
|
|
628
|
+
synonyms: prepared.synonyms?.encoded ?? null,
|
|
629
|
+
analysisGeneration: prepared.analysisGeneration,
|
|
630
|
+
pendingAnalysis: null,
|
|
631
|
+
durableAnalysis: null
|
|
779
632
|
};
|
|
780
633
|
}
|
|
781
634
|
|
|
@@ -871,6 +724,7 @@ function activate(indexName, restored, entry, deps) {
|
|
|
871
724
|
async function replaceExisting(indexName, restored, entry, deps) {
|
|
872
725
|
const release = { reloads: false, held: [], settled: Promise.resolve(), timedOut: false };
|
|
873
726
|
try {
|
|
727
|
+
await deps.abandonAnalysisChange(indexName);
|
|
874
728
|
await withinLimit(indexName, release, releaseCopies(indexName, release, deps), deps);
|
|
875
729
|
await deps.indexState.replace(indexName, async (wasOpen) => {
|
|
876
730
|
await deps.abandonAnalysisRebuild(indexName);
|
|
@@ -1174,9 +1028,24 @@ async function checkpointForArchive(deps) {
|
|
|
1174
1028
|
});
|
|
1175
1029
|
}
|
|
1176
1030
|
async function createSnapshot(deps) {
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1031
|
+
for (let attempt = 0; attempt < MAX_SNAPSHOT_ANALYSIS_ATTEMPTS; attempt += 1) {
|
|
1032
|
+
const checkpoint = await checkpointForArchive(deps);
|
|
1033
|
+
try {
|
|
1034
|
+
if ((checkpoint.manifest.analysisGeneration ?? 0) === deps.entry.analysisGeneration) {
|
|
1035
|
+
const config = await encodeIndexConfiguration(deps.entry);
|
|
1036
|
+
return archiveStream({ ...checkpoint, config, sourceIndexName: deps.indexName });
|
|
1037
|
+
}
|
|
1038
|
+
} catch (error) {
|
|
1039
|
+
checkpoint.release();
|
|
1040
|
+
throw error;
|
|
1041
|
+
}
|
|
1042
|
+
checkpoint.release();
|
|
1043
|
+
}
|
|
1044
|
+
throw new NarsilError(
|
|
1045
|
+
ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
|
|
1046
|
+
`The analysis of index "${deps.indexName}" changed while the engine wrote its snapshot, so try the snapshot again once the change has a durable checkpoint`,
|
|
1047
|
+
{ indexName: deps.indexName }
|
|
1048
|
+
);
|
|
1180
1049
|
}
|
|
1181
1050
|
function requireSnapshotSource(data) {
|
|
1182
1051
|
if (data instanceof Uint8Array) return data;
|
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import
|
|
1
|
+
import { NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-XVMOQQ34.mjs';
|
|
2
|
+
import './chunk-QHNNULPW.mjs';
|
|
3
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-6BSB4N6B.mjs';
|
|
4
|
+
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-5LYWZJE2.mjs';
|
|
5
|
+
import { decodeStoredDocument, openChecksummedEnvelope } from './chunk-ZAIZQOTM.mjs';
|
|
3
6
|
import './chunk-26A5JFT2.mjs';
|
|
4
|
-
import
|
|
5
|
-
import './chunk-AIIJDWMG.mjs';
|
|
7
|
+
import './chunk-4MFH226Q.mjs';
|
|
6
8
|
import './chunk-JPTQZWJQ.mjs';
|
|
7
|
-
import { EXPORT_CHUNK_BYTES } from './chunk-
|
|
8
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
9
|
+
import { EXPORT_CHUNK_BYTES } from './chunk-MZI3TEOW.mjs';
|
|
10
|
+
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
9
11
|
import { decode } from '@msgpack/msgpack';
|
|
10
12
|
|
|
11
13
|
// src/engine/export/json-line.ts
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
3
|
-
import {
|
|
2
|
+
import { C as ClusterNamespace } from './config-Brysl9NO.js';
|
|
3
|
+
import { E as ErrorDetail, i as TaskStore } from './tasks-DjGjK70Y.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Context handed to {@link OnRequestHook} for every inbound request, captured
|
|
@@ -256,14 +256,7 @@ interface NarsilServer {
|
|
|
256
256
|
*/
|
|
257
257
|
interface ErrorEnvelope {
|
|
258
258
|
/** This describes the failure. */
|
|
259
|
-
error:
|
|
260
|
-
/** This code comes from either {@link ServerErrorCodes} or {@link ErrorCodes}. */
|
|
261
|
-
code: string;
|
|
262
|
-
/** This describes the failure, for a log or a person. */
|
|
263
|
-
message: string;
|
|
264
|
-
/** These values are behind the failure, such as the field or limit involved. */
|
|
265
|
-
details?: Record<string, unknown>;
|
|
266
|
-
};
|
|
259
|
+
error: ErrorDetail;
|
|
267
260
|
}
|
|
268
261
|
|
|
269
262
|
/**
|
|
@@ -293,6 +286,13 @@ declare const TASK_FILTER_CAPABILITY = "tasks.filter";
|
|
|
293
286
|
* @public
|
|
294
287
|
*/
|
|
295
288
|
declare const REBUILD_ANALYSIS_CAPABILITY = "indexes.rebuildAnalysis";
|
|
289
|
+
/**
|
|
290
|
+
* Marks that `POST /indexes/{name}/_change-analysis` changes the language, the
|
|
291
|
+
* tokeniser, or the stop words of a live index.
|
|
292
|
+
*
|
|
293
|
+
* @public
|
|
294
|
+
*/
|
|
295
|
+
declare const CHANGE_ANALYSIS_CAPABILITY = "indexes.changeAnalysis";
|
|
296
296
|
/**
|
|
297
297
|
* Marks that `POST /indexes/{name}/_open` and `POST /indexes/{name}/_close`
|
|
298
298
|
* load and release an index's server memory.
|
|
@@ -334,4 +334,4 @@ declare const BM25_TUNING_CAPABILITY = "indexes.bm25";
|
|
|
334
334
|
*/
|
|
335
335
|
declare const SERVER_CAPABILITIES: readonly string[];
|
|
336
336
|
|
|
337
|
-
export { ASYNC_IMPORT_CAPABILITY as A, BM25_TUNING_CAPABILITY as B,
|
|
337
|
+
export { ASYNC_IMPORT_CAPABILITY as A, BM25_TUNING_CAPABILITY as B, CHANGE_ANALYSIS_CAPABILITY as C, DOCUMENT_EXPORT_CAPABILITY as D, type ErrorEnvelope as E, INDEX_LIFECYCLE_CAPABILITY as I, type NarsilServer as N, type OnRequestHook as O, REBUILD_ANALYSIS_CAPABILITY as R, SERVER_CAPABILITIES as S, TASK_CANCEL_CAPABILITY as T, SYNONYMS_CAPABILITY as a, TASK_FILTER_CAPABILITY as b, type ServerOptions as c, type CorsOptions as d, type RequestContext as e, type RequestDenial as f, type ServerLimits as g };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { nodeThreadId } from './chunk-W55X2BYN.mjs';
|
|
2
2
|
import { getFs, getPath } from './chunk-HLPRHOSO.mjs';
|
|
3
|
-
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-
|
|
3
|
+
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-CPCGEIOM.mjs';
|
|
4
4
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
5
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
5
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
6
6
|
import { openSync, close, readFileSync } from 'fs';
|
|
7
7
|
import { rename, readFile, writeFile, unlink, realpath, readdir, readlink } from 'fs/promises';
|
|
8
8
|
import { endianness } from 'os';
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-
|
|
2
|
-
import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes, createOsqScratch, osqQuantize } from './chunk-
|
|
1
|
+
import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-ICW45TTU.mjs';
|
|
2
|
+
import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes, createOsqScratch, osqQuantize } from './chunk-K3AQOZVM.mjs';
|
|
3
3
|
import { osqRecordBytes, writeRecord } from './chunk-37CD3OKS.mjs';
|
|
4
|
-
import { MAX_DOC_ID_LENGTH } from './chunk-
|
|
4
|
+
import { MAX_DOC_ID_LENGTH } from './chunk-EEBCDP36.mjs';
|
|
5
5
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
6
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
6
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
7
7
|
|
|
8
8
|
// src/vector/filter-ordinals.ts
|
|
9
9
|
var UNRESOLVED_ORDINAL = -1;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { sendRelayed, parseJson, validateQuery, rejectInvalid, refuseRegex, respondJson, withServerRerankWindow, respondError, loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedBodyBudget, createSharedRequestGate, registerServerRoutes, closeWhenTheProcessExits, statusLine, sendError, REQUEST_THREAD_MOVE_AFTER_MS, MAX_LONG_QUERIES_IN_FLIGHT, createDocumentReadHandlers, SERVER_CAPABILITIES, versionReport, serverRoutes, validateSuggest, SEARCH_CLOCK_SAMPLE_UNITS, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, encodeJson } from './chunk-
|
|
1
|
+
import { sendRelayed, parseJson, validateQuery, rejectInvalid, refuseRegex, respondJson, withServerRerankWindow, respondError, loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedBodyBudget, createSharedRequestGate, registerServerRoutes, closeWhenTheProcessExits, statusLine, sendError, REQUEST_THREAD_MOVE_AFTER_MS, MAX_LONG_QUERIES_IN_FLIGHT, createDocumentReadHandlers, SERVER_CAPABILITIES, versionReport, serverRoutes, validateSuggest, SEARCH_CLOCK_SAMPLE_UNITS, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, encodeJson } from './chunk-SBMAWTRG.mjs';
|
|
2
2
|
import { withRerankQueryText } from './chunk-KZ3NX2P7.mjs';
|
|
3
|
-
import { executeListDocuments, executeSuggest, executeQuery, lazyQueryBinding, executePreflight } from './chunk-
|
|
4
|
-
import { holdsQuotedText, resolveProjection, projectionKeepsField, setNestedValue } from './chunk-
|
|
5
|
-
import { extractVectorFieldsFromSchema } from './chunk-
|
|
6
|
-
import { ServerErrorCodes, ErrorCodes, NarsilError, isNarsilError } from './chunk-
|
|
3
|
+
import { executeListDocuments, executeSuggest, executeQuery, lazyQueryBinding, executePreflight } from './chunk-XMQNOD5V.mjs';
|
|
4
|
+
import { holdsQuotedText, resolveProjection, projectionKeepsField, setNestedValue } from './chunk-U4X4AWKL.mjs';
|
|
5
|
+
import { extractVectorFieldsFromSchema } from './chunk-UBXJQJII.mjs';
|
|
6
|
+
import { ServerErrorCodes, ErrorCodes, NarsilError, isNarsilError } from './chunk-HNHCNTVG.mjs';
|
|
7
7
|
|
|
8
8
|
// src/server/request-threads/messages.ts
|
|
9
9
|
function isLongQueryPortNote(message) {
|
|
@@ -273,7 +273,8 @@ var SearchMoved = class extends Error {
|
|
|
273
273
|
}
|
|
274
274
|
};
|
|
275
275
|
function moveClock(moveAfterMs) {
|
|
276
|
-
|
|
276
|
+
let moveAt = performance.now() + moveAfterMs;
|
|
277
|
+
let overranOnce = false;
|
|
277
278
|
let pending = 0;
|
|
278
279
|
const clock = {
|
|
279
280
|
tick(units) {
|
|
@@ -282,7 +283,11 @@ function moveClock(moveAfterMs) {
|
|
|
282
283
|
},
|
|
283
284
|
check() {
|
|
284
285
|
pending = 0;
|
|
285
|
-
|
|
286
|
+
const now = performance.now();
|
|
287
|
+
if (now <= moveAt) return;
|
|
288
|
+
if (overranOnce) throw new SearchMoved();
|
|
289
|
+
overranOnce = true;
|
|
290
|
+
moveAt = now + moveAfterMs;
|
|
286
291
|
}
|
|
287
292
|
};
|
|
288
293
|
return clock;
|