@delali/narsil 0.3.1-next.95 → 0.3.1-next.97
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
- package/dist/browser/{chunk-3U636ADD.mjs → chunk-CN4I4FZU.mjs} +2 -2
- package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
- package/dist/browser/{chunk-WKPJ4E73.mjs → chunk-LD55ZGU3.mjs} +2 -1
- package/dist/browser/{chunk-D3CC2NMZ.mjs → chunk-NAFT3K6X.mjs} +240 -135
- package/dist/browser/{chunk-LULHQXCK.mjs → chunk-NNG3EEB6.mjs} +5 -4
- package/dist/browser/{chunk-HARBSUWP.mjs → chunk-QWES44AW.mjs} +2 -197
- package/dist/browser/chunk-RELZRJNR.mjs +203 -0
- package/dist/browser/{chunk-FYFQJKFW.mjs → chunk-Z7QH2XHB.mjs} +40 -45
- package/dist/browser/{durability-wiring-3ZRKSPJD.mjs → durability-wiring-PY3L7DES.mjs} +7 -5
- package/dist/browser/{module-IEOQI5GI.mjs → module-PDXYHT75.mjs} +18 -10
- package/dist/browser/{restore-queue-RBULSOVX.mjs → restore-queue-DVIYJ7RU.mjs} +1 -1
- package/dist/browser/runtime-NYE5TB6O.mjs +557 -0
- package/dist/browser/{snapshot-Y3NGUJHX.mjs → snapshot-J57MZK6L.mjs} +115 -61
- package/dist/browser/{write-ZUBDNLCH.mjs → write-L7FAQA47.mjs} +6 -4
- package/dist/{capabilities-BvxvFzVA.d.ts → capabilities-BTTIs96E.d.ts} +19 -3
- package/dist/{chunk-GJZHU4FC.mjs → chunk-2WYMG3OZ.mjs} +4 -4
- package/dist/{chunk-7QONURHW.mjs → chunk-45AVKHMJ.mjs} +2 -2
- package/dist/{chunk-QM3VPX4Z.mjs → chunk-ASY5PHYI.mjs} +2 -2
- package/dist/{chunk-RDJKYKMQ.mjs → chunk-AY3AQ3UH.mjs} +1 -1
- package/dist/{chunk-54HYC5FT.mjs → chunk-BKB3XTMI.mjs} +6 -6
- package/dist/{chunk-MWT5WCJ5.mjs → chunk-COVCQ73F.mjs} +5 -5
- package/dist/{chunk-FGMGUDHX.mjs → chunk-E74YLAIG.mjs} +4 -1
- package/dist/chunk-FMWRCP2C.mjs +57 -0
- package/dist/{chunk-ES33L3HO.mjs → chunk-GGSH6KBQ.mjs} +1 -1
- package/dist/chunk-HJSN7UYG.mjs +10 -0
- package/dist/{chunk-ICL2AELD.mjs → chunk-HXJBSHXF.mjs} +40 -46
- package/dist/{chunk-ZHS44YKF.mjs → chunk-JFRNNBOM.mjs} +1 -1
- package/dist/{chunk-CHOGQSHK.mjs → chunk-K25LZA54.mjs} +39 -13
- package/dist/chunk-K7D4ZTSW.mjs +126 -0
- package/dist/{chunk-SYWGORQB.mjs → chunk-KV3AZFT3.mjs} +2 -0
- package/dist/chunk-KZ3NX2P7.mjs +21 -0
- package/dist/{chunk-J7SDAOTP.mjs → chunk-L3L5YFXC.mjs} +4 -4
- package/dist/{chunk-Q4UTU5ZE.mjs → chunk-LH4NZNJJ.mjs} +10 -10
- package/dist/{chunk-7KC56574.mjs → chunk-LU2CE5AE.mjs} +1 -1
- package/dist/{chunk-4ZJWRETC.mjs → chunk-LXPIXDUJ.mjs} +2 -2
- package/dist/{chunk-Q5DC6WAB.mjs → chunk-LXWDQLTW.mjs} +1 -1
- package/dist/{chunk-SBCEHIPL.mjs → chunk-MHOSCQCQ.mjs} +4 -4
- package/dist/{chunk-R36MIGBU.mjs → chunk-NKRBEGJZ.mjs} +169 -130
- package/dist/{chunk-L26LEDHO.mjs → chunk-NTRMFARG.mjs} +7 -7
- package/dist/chunk-NW4XDSP2.mjs +212 -0
- package/dist/{chunk-BB7MASRQ.mjs → chunk-OOEXLUGR.mjs} +1 -1
- package/dist/{chunk-3LYY62M6.mjs → chunk-OWNZYMRE.mjs} +10 -10
- package/dist/{chunk-HBVLF3HA.mjs → chunk-PPY2GSIB.mjs} +1 -1
- package/dist/{chunk-WLOF6HSP.mjs → chunk-R7EEKAJE.mjs} +12 -3
- package/dist/{chunk-2GZGBOQI.mjs → chunk-RONEOEON.mjs} +2 -2
- package/dist/{chunk-T2QN2FA4.mjs → chunk-UCDF35PU.mjs} +2 -2
- package/dist/{chunk-JFUE6F2Y.mjs → chunk-VWULPWRJ.mjs} +60 -21
- package/dist/{chunk-F52QGNZ2.mjs → chunk-WCJKUGHB.mjs} +1 -1
- package/dist/{chunk-RUIXODYU.mjs → chunk-YA7XLE2R.mjs} +32 -5
- package/dist/{chunk-MASE2LTJ.mjs → chunk-ZG477IPV.mjs} +7 -7
- package/dist/chunk-ZYNAMIYX.mjs +113 -0
- package/dist/{client-DKwterMb.d.ts → client-i9eLJrw5.d.ts} +3 -3
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -0
- package/dist/{config-6YBtxekG.d.ts → config-BGB_fDtq.d.ts} +2 -2
- package/dist/{config-DPphZfuM.d.ts → config-P__Diiy4.d.ts} +25 -3
- package/dist/direct-executor-CXZ66ACP.mjs +22 -0
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -12
- package/dist/distribution.mjs +177 -54
- package/dist/{durability-wiring-FRPV6JLZ.mjs → durability-wiring-QTHOGFMZ.mjs} +26 -25
- package/dist/embeddings/openai.d.ts +2 -2
- package/dist/embeddings/openai.mjs +22 -133
- package/dist/{engine-BGt9lI1V.d.ts → engine-BGCEm9E_.d.ts} +50 -20
- package/dist/{index-ONAEaqtx.d.ts → index-lcdGLyJH.d.ts} +1 -1
- package/dist/index.browser.mjs +4 -2
- package/dist/index.d.ts +8 -8
- package/dist/index.mjs +19 -18
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{memory-COdoGthn.d.ts → memory-CUEDwr-F.d.ts} +70 -2
- package/dist/{module-IUTNXWMV.mjs → module-3NONKBZX.mjs} +25 -18
- package/dist/partial-words.mjs +6 -6
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -22
- package/dist/react.d.ts +8 -8
- package/dist/react.mjs +2 -0
- package/dist/rerank-NWTKK636.mjs +60 -0
- package/dist/rerankers/http.d.ts +50 -0
- package/dist/rerankers/http.mjs +104 -0
- package/dist/{restore-queue-I5SI6AUQ.mjs → restore-queue-JPRBUYII.mjs} +1 -1
- package/dist/runtime-ZKYZFJJC.mjs +177 -0
- package/dist/{schema-DttzM-ln.d.ts → schema-4W_7tk1f.d.ts} +2 -2
- package/dist/server.d.ts +11 -11
- package/dist/server.mjs +63 -22
- package/dist/{snapshot-YC6TS4OP.mjs → snapshot-HOGXNO5W.mjs} +138 -83
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-BJmamiC4.d.ts → tasks-HG3rlxx6.d.ts} +1 -1
- package/dist/{thread-U7XYJWGG.mjs → thread-B657LQXK.mjs} +49 -30
- package/dist/vector/search-worker.d.ts +6 -2
- package/dist/vector/search-worker.mjs +9 -7
- package/dist/workers/entry.mjs +5 -5
- package/dist/{write-M2OALSA4.mjs → write-XLIGTXBU.mjs} +21 -20
- package/package.json +11 -7
- package/dist/direct-executor-5KJGVQHM.mjs +0 -22
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import { vectorsPerFile } from './chunk-
|
|
2
|
-
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig,
|
|
5
|
-
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-
|
|
6
|
-
import {
|
|
1
|
+
import { vectorsPerFile } from './chunk-Z7QH2XHB.mjs';
|
|
2
|
+
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-NNG3EEB6.mjs';
|
|
3
|
+
import './chunk-CN4I4FZU.mjs';
|
|
4
|
+
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-NAFT3K6X.mjs';
|
|
5
|
+
import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-LD55ZGU3.mjs';
|
|
6
|
+
import { MAX_FILE_BYTES } from './chunk-QWES44AW.mjs';
|
|
7
|
+
import './chunk-JPTQZWJQ.mjs';
|
|
8
|
+
import { NarsilError, ErrorCodes } from './chunk-RELZRJNR.mjs';
|
|
7
9
|
import { decode } from '@msgpack/msgpack';
|
|
8
10
|
|
|
9
11
|
// src/persistence/archive/memory-store.ts
|
|
@@ -499,43 +501,84 @@ async function validateStagedCheckpoint(store, manifest) {
|
|
|
499
501
|
}
|
|
500
502
|
|
|
501
503
|
// src/engine/snapshot/constants.ts
|
|
502
|
-
var
|
|
503
|
-
var
|
|
504
|
-
var
|
|
505
|
-
var
|
|
506
|
-
var
|
|
507
|
-
var
|
|
508
|
-
var
|
|
509
|
-
var
|
|
510
|
-
var
|
|
511
|
-
var
|
|
512
|
-
var
|
|
513
|
-
var
|
|
504
|
+
var RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE = 0.2;
|
|
505
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 7.86;
|
|
506
|
+
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR = 264;
|
|
507
|
+
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.24;
|
|
508
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 738;
|
|
509
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.2;
|
|
510
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE = 4.06;
|
|
511
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 14.5;
|
|
512
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR = 1482;
|
|
513
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 9.29;
|
|
514
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2141;
|
|
515
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 24.15;
|
|
516
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE = 0.16;
|
|
517
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 6.16;
|
|
518
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR = 204;
|
|
519
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.17;
|
|
520
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 695;
|
|
521
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.04;
|
|
522
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE = 2.14;
|
|
523
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 5.05;
|
|
524
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR = 1080;
|
|
525
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 0.72;
|
|
526
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 1744;
|
|
527
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 20.05;
|
|
528
|
+
var RESTORE_ESTIMATE_MARGIN = 1.25;
|
|
514
529
|
|
|
515
530
|
// src/engine/snapshot/memory-budget.ts
|
|
516
531
|
var MEGABYTE_BYTES = 1048576;
|
|
517
532
|
var ARCHIVE_RATES = {
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
533
|
+
worst: {
|
|
534
|
+
heap: {
|
|
535
|
+
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
|
|
536
|
+
perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
|
|
537
|
+
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
538
|
+
},
|
|
539
|
+
external: {
|
|
540
|
+
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
541
|
+
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
|
|
542
|
+
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
543
|
+
}
|
|
522
544
|
},
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
545
|
+
floor: {
|
|
546
|
+
heap: {
|
|
547
|
+
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
|
|
548
|
+
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
|
|
549
|
+
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
550
|
+
},
|
|
551
|
+
external: {
|
|
552
|
+
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
553
|
+
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
|
|
554
|
+
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
555
|
+
}
|
|
527
556
|
}
|
|
528
557
|
};
|
|
529
558
|
var EARLIER_SNAPSHOT_RATES = {
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
559
|
+
worst: {
|
|
560
|
+
heap: {
|
|
561
|
+
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
562
|
+
perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
563
|
+
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
564
|
+
},
|
|
565
|
+
external: {
|
|
566
|
+
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
567
|
+
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
568
|
+
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
569
|
+
}
|
|
534
570
|
},
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
571
|
+
floor: {
|
|
572
|
+
heap: {
|
|
573
|
+
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
574
|
+
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
575
|
+
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
576
|
+
},
|
|
577
|
+
external: {
|
|
578
|
+
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
579
|
+
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
580
|
+
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
581
|
+
}
|
|
539
582
|
}
|
|
540
583
|
};
|
|
541
584
|
function bytesAt(rate, storedBytes, vectors) {
|
|
@@ -543,10 +586,12 @@ function bytesAt(rate, storedBytes, vectors) {
|
|
|
543
586
|
for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
|
|
544
587
|
return bytes;
|
|
545
588
|
}
|
|
546
|
-
function footprintAt(rates, storedBytes, vectors) {
|
|
589
|
+
function footprintAt(rates, replacesIndex, storedBytes, vectors) {
|
|
590
|
+
const chosen = replacesIndex ? rates.worst : rates.floor;
|
|
591
|
+
const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
|
|
547
592
|
return {
|
|
548
|
-
heapBytes: bytesAt(
|
|
549
|
-
externalBytes: bytesAt(
|
|
593
|
+
heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
|
|
594
|
+
externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
|
|
550
595
|
};
|
|
551
596
|
}
|
|
552
597
|
function deadCountOf(dead) {
|
|
@@ -557,7 +602,7 @@ function deadCountOf(dead) {
|
|
|
557
602
|
}
|
|
558
603
|
return count;
|
|
559
604
|
}
|
|
560
|
-
function archiveFootprint(prepared, manifest) {
|
|
605
|
+
function archiveFootprint(prepared, manifest, replacesIndex) {
|
|
561
606
|
let segmentBytes = 0;
|
|
562
607
|
for (const partition of manifest.partitions) {
|
|
563
608
|
for (const segment of partition.segments) segmentBytes += segment.bytes;
|
|
@@ -568,36 +613,45 @@ function archiveFootprint(prepared, manifest) {
|
|
|
568
613
|
const dimension = dimensions.get(field.fieldPath) ?? 0;
|
|
569
614
|
for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
|
|
570
615
|
}
|
|
571
|
-
return footprintAt(ARCHIVE_RATES, segmentBytes, vectors);
|
|
616
|
+
return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
|
|
572
617
|
}
|
|
573
|
-
function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts) {
|
|
618
|
+
function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
|
|
574
619
|
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
575
620
|
const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
|
|
576
|
-
return footprintAt(EARLIER_SNAPSHOT_RATES, partitionBytes, vectors);
|
|
621
|
+
return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
|
|
577
622
|
}
|
|
578
623
|
function inMegabytes(bytes) {
|
|
579
624
|
return Math.ceil(bytes / MEGABYTE_BYTES);
|
|
580
625
|
}
|
|
581
|
-
function
|
|
582
|
-
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
583
|
-
throw new NarsilError(
|
|
584
|
-
ErrorCodes.RESTORE_MEMORY_EXCEEDED,
|
|
585
|
-
`The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again`,
|
|
586
|
-
{ indexName, budget, neededBytes, availableBytes }
|
|
587
|
-
);
|
|
588
|
-
}
|
|
589
|
-
function requireRestoreMemory(indexName, footprint) {
|
|
626
|
+
function shortfallOf(heapBytes, processBytes, countInUse) {
|
|
590
627
|
const heap = readHeapStatistics();
|
|
591
628
|
if (heap !== null) {
|
|
592
|
-
const
|
|
593
|
-
|
|
629
|
+
const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
|
|
630
|
+
const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
|
|
631
|
+
if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
|
|
594
632
|
}
|
|
595
633
|
const resident = readResidentMemory();
|
|
596
634
|
if (resident !== null) {
|
|
597
|
-
const
|
|
598
|
-
|
|
599
|
-
if (neededBytes > availableBytes) refuse(indexName, "process", neededBytes, availableBytes);
|
|
635
|
+
const availableBytes = countInUse ? resident.limitBytes - resident.residentBytes : resident.limitBytes;
|
|
636
|
+
if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
|
|
600
637
|
}
|
|
638
|
+
return null;
|
|
639
|
+
}
|
|
640
|
+
function refuse(indexName, replacesIndex, shortfall) {
|
|
641
|
+
const { budget, neededBytes, availableBytes } = shortfall;
|
|
642
|
+
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
643
|
+
const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
|
|
644
|
+
throw new NarsilError(
|
|
645
|
+
ErrorCodes.RESTORE_MEMORY_EXCEEDED,
|
|
646
|
+
`The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
|
|
647
|
+
{ indexName, budget, neededBytes, availableBytes }
|
|
648
|
+
);
|
|
649
|
+
}
|
|
650
|
+
function requireRestoreMemory(indexName, footprint, replacesIndex) {
|
|
651
|
+
const processBytes = footprint.heapBytes + footprint.externalBytes;
|
|
652
|
+
if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
|
|
653
|
+
const shortfall = shortfallOf(footprint.heapBytes, processBytes, replacesIndex);
|
|
654
|
+
if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
|
|
601
655
|
}
|
|
602
656
|
|
|
603
657
|
// src/workers/detached-index.ts
|
|
@@ -898,9 +952,8 @@ async function stageArchive(indexName, reader, staging, deps) {
|
|
|
898
952
|
manifest: async (manifest2, sourceName) => {
|
|
899
953
|
if (staged.prepared === null) missingPart("configuration");
|
|
900
954
|
requireSameSchema(staged.prepared, manifest2);
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
}
|
|
955
|
+
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
956
|
+
requireRestoreMemory(indexName, archiveFootprint(staged.prepared, manifest2, replacesIndex), replacesIndex);
|
|
904
957
|
staged.manifest = renamed(manifest2, sourceName, indexName);
|
|
905
958
|
},
|
|
906
959
|
file: async (entryName, _ref, bytes) => {
|
|
@@ -990,7 +1043,7 @@ function vectorsOfSnapshot(envelope) {
|
|
|
990
1043
|
}
|
|
991
1044
|
return byDocument;
|
|
992
1045
|
}
|
|
993
|
-
function footprintOf(envelope, prepared) {
|
|
1046
|
+
function footprintOf(envelope, prepared, replacesIndex) {
|
|
994
1047
|
let partitionBytes = 0;
|
|
995
1048
|
for (const payload of envelope.partitions ?? []) partitionBytes += payload.byteLength;
|
|
996
1049
|
const vectorCounts = /* @__PURE__ */ new Map();
|
|
@@ -999,7 +1052,7 @@ function footprintOf(envelope, prepared) {
|
|
|
999
1052
|
for (const part of decodeVectorIndexParts(rawParts)) count += part.docIds.length;
|
|
1000
1053
|
vectorCounts.set(fieldPath, count);
|
|
1001
1054
|
}
|
|
1002
|
-
return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts);
|
|
1055
|
+
return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex);
|
|
1003
1056
|
}
|
|
1004
1057
|
function documentsOfPartition(envelope, payload) {
|
|
1005
1058
|
return envelope.version === 2 ? documentsOfPayloadV2(payload).documents : deserializePayloadV1(payload).documents;
|
|
@@ -1059,7 +1112,8 @@ async function buildRestoredIndex(indexName, envelope, prepared, staging, deps)
|
|
|
1059
1112
|
async function restoreEarlierSnapshot(indexName, data, deps) {
|
|
1060
1113
|
const envelope = await decodeEarlierSnapshot(data);
|
|
1061
1114
|
const prepared = prepareRestoredIndex(envelope, deps);
|
|
1062
|
-
|
|
1115
|
+
const replacesIndex = deps.indexRegistry.has(indexName);
|
|
1116
|
+
requireRestoreMemory(indexName, footprintOf(envelope, prepared, replacesIndex), replacesIndex);
|
|
1063
1117
|
const analysedNow = { ...prepared, storedRevision: prepared.language.revision };
|
|
1064
1118
|
const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
|
|
1065
1119
|
try {
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-
|
|
2
|
-
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-
|
|
3
|
-
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-
|
|
4
|
-
import
|
|
1
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-CN4I4FZU.mjs';
|
|
2
|
+
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-NAFT3K6X.mjs';
|
|
3
|
+
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-LD55ZGU3.mjs';
|
|
4
|
+
import './chunk-QWES44AW.mjs';
|
|
5
|
+
import './chunk-JPTQZWJQ.mjs';
|
|
6
|
+
import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-RELZRJNR.mjs';
|
|
5
7
|
import { decode } from '@msgpack/msgpack';
|
|
6
8
|
|
|
7
9
|
// src/engine/export/json-line.ts
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { E as EmbeddingAdapter } from './adapters-
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
3
|
-
import { h as TaskStore } from './tasks-
|
|
1
|
+
import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
|
|
2
|
+
import { C as ClusterNamespace } from './config-BGB_fDtq.js';
|
|
3
|
+
import { h as TaskStore } from './tasks-HG3rlxx6.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Context handed to {@link OnRequestHook} for every inbound request, captured
|
|
@@ -89,6 +89,15 @@ interface ServerLimits {
|
|
|
89
89
|
* request cannot ask for an unbounded result set. Excess → 400. Defaults to
|
|
90
90
|
* 10000, matching the cluster query result window. */
|
|
91
91
|
maxResultWindow?: number;
|
|
92
|
+
/**
|
|
93
|
+
* The server answers 400 for a search whose `rerank.window` is above this
|
|
94
|
+
* number, so that one remote request can send at most this many documents
|
|
95
|
+
* to a rerank model. A search that leaves `rerank.window` out reranks at
|
|
96
|
+
* most this many hits. Set it from 1 to the engine's `maxRerankWindow`,
|
|
97
|
+
* which the server checks when it starts. It defaults to 100, or to the
|
|
98
|
+
* engine's `maxRerankWindow` where that is lower.
|
|
99
|
+
*/
|
|
100
|
+
maxRerankWindow?: number;
|
|
92
101
|
/** Ceiling for the number of document ids in one multi-get request, so one
|
|
93
102
|
* request cannot pull an unbounded number of documents. Excess → 400.
|
|
94
103
|
* Defaults to 10000. */
|
|
@@ -140,6 +149,13 @@ interface ServerOptions {
|
|
|
140
149
|
* needs query-time or ingest-time embedding names a server-registered adapter.
|
|
141
150
|
*/
|
|
142
151
|
embeddingAdapters?: Record<string, EmbeddingAdapter>;
|
|
152
|
+
/**
|
|
153
|
+
* The server registers these rerank adapters on its engine under their
|
|
154
|
+
* keys, so that a JSON query can give a key in `rerank.adapter`. A rerank
|
|
155
|
+
* adapter is an object with methods, which JSON cannot carry, so a query
|
|
156
|
+
* reaches it by its key.
|
|
157
|
+
*/
|
|
158
|
+
rerankAdapters?: Record<string, RerankAdapter>;
|
|
143
159
|
/** These ceilings apply before the server hands any work to the engine. */
|
|
144
160
|
limits?: ServerLimits;
|
|
145
161
|
/**
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-P36MUFEJ.mjs';
|
|
2
|
-
import { readSharedConfigFields, restoredEmbedding, restoredPartitionConfig } from './chunk-
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
2
|
+
import { readSharedConfigFields, restoredEmbedding, restoredPartitionConfig } from './chunk-R7EEKAJE.mjs';
|
|
3
|
+
import { generateId, getLanguage } from './chunk-JFRNNBOM.mjs';
|
|
4
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-LXWDQLTW.mjs';
|
|
5
5
|
import { decodeSynonymRules } from './chunk-7X5JST5L.mjs';
|
|
6
6
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
7
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
7
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
8
8
|
import { encode, decode } from '@msgpack/msgpack';
|
|
9
9
|
|
|
10
10
|
// src/distribution/cluster/constants.ts
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { nodeThreadId } from './chunk-C7SLA6CM.mjs';
|
|
2
|
-
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-
|
|
2
|
+
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-LH4NZNJJ.mjs';
|
|
3
3
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
5
5
|
import { openSync, close, readFileSync } from 'fs';
|
|
6
6
|
import { rename, readFile, writeFile, unlink, realpath, readdir, readlink } from 'fs/promises';
|
|
7
7
|
import { endianness } from 'os';
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { MAX_FILTER_ARRAY_SIZE } from './chunk-WQBCBKY3.mjs';
|
|
2
2
|
import { tabulatedLevenshtein } from './chunk-GES7U2PM.mjs';
|
|
3
|
-
import { parsedTypeOf } from './chunk-
|
|
3
|
+
import { parsedTypeOf } from './chunk-OOEXLUGR.mjs';
|
|
4
4
|
import { codePointCount, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
5
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
5
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
6
6
|
|
|
7
7
|
// src/core/bitset.ts
|
|
8
8
|
function createBitSet(capacity) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { SCRATCH_SLOTS_PER_THREAD_POOL } from './chunk-QY7GC2MW.mjs';
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
3
3
|
|
|
4
4
|
// src/serialization/constants.ts
|
|
5
5
|
var CHECKSUM_YIELD_CHUNK_BYTES = 4 * 1024 * 1024;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { decodeSegmentManifest, loadPartitionSegments, loadVectorField } from './chunk-
|
|
2
|
-
import { manifestKey, segmentsPrefix } from './chunk-
|
|
3
|
-
import { readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader } from './chunk-
|
|
4
|
-
import { VALID_HNSW_METRICS } from './chunk-
|
|
5
|
-
import { requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-
|
|
1
|
+
import { decodeSegmentManifest, loadPartitionSegments, loadVectorField } from './chunk-ZG477IPV.mjs';
|
|
2
|
+
import { manifestKey, segmentsPrefix } from './chunk-LH4NZNJJ.mjs';
|
|
3
|
+
import { readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader } from './chunk-VWULPWRJ.mjs';
|
|
4
|
+
import { VALID_HNSW_METRICS } from './chunk-AY3AQ3UH.mjs';
|
|
5
|
+
import { requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-LXPIXDUJ.mjs';
|
|
6
6
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
7
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
7
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
8
8
|
import { decode, encode } from '@msgpack/msgpack';
|
|
9
9
|
|
|
10
10
|
// src/serialization/stored-vector-values.ts
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-
|
|
2
|
-
import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes } from './chunk-
|
|
1
|
+
import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-WCJKUGHB.mjs';
|
|
2
|
+
import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes } from './chunk-HXJBSHXF.mjs';
|
|
3
3
|
import { osqRecordBytes } from './chunk-37CD3OKS.mjs';
|
|
4
|
-
import { MAX_FILE_BYTES } from './chunk-
|
|
5
|
-
import { MAX_DOC_ID_LENGTH } from './chunk-
|
|
6
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
4
|
+
import { MAX_FILE_BYTES } from './chunk-AY3AQ3UH.mjs';
|
|
5
|
+
import { MAX_DOC_ID_LENGTH } from './chunk-LXWDQLTW.mjs';
|
|
6
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
7
7
|
|
|
8
8
|
// src/vector/filter-ordinals.ts
|
|
9
9
|
var UNRESOLVED_ORDINAL = -1;
|
|
@@ -17,5 +17,8 @@ var MIN_RANK_CONSTANT = 1;
|
|
|
17
17
|
var DEFAULT_LINEAR_ALPHA = 0.5;
|
|
18
18
|
var MIN_LINEAR_ALPHA = 0;
|
|
19
19
|
var MAX_LINEAR_ALPHA = 1;
|
|
20
|
+
var MIN_HYBRID_DEPTH = 1;
|
|
21
|
+
var MAX_HYBRID_DEPTH = 1e4;
|
|
22
|
+
var HYBRID_RESCORE_WINDOW = 10;
|
|
20
23
|
|
|
21
|
-
export { DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PER_GROUP, DEFAULT_PAGE_SIZE, DEFAULT_RANK_CONSTANT, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_LINEAR_ALPHA, MAX_PREFIX_LENGTH, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_TOLERANCE, MIN_LINEAR_ALPHA, MIN_RANK_CONSTANT, PARTIAL_WORDS_ADDED_PER_QUERY, PARTIAL_WORD_MIN_CODE_POINTS, PREFIX_MAX_EXPANSIONS, RESULT_WINDOW };
|
|
24
|
+
export { DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PER_GROUP, DEFAULT_PAGE_SIZE, DEFAULT_RANK_CONSTANT, HYBRID_RESCORE_WINDOW, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_HYBRID_DEPTH, MAX_LINEAR_ALPHA, MAX_PREFIX_LENGTH, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_TOLERANCE, MIN_HYBRID_DEPTH, MIN_LINEAR_ALPHA, MIN_RANK_CONSTANT, PARTIAL_WORDS_ADDED_PER_QUERY, PARTIAL_WORD_MIN_CODE_POINTS, PREFIX_MAX_EXPANSIONS, RESULT_WINDOW };
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH } from './chunk-E74YLAIG.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
3
|
+
|
|
4
|
+
// src/search/hybrid-depth.ts
|
|
5
|
+
function fixedHybridDepthOf(hybrid) {
|
|
6
|
+
const depth = hybrid?.depth;
|
|
7
|
+
if (depth === void 0 || depth === null) return void 0;
|
|
8
|
+
if (!Number.isInteger(depth) || depth < MIN_HYBRID_DEPTH || depth > MAX_HYBRID_DEPTH) {
|
|
9
|
+
throw new NarsilError(
|
|
10
|
+
ErrorCodes.CONFIG_INVALID,
|
|
11
|
+
`hybrid.depth must be a whole number from ${MIN_HYBRID_DEPTH} to ${MAX_HYBRID_DEPTH}`,
|
|
12
|
+
{ depth }
|
|
13
|
+
);
|
|
14
|
+
}
|
|
15
|
+
return depth;
|
|
16
|
+
}
|
|
17
|
+
function requireWithinHybridDepth(cursorDepth, offset, depth) {
|
|
18
|
+
const start = cursorDepth + offset;
|
|
19
|
+
if (start < depth) return;
|
|
20
|
+
throw new NarsilError(
|
|
21
|
+
ErrorCodes.SEARCH_RESULT_WINDOW_EXCEEDED,
|
|
22
|
+
`A hybrid query pages through the first ${depth} fused results, and this page starts at ${start}. Raise hybrid.depth to page further`,
|
|
23
|
+
{ offset, cursorDepth, depth }
|
|
24
|
+
);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// src/search/position-page.ts
|
|
28
|
+
function lastOrganic(page, pinnedIds) {
|
|
29
|
+
for (let i = page.length - 1; i >= 0; i--) {
|
|
30
|
+
if (pinnedIds === void 0 || !pinnedIds.has(page[i].id)) return page[i];
|
|
31
|
+
}
|
|
32
|
+
return void 0;
|
|
33
|
+
}
|
|
34
|
+
function positionOf(organic, id) {
|
|
35
|
+
for (let i = 0; i < organic.length; i++) {
|
|
36
|
+
if (organic[i].id === id) return i;
|
|
37
|
+
}
|
|
38
|
+
return -1;
|
|
39
|
+
}
|
|
40
|
+
function pageByPosition(placed, organic, { start, limit, pinnedIds, binding, encodeCursor }) {
|
|
41
|
+
if (limit <= 0) return { page: [] };
|
|
42
|
+
const page = placed.slice(start, start + limit);
|
|
43
|
+
const last = lastOrganic(page, pinnedIds);
|
|
44
|
+
if (placed.length <= start + limit || last === void 0) return { page };
|
|
45
|
+
const depth = pinnedIds === void 0 ? start + page.lastIndexOf(last) + 1 : positionOf(organic, last.id) + 1;
|
|
46
|
+
const cursor = encodeCursor({
|
|
47
|
+
anchor: last.id,
|
|
48
|
+
score: last.score ?? null,
|
|
49
|
+
sortKey: null,
|
|
50
|
+
sortSignature: null,
|
|
51
|
+
depth,
|
|
52
|
+
binding
|
|
53
|
+
});
|
|
54
|
+
return { page, cursor };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export { fixedHybridDepthOf, pageByPosition, requireWithinHybridDepth };
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// src/rerank/constants.ts
|
|
2
|
+
var DEFAULT_RERANK_TIMEOUT_MS = 3e4;
|
|
3
|
+
var MIN_RERANK_TIMEOUT_MS = 1;
|
|
4
|
+
var MAX_RERANK_TIMEOUT_MS = 2147483647;
|
|
5
|
+
var DEFAULT_MAX_RERANK_WINDOW = 100;
|
|
6
|
+
var MIN_RERANK_WINDOW = 1;
|
|
7
|
+
var MAX_RERANK_WINDOW = 1e3;
|
|
8
|
+
var DEFAULT_RERANK_WINDOW = 100;
|
|
9
|
+
|
|
10
|
+
export { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS, DEFAULT_RERANK_WINDOW, MAX_RERANK_TIMEOUT_MS, MAX_RERANK_WINDOW, MIN_RERANK_TIMEOUT_MS, MIN_RERANK_WINDOW };
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { readTrailer, unpackLevels, stageQueryLevels, osqCodeBytes, osqStagedQueryBytes, osqRecordBytes, writeRecord } from './chunk-37CD3OKS.mjs';
|
|
2
1
|
import { stopUsingTheNativeSearchCore, noteFallback, nativeSearchCoreIsEnabled } from './chunk-DJHHF2L5.mjs';
|
|
2
|
+
import { readTrailer, unpackLevels, stageQueryLevels, osqCodeBytes, osqStagedQueryBytes, osqRecordBytes, writeRecord } from './chunk-37CD3OKS.mjs';
|
|
3
3
|
import { markSimdUnavailable } from './chunk-KOKM4PQN.mjs';
|
|
4
4
|
import { MAX_WASM_PAGES, WASM_PAGE_BYTES, VECTOR_SCRATCH_SLOTS, UNPLACED_FLAG_CHUNK_ORDINALS, VECTOR_BLOCK_INITIAL_SLOTS, MAX_VECTOR_ORDINALS, VECTOR_STORE_INITIAL_CAPACITY, LOCK_SPIN_ITERATIONS, LOCK_WAIT_SLICE_MS, OSQ_STEPS, OSQ_GRID, OSQ_QUERY_BITS, NATIVE_ATTACH_RETRY_MS, GRAPH_LOCK_WAIT_SLICE_MS, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ_LAMBDA, MAX_DOC_ID_TABLE_BYTES } from './chunk-QY7GC2MW.mjs';
|
|
5
|
-
import { loadNativeCore } from './chunk-
|
|
5
|
+
import { loadNativeCore } from './chunk-GGSH6KBQ.mjs';
|
|
6
6
|
import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, PLACEMENT_ROOM_REQUESTS, MAX_LAYER_CAP, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, DEFAULT_EF_SEARCH } from './chunk-KEHK2O6P.mjs';
|
|
7
7
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
8
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
8
|
+
import { NarsilError, ErrorCodes } from './chunk-KV3AZFT3.mjs';
|
|
9
9
|
|
|
10
10
|
// src/vector/shared-buffers/growable.ts
|
|
11
11
|
function sharedMemoryAvailable() {
|
|
@@ -2469,11 +2469,12 @@ function traverse(state, query, k, searchMetric, options) {
|
|
|
2469
2469
|
const calibrated = activeQuantizer(state);
|
|
2470
2470
|
const quantizer = calibrated === void 0 || calibrated.size === 0 ? void 0 : calibrated;
|
|
2471
2471
|
const useQuantized = quantizer !== void 0;
|
|
2472
|
-
|
|
2472
|
+
const window = options.rescoreWindow ?? k;
|
|
2473
|
+
let depth = window;
|
|
2473
2474
|
if (quantizer !== void 0) {
|
|
2474
|
-
depth = options.oversample === void 0 ? defaultRescoreDepth(quantizer.bits, state.dimension,
|
|
2475
|
+
depth = options.oversample === void 0 ? defaultRescoreDepth(quantizer.bits, state.dimension, window) : Math.ceil(window * options.oversample);
|
|
2475
2476
|
}
|
|
2476
|
-
const ef = Math.max(options.efSearch ?? defaultEfSearch(state.dimension,
|
|
2477
|
+
const ef = Math.max(options.efSearch ?? defaultEfSearch(state.dimension, window), depth);
|
|
2477
2478
|
const filter = options.filter;
|
|
2478
2479
|
const workspace = state.workspace;
|
|
2479
2480
|
const candidates = workspace.traversal;
|
|
@@ -2560,55 +2561,48 @@ function collectHits(state, query, k, searchMetric, minSimilarity, options) {
|
|
|
2560
2561
|
if (entryPointOf(state) === -1 || liveSize === 0) {
|
|
2561
2562
|
return [];
|
|
2562
2563
|
}
|
|
2563
|
-
const
|
|
2564
|
-
|
|
2565
|
-
if (native === null) {
|
|
2566
|
-
return rescoreWithFullPrecision(
|
|
2567
|
-
state,
|
|
2568
|
-
candidates,
|
|
2569
|
-
query,
|
|
2570
|
-
qMag,
|
|
2571
|
-
arenaQuery,
|
|
2572
|
-
depth,
|
|
2573
|
-
searchMetric,
|
|
2574
|
-
minSimilarity,
|
|
2575
|
-
options.filter
|
|
2576
|
-
);
|
|
2577
|
-
}
|
|
2578
|
-
const rescored = nativeRescoredHits(
|
|
2579
|
-
native.store,
|
|
2580
|
-
candidates,
|
|
2581
|
-
query,
|
|
2582
|
-
depth,
|
|
2583
|
-
searchMetric,
|
|
2584
|
-
minSimilarity,
|
|
2585
|
-
options.filter
|
|
2586
|
-
);
|
|
2587
|
-
if (rescored !== null) return rescored;
|
|
2588
|
-
const arena = state.store.prepareQueryArena(query);
|
|
2589
|
-
return rescoreWithFullPrecision(
|
|
2590
|
-
state,
|
|
2591
|
-
candidates,
|
|
2592
|
-
query,
|
|
2593
|
-
arena ? arena.magnitude : magnitude(query),
|
|
2594
|
-
arena,
|
|
2595
|
-
depth,
|
|
2596
|
-
searchMetric,
|
|
2597
|
-
minSimilarity,
|
|
2598
|
-
options.filter
|
|
2599
|
-
);
|
|
2600
|
-
}
|
|
2601
|
-
const hits = [];
|
|
2564
|
+
const traversal = traverse(state, query, k, searchMetric, options);
|
|
2565
|
+
const { candidates } = traversal;
|
|
2602
2566
|
const filter = options.filter;
|
|
2567
|
+
if (traversal.depth === 0) return estimatedHits(candidates, 0, searchMetric, minSimilarity, filter);
|
|
2568
|
+
const rescored = rescoredHits(state, traversal, query, traversal.depth, searchMetric, minSimilarity, filter);
|
|
2569
|
+
if (options.rescoreWindow === void 0 || minSimilarity !== Number.NEGATIVE_INFINITY) return rescored;
|
|
2570
|
+
const rest = endOfRescored(candidates, traversal.depth, filter);
|
|
2571
|
+
return rescored.concat(estimatedHits(candidates, rest, searchMetric, minSimilarity, filter));
|
|
2572
|
+
}
|
|
2573
|
+
function endOfRescored(candidates, depth, filter) {
|
|
2574
|
+
if (filter === void 0) return Math.min(depth, candidates.size);
|
|
2575
|
+
let taken = 0;
|
|
2603
2576
|
for (let i = 0; i < candidates.size; i++) {
|
|
2577
|
+
if (!ordinalFilterHas(filter, candidates.ords[i])) continue;
|
|
2578
|
+
taken += 1;
|
|
2579
|
+
if (taken === depth) return i + 1;
|
|
2580
|
+
}
|
|
2581
|
+
return candidates.size;
|
|
2582
|
+
}
|
|
2583
|
+
function estimatedHits(candidates, from, metric, minSimilarity, filter) {
|
|
2584
|
+
const hits = [];
|
|
2585
|
+
for (let i = from; i < candidates.size; i++) {
|
|
2604
2586
|
const ord = candidates.ords[i];
|
|
2605
2587
|
if (filter && !ordinalFilterHas(filter, ord)) continue;
|
|
2606
|
-
const score = toScore(candidates.distances[i],
|
|
2588
|
+
const score = toScore(candidates.distances[i], metric);
|
|
2607
2589
|
if (score < minSimilarity) continue;
|
|
2608
2590
|
hits.push({ ord, score });
|
|
2609
2591
|
}
|
|
2610
2592
|
return hits;
|
|
2611
2593
|
}
|
|
2594
|
+
function rescoredHits(state, traversal, query, depth, metric, minSimilarity, filter) {
|
|
2595
|
+
const { candidates, native } = traversal;
|
|
2596
|
+
if (native !== null) {
|
|
2597
|
+
const rescored = nativeRescoredHits(native.store, candidates, query, depth, metric, minSimilarity, filter);
|
|
2598
|
+
if (rescored !== null) return rescored;
|
|
2599
|
+
const arena = state.store.prepareQueryArena(query);
|
|
2600
|
+
const qMag2 = arena ? arena.magnitude : magnitude(query);
|
|
2601
|
+
return rescoreWithFullPrecision(state, candidates, query, qMag2, arena, depth, metric, minSimilarity, filter);
|
|
2602
|
+
}
|
|
2603
|
+
const { qMag, arenaQuery } = traversal;
|
|
2604
|
+
return rescoreWithFullPrecision(state, candidates, query, qMag, arenaQuery, depth, metric, minSimilarity, filter);
|
|
2605
|
+
}
|
|
2612
2606
|
function rescoreWithFullPrecision(state, candidates, query, qMag, arenaQuery, depth, metric, minSimilarity, filter) {
|
|
2613
2607
|
noteFallback("score");
|
|
2614
2608
|
const rescored = [];
|