@delali/narsil 0.3.1-next.151 → 0.3.1-next.153

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/dist/{analysis-change-ZCJNYBJM.mjs → analysis-change-2OYM7ARW.mjs} +4 -4
  2. package/dist/{react-native/chunks/analysis-change-entry-BNRU6STO.mjs → analysis-change-entry-QD7YNDEE.mjs} +1 -1
  3. package/dist/browser/{analysis-change-M65AZQQG.mjs → analysis-change-2V2HZ2NY.mjs} +2 -2
  4. package/dist/{analysis-change-entry-YN3ELL6C.mjs → browser/analysis-change-entry-JL5U2N3Q.mjs} +1 -1
  5. package/dist/browser/{chunk-SRL47USS.mjs → chunk-2A2IPRC3.mjs} +1 -1
  6. package/dist/browser/chunk-3I27KXQD.mjs +34 -0
  7. package/dist/browser/{chunk-KAEVGLYN.mjs → chunk-55WYRLDZ.mjs} +15 -0
  8. package/dist/browser/{chunk-6OWX3OP2.mjs → chunk-7ZJTRUVW.mjs} +1 -1
  9. package/dist/browser/{chunk-QMNZFFD3.mjs → chunk-B3JM6Z74.mjs} +6 -2
  10. package/dist/browser/{chunk-BRWY3JCN.mjs → chunk-CSCKZBVJ.mjs} +4 -29
  11. package/dist/browser/{chunk-D6UFGBF5.mjs → chunk-GJKXW3SF.mjs} +43 -24
  12. package/dist/browser/{chunk-5V74FFZE.mjs → chunk-HT7J73QL.mjs} +1 -1
  13. package/dist/browser/{chunk-BVX2MEHA.mjs → chunk-KWYQ6KX4.mjs} +17 -17
  14. package/dist/browser/{chunk-2JABMB4S.mjs → chunk-PM4MUM5N.mjs} +2 -2
  15. package/dist/browser/{chunk-W643QV3L.mjs → chunk-QJCO3TJU.mjs} +8 -4
  16. package/dist/browser/{chunk-ODQP7P3H.mjs → chunk-TQ5UMFG5.mjs} +1 -1
  17. package/dist/browser/{chunk-4DYLCICG.mjs → chunk-VBX2TI5V.mjs} +2 -2
  18. package/dist/browser/{chunk-XILIT5MT.mjs → chunk-VD7JEUIR.mjs} +5 -5
  19. package/dist/browser/{cold-module-A5S6JFDN.mjs → cold-module-YECAK4MN.mjs} +18 -12
  20. package/dist/browser/{durability-wiring-W5AVV3RF.mjs → durability-wiring-LM7WFL6X.mjs} +61 -50
  21. package/dist/browser/{import-PTT63AGK.mjs → import-QOY6ZH6X.mjs} +3 -3
  22. package/dist/browser/{module-CEKA7MQH.mjs → module-KAMRAJLE.mjs} +6 -6
  23. package/dist/browser/{module-X5PGOE47.mjs → module-XQLEOYNM.mjs} +24 -23
  24. package/dist/{react-native/chunks/restore-queue-OI5EH7R3.mjs → browser/restore-queue-UQ6AAUVX.mjs} +2 -2
  25. package/dist/browser/{snapshot-DQ6A4PQP.mjs → snapshot-UGZVMP4H.mjs} +11 -10
  26. package/dist/browser/{write-MAGRAIEH.mjs → write-I5P5MW4O.mjs} +7 -7
  27. package/dist/{chunk-VQDZVWN3.mjs → chunk-2PNYKG2D.mjs} +16 -16
  28. package/dist/{chunk-U6NE6RCV.mjs → chunk-3E4RRF5J.mjs} +1 -1
  29. package/dist/{chunk-ZYKUD5NX.mjs → chunk-4QCSHLNM.mjs} +56 -6
  30. package/dist/{chunk-YKEHQUSZ.mjs → chunk-52QQWEBS.mjs} +2 -2
  31. package/dist/{chunk-KZ5GQE4Z.mjs → chunk-6RKSY3BX.mjs} +6 -6
  32. package/dist/{chunk-6C5EX2QR.mjs → chunk-7LDMWKP6.mjs} +1 -1
  33. package/dist/{chunk-ADBS5XRI.mjs → chunk-7TE2AJIH.mjs} +6 -6
  34. package/dist/{chunk-QN5WVV5K.mjs → chunk-CSHHU7PA.mjs} +5 -5
  35. package/dist/{chunk-ZAYWOHVG.mjs → chunk-DGRM4EZZ.mjs} +1 -1
  36. package/dist/{chunk-7N223GWV.mjs → chunk-DMNPYF54.mjs} +6 -2
  37. package/dist/{chunk-7HKWO4BJ.mjs → chunk-FDG4KXX5.mjs} +1 -1
  38. package/dist/{chunk-QEYF2SQ6.mjs → chunk-FNBNZCGP.mjs} +1 -1
  39. package/dist/{chunk-ZEKDO7YX.mjs → chunk-G5KHSM6E.mjs} +15 -0
  40. package/dist/{chunk-3BVGJ4JR.mjs → chunk-IJSFWGVW.mjs} +3 -28
  41. package/dist/{chunk-B7XIOAMK.mjs → chunk-J4BDAH2W.mjs} +1 -1
  42. package/dist/{chunk-CYYSZPVV.mjs → chunk-KQJVNPRM.mjs} +18 -19
  43. package/dist/chunk-MATMVH65.mjs +9 -0
  44. package/dist/{react-native/chunks/chunk-R4EEMYM6.mjs → chunk-NITCPDFL.mjs} +2 -2
  45. package/dist/{chunk-U2726TL5.mjs → chunk-NPXVQS33.mjs} +6 -2
  46. package/dist/{chunk-DOY6EG76.mjs → chunk-RBEHLIKX.mjs} +2 -2
  47. package/dist/{chunk-F22IOCR2.mjs → chunk-SEAJS664.mjs} +1 -1
  48. package/dist/{chunk-O4KUNDVF.mjs → chunk-UGVUW27S.mjs} +3 -3
  49. package/dist/chunk-VD7J552I.mjs +28 -0
  50. package/dist/{chunk-L7TIAYPX.mjs → chunk-ZHAXP7SD.mjs} +7 -7
  51. package/dist/{chunk-XKPNKR3V.mjs → chunk-ZS6CMXAE.mjs} +31 -10
  52. package/dist/{cold-module-2WWGBFJC.mjs → cold-module-PYSSBIL7.mjs} +18 -12
  53. package/dist/{direct-executor-QLVHMELJ.mjs → direct-executor-WYALQSMD.mjs} +2 -2
  54. package/dist/distribution/coordinator/etcd.d.ts +9 -0
  55. package/dist/distribution/coordinator/etcd.mjs +267 -106
  56. package/dist/distribution.mjs +23 -21
  57. package/dist/{durability-wiring-GNWQSZI3.mjs → durability-wiring-7VBZEJLG.mjs} +19 -45
  58. package/dist/{import-DX55PN4G.mjs → import-GJOMB2HC.mjs} +4 -4
  59. package/dist/index.browser.mjs +1 -1
  60. package/dist/index.mjs +3 -3
  61. package/dist/index.react-native.mjs +1 -1
  62. package/dist/{module-V5K4ZP23.mjs → module-6RLIKSZQ.mjs} +8 -8
  63. package/dist/{module-HW6IXUM6.mjs → module-AVQEIB7O.mjs} +23 -21
  64. package/dist/{paged-partition-archive-FDIIWZNR.mjs → paged-partition-archive-KGRW366Y.mjs} +11 -10
  65. package/dist/persistence/durability/checkpoint-worker.mjs +8 -8
  66. package/dist/react-native/chunks/{analysis-change-HGW3L6NY.mjs → analysis-change-DXPZRGWF.mjs} +2 -2
  67. package/dist/{browser/analysis-change-entry-H3YIOLXZ.mjs → react-native/chunks/analysis-change-entry-CXNRW324.mjs} +1 -1
  68. package/dist/react-native/chunks/{chunk-5FV4ZEVA.mjs → chunk-7IQR3SZO.mjs} +43 -24
  69. package/dist/react-native/chunks/{chunk-7VWQLEFV.mjs → chunk-CLE3LZ77.mjs} +6 -2
  70. package/dist/react-native/chunks/{chunk-SLGIAD6B.mjs → chunk-D7LFYPYI.mjs} +4 -4
  71. package/dist/{chunk-KZQIVGW7.mjs → react-native/chunks/chunk-DFQJ4IJB.mjs} +2 -2
  72. package/dist/react-native/chunks/{chunk-B6OD63Z3.mjs → chunk-FS7BYE23.mjs} +8 -4
  73. package/dist/react-native/chunks/chunk-HJBJBH26.mjs +34 -0
  74. package/dist/react-native/chunks/{chunk-YJSMEKRE.mjs → chunk-IDFMNRIC.mjs} +17 -17
  75. package/dist/react-native/chunks/{chunk-FKGG4UOF.mjs → chunk-JYL35A4A.mjs} +1 -1
  76. package/dist/react-native/chunks/{chunk-GV5KFQ6M.mjs → chunk-KLZGAOLT.mjs} +4 -29
  77. package/dist/react-native/chunks/{chunk-HURV3ASY.mjs → chunk-KOCTCUXW.mjs} +1 -1
  78. package/dist/react-native/chunks/{chunk-KUJOBRKS.mjs → chunk-UCJU6WNH.mjs} +1 -1
  79. package/dist/react-native/chunks/{chunk-LQ6Q4L5B.mjs → chunk-VGUWIZFQ.mjs} +15 -0
  80. package/dist/react-native/chunks/{cold-module-TSFWLYWC.mjs → cold-module-KFGEWYEL.mjs} +16 -10
  81. package/dist/react-native/chunks/{durability-wiring-PZR6KOQK.mjs → durability-wiring-QKJB2QBT.mjs} +61 -50
  82. package/dist/react-native/chunks/{import-VTODLPLO.mjs → import-XFBCCENV.mjs} +2 -2
  83. package/dist/react-native/chunks/{module-VEK2KCKU.mjs → module-FHJ2GOBI.mjs} +4 -4
  84. package/dist/react-native/chunks/{module-JBBH6PZH.mjs → module-UJ5VUARS.mjs} +24 -23
  85. package/dist/{browser/restore-queue-DD4LWZK7.mjs → react-native/chunks/restore-queue-H5VCN453.mjs} +2 -2
  86. package/dist/react-native/chunks/{snapshot-AUZMEV36.mjs → snapshot-KVBD4LRC.mjs} +11 -10
  87. package/dist/react-native/chunks/{write-DDC2QBRJ.mjs → write-ES5A3WUA.mjs} +5 -5
  88. package/dist/{restore-queue-BQWWNNQI.mjs → restore-queue-AOF5OTQV.mjs} +2 -2
  89. package/dist/server.mjs +5 -5
  90. package/dist/{snapshot-EHXTOTW5.mjs → snapshot-WMX6JCLP.mjs} +18 -16
  91. package/dist/workers/entry.mjs +2 -2
  92. package/dist/{write-SDJXXQ2G.mjs → write-SKJ6HW7O.mjs} +7 -7
  93. package/package.json +7 -7
@@ -1,11 +1,13 @@
1
- import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-KZ5GQE4Z.mjs';
1
+ import { loadVectorField, VECTOR_FILE_ENVELOPE_VERSION, GRAPH_PART_ENVELOPE_VERSION } from './chunk-6RKSY3BX.mjs';
2
2
  import { gunzip, gzip } from './chunk-ZXNNR73R.mjs';
3
- import { SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-7N223GWV.mjs';
4
- import { VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-ZEKDO7YX.mjs';
5
- import { readSegmentFile, STABLE_HEAD_ATTEMPTS } from './chunk-QN5WVV5K.mjs';
6
- import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, manifestKey, segmentsPrefix, vectorFileKey, graphPartKey } from './chunk-6C5EX2QR.mjs';
7
- import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks } from './chunk-VQDZVWN3.mjs';
8
- import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-O4KUNDVF.mjs';
3
+ import { SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-DMNPYF54.mjs';
4
+ import { defaultQuantizationFor } from './chunk-MATMVH65.mjs';
5
+ import { DEFAULT_VECTOR_METRIC } from './chunk-VD7J552I.mjs';
6
+ import { VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-G5KHSM6E.mjs';
7
+ import { readSegmentFile, STABLE_HEAD_ATTEMPTS } from './chunk-CSHHU7PA.mjs';
8
+ import { MAX_SEGMENT_ID, MANIFEST_ENVELOPE_VERSION, EARLIER_SEGMENT_MANIFEST_VERSION, SEGMENT_MANIFEST_VERSION, MAX_SEGMENTS_PER_PARTITION, segmentKey, manifestKey, segmentsPrefix, vectorFileKey, graphPartKey } from './chunk-7LDMWKP6.mjs';
9
+ import { VERSION, rulesOfStoredSynonyms, openChecksummedEnvelope, readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader, packChecksummedEnvelope, packEnvelopeWithChecksum, checksumOfChunks } from './chunk-2PNYKG2D.mjs';
10
+ import { decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays } from './chunk-UGVUW27S.mjs';
9
11
  import { DEFAULT_BM25_B, DEFAULT_BM25_K1 } from './chunk-3HV7M5OJ.mjs';
10
12
  import { flattenSchema, requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-SOUZMPME.mjs';
11
13
  import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
@@ -15,6 +17,21 @@ import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
15
17
  import { decode, encode } from '@msgpack/msgpack';
16
18
 
17
19
  // src/engine/durability-metadata.ts
20
+ var VECTOR_TYPE = /^vector\[(\d+)\]$/;
21
+ function vectorFieldsOf(schema, promotion) {
22
+ const fields = {};
23
+ for (const [fieldPath, type] of Object.entries(schema)) {
24
+ const match = VECTOR_TYPE.exec(type);
25
+ if (match === null) continue;
26
+ const dimension = Number(match[1]);
27
+ fields[fieldPath] = {
28
+ dimension,
29
+ metric: promotion?.hnswConfig?.metric ?? DEFAULT_VECTOR_METRIC,
30
+ quantization: promotion?.quantization ?? defaultQuantizationFor(dimension)
31
+ };
32
+ }
33
+ return Object.keys(fields).length === 0 ? null : fields;
34
+ }
18
35
  function metadataFromConfig(indexName, config, partitionCount, documentCount) {
19
36
  const metadata = {
20
37
  indexName,
@@ -30,6 +47,10 @@ function metadataFromConfig(indexName, config, partitionCount, documentCount) {
30
47
  engineVersion: VERSION
31
48
  };
32
49
  metadata.documentCount = documentCount ?? config.documentCount;
50
+ const vectorFields = vectorFieldsOf(config.schema, config.vectorPromotion);
51
+ if (vectorFields !== null) {
52
+ metadata.vectorFields = vectorFields;
53
+ }
33
54
  if (config.embedding) {
34
55
  metadata.embedding = config.embedding;
35
56
  }
@@ -1420,7 +1441,7 @@ function removeFromPartition(manager, partitionId, docIds) {
1420
1441
  }
1421
1442
  }
1422
1443
  async function attachPagedSegment(directory, ref, partitionId, manager, options) {
1423
- const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-V5K4ZP23.mjs');
1444
+ const { checkPagedSegmentFully, createPagedFrozenSegment, openPagedSnapshot, pagedListingOf } = await import('./module-6RLIKSZQ.mjs');
1424
1445
  if (options.decodesEveryDocument) await checkPagedSegmentFully(directory, ref);
1425
1446
  const snapshot = openPagedSnapshot(await pagedListingOf(directory, ref));
1426
1447
  requireAgreedSchema(options.schema, snapshot.schema, ref.key);
@@ -1434,7 +1455,7 @@ async function loadPartitionSegments(store, partition, manager, options) {
1434
1455
  const { directory } = store;
1435
1456
  for (const ref of partition.segments) {
1436
1457
  if (manager.config.textStorage === "disk" && directory !== void 0) {
1437
- const { isPagedSegmentFile } = await import('./module-V5K4ZP23.mjs');
1458
+ const { isPagedSegmentFile } = await import('./module-6RLIKSZQ.mjs');
1438
1459
  if (await isPagedSegmentFile(directory, ref.key)) {
1439
1460
  await attachPagedSegment(directory, ref, partition.partitionId, manager, options);
1440
1461
  await yieldToEventLoop();
@@ -1490,7 +1511,7 @@ async function loadSegmentedCheckpoint(store, indexName, manifest, deps, decodes
1490
1511
  (field) => deps.vectorIndexes.get(field.fieldPath)?.storage === "disk" && field.files.some((file) => file.headOffset !== void 0)
1491
1512
  );
1492
1513
  if (!decodesEveryDocument && directory !== void 0 && opensVectorsOnDemand) {
1493
- const { startReadingOpeningSectionsAhead } = await import('./module-V5K4ZP23.mjs');
1514
+ const { startReadingOpeningSectionsAhead } = await import('./module-6RLIKSZQ.mjs');
1494
1515
  await startReadingOpeningSectionsAhead(directory, manifest);
1495
1516
  }
1496
1517
  for (const partition of manifest.partitions) {
@@ -1,20 +1,20 @@
1
- import { readWholeSectionChecked, checkEverySectionPage, readFileRange, openPagedFile, readCacheOfThisThread, createSectionReader, closePagedFile } from './chunk-ADBS5XRI.mjs';
2
- export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-ADBS5XRI.mjs';
1
+ import { readWholeSectionChecked, checkEverySectionPage, readFileRange, openPagedFile, readCacheOfThisThread, createSectionReader, closePagedFile } from './chunk-7TE2AJIH.mjs';
2
+ export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-7TE2AJIH.mjs';
3
3
  import './chunk-3SBJVQLL.mjs';
4
4
  import './chunk-NYSONUB7.mjs';
5
- import { loadFailed, decodeSections, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-7N223GWV.mjs';
6
- import { isDead, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-ZEKDO7YX.mjs';
5
+ import { loadFailed, decodeSections, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-DMNPYF54.mjs';
6
+ import { isDead, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-G5KHSM6E.mjs';
7
7
  import './chunk-FOCV4CBZ.mjs';
8
8
  import { magnitude, defaultRescoreDepth, defaultEfSearch, computeCodeBlockLayout, toScore, osqQuantize, createOsqScratch, scalarQueryProducts, osqEstimate, osqDistance, toDistance } from './chunk-AWNHDBKV.mjs';
9
9
  export { createHNSWIndex, osqBitsOf } from './chunk-AWNHDBKV.mjs';
10
10
  import { stageQueryLevels, readTrailer } from './chunk-37CD3OKS.mjs';
11
11
  import './chunk-DJHHF2L5.mjs';
12
- import { COLD_LOAD_WAIT_WITHOUT_SEARCH_MS, COLD_WALK_NODES_READ_AHEAD } from './chunk-QN5WVV5K.mjs';
13
- import './chunk-U2726TL5.mjs';
14
- import './chunk-6C5EX2QR.mjs';
15
- import { yieldToEventLoop, HEADER_SIZE, crc32 } from './chunk-VQDZVWN3.mjs';
12
+ import { COLD_LOAD_WAIT_WITHOUT_SEARCH_MS, COLD_WALK_NODES_READ_AHEAD } from './chunk-CSHHU7PA.mjs';
13
+ import './chunk-NPXVQS33.mjs';
14
+ import './chunk-7LDMWKP6.mjs';
15
+ import { yieldToEventLoop, HEADER_SIZE, crc32, readHeader } from './chunk-2PNYKG2D.mjs';
16
16
  import './chunk-6F6USYII.mjs';
17
- import './chunk-O4KUNDVF.mjs';
17
+ import './chunk-UGVUW27S.mjs';
18
18
  import './chunk-3HV7M5OJ.mjs';
19
19
  import './chunk-KAKQ74LS.mjs';
20
20
  import './chunk-HJSN7UYG.mjs';
@@ -27,7 +27,7 @@ import './chunk-2HJWQPR6.mjs';
27
27
  import './chunk-X5CV2E2B.mjs';
28
28
  import './chunk-TGP5M5R7.mjs';
29
29
  import './chunk-U2BKYEC7.mjs';
30
- import './chunk-ZAYWOHVG.mjs';
30
+ import './chunk-DGRM4EZZ.mjs';
31
31
  import './chunk-QHPYS427.mjs';
32
32
  import './chunk-QYRNLLLI.mjs';
33
33
  import './chunk-ZXNQEW43.mjs';
@@ -339,9 +339,15 @@ var BIN16_MAX_BYTES = 65535;
339
339
  var VECTOR_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
340
340
  var NEIGHBOUR_PAGE_SIZES = /* @__PURE__ */ new Set([RECORD_PAGE_BYTES]);
341
341
  var LEVEL_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES]);
342
+ function holdsRawPayload(path) {
343
+ const header = new Uint8Array(HEADER_SIZE);
344
+ readFileRange(path, 0, header);
345
+ return !readHeader(header).flags.compressionEnabled;
346
+ }
342
347
  function readHead(path, ref) {
343
348
  const { headOffset, headLength, headChecksum } = ref;
344
349
  if (headOffset === void 0 || headLength === void 0 || headChecksum === void 0) return null;
350
+ if (!holdsRawPayload(path)) return null;
345
351
  const head = new Uint8Array(headLength);
346
352
  readFileRange(path, HEADER_SIZE + headOffset, head);
347
353
  if (crc32(head) !== headChecksum) {
@@ -695,11 +701,11 @@ function openColdVectorField(input) {
695
701
  var FILE_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES]);
696
702
  var FLOAT32_BYTES = 4;
697
703
  var FLOAT64_BYTES2 = 8;
698
- function waitBeforeColdLoad(firstSearchAnswered) {
704
+ function waitBeforeColdLoad(loadingWanted) {
699
705
  return new Promise((resolve) => {
700
706
  const timer = setTimeout(resolve, COLD_LOAD_WAIT_WITHOUT_SEARCH_MS);
701
707
  if (typeof timer.unref === "function") timer.unref();
702
- void firstSearchAnswered.then(() => {
708
+ void loadingWanted.then(() => {
703
709
  clearTimeout(timer);
704
710
  resolve();
705
711
  });
@@ -1,4 +1,4 @@
1
- export { createDirectExecutor } from './chunk-O4KUNDVF.mjs';
1
+ export { createDirectExecutor } from './chunk-UGVUW27S.mjs';
2
2
  import './chunk-3HV7M5OJ.mjs';
3
3
  import './chunk-KAKQ74LS.mjs';
4
4
  import './chunk-Q2UL2TPC.mjs';
@@ -6,7 +6,7 @@ import './chunk-2HJWQPR6.mjs';
6
6
  import './chunk-X5CV2E2B.mjs';
7
7
  import './chunk-TGP5M5R7.mjs';
8
8
  import './chunk-U2BKYEC7.mjs';
9
- import './chunk-ZAYWOHVG.mjs';
9
+ import './chunk-DGRM4EZZ.mjs';
10
10
  import './chunk-ZXNQEW43.mjs';
11
11
  import './chunk-LQG5G2LY.mjs';
12
12
  import './chunk-U46IZNCW.mjs';
@@ -63,6 +63,15 @@ interface EtcdCoordinatorConfig {
63
63
  * address alone, so that the password never crosses a network in clear text.
64
64
  */
65
65
  auth?: EtcdAuthConfig;
66
+ /**
67
+ * The coordinator calls this with every error that a watch handler throws,
68
+ * including one raised by a value under a watched prefix that the
69
+ * coordinator cannot read. The coordinator then skips that change, reopens
70
+ * the watch, and lists the prefix again, so that the error never ends the
71
+ * process and every later change still arrives. The coordinator drops
72
+ * these errors when you leave this out.
73
+ */
74
+ onWatchError?: (error: Error) => void;
66
75
  }
67
76
 
68
77
  /**
@@ -227,44 +227,202 @@ async function loadEtcd3Module() {
227
227
  }
228
228
  }
229
229
 
230
- // src/distribution/coordinator/etcd/prefix-watch.ts
231
- function connectedOrDropped(watcher) {
232
- return new Promise((resolve, reject) => {
233
- const timer = setTimeout(() => {
234
- detach();
235
- reject(
236
- new NarsilError(
237
- ErrorCodes.COORDINATOR_UNAVAILABLE,
238
- `etcd did not attach the watch within ${ETCD_WATCH_ATTACH_TIMEOUT_MS} ms`
239
- )
240
- );
241
- }, ETCD_WATCH_ATTACH_TIMEOUT_MS);
242
- timer.unref?.();
243
- const onConnected = () => {
244
- detach();
245
- resolve();
230
+ // src/distribution/coordinator/etcd/watch-stream.ts
231
+ var WatchAttachError = class extends NarsilError {
232
+ };
233
+ function shutDownError() {
234
+ return new NarsilError(ErrorCodes.CONFIG_INVALID, "Coordinator has been shut down");
235
+ }
236
+ function attachFailure(message) {
237
+ return new WatchAttachError(ErrorCodes.COORDINATOR_UNAVAILABLE, message);
238
+ }
239
+ function prefixRangeEnd(prefix) {
240
+ for (let position = prefix.length - 1; position >= 0; position--) {
241
+ const byte = prefix[position];
242
+ if (byte !== void 0 && byte < 255) {
243
+ const end = Buffer.from(prefix.subarray(0, position + 1));
244
+ end[position] = byte + 1;
245
+ return end;
246
+ }
247
+ }
248
+ return Buffer.from([0]);
249
+ }
250
+ async function openOne(client, prefix, startRevision, listener, registry) {
251
+ const stream = await client.watchClient.watch();
252
+ let attached = false;
253
+ let closing = false;
254
+ let finished = false;
255
+ let settleAttach = () => {
256
+ };
257
+ let markFinished = () => {
258
+ };
259
+ const done = new Promise((resolve) => {
260
+ markFinished = resolve;
261
+ });
262
+ const attachedOrFailed = new Promise((resolve) => {
263
+ settleAttach = resolve;
264
+ });
265
+ const watch = {
266
+ close() {
267
+ if (!closing) {
268
+ closing = true;
269
+ try {
270
+ stream.end();
271
+ stream.cancel();
272
+ } catch (_) {
273
+ }
274
+ }
275
+ return done;
276
+ }
277
+ };
278
+ function finish() {
279
+ if (finished) return;
280
+ finished = true;
281
+ registry.forget(watch);
282
+ markFinished();
283
+ if (!attached) settleAttach(attachFailure("the etcd watch stream ended before etcd attached the watch"));
284
+ else if (!closing) listener.lost();
285
+ }
286
+ function receive(response) {
287
+ if (closing || finished) return;
288
+ if (!attached) {
289
+ if (!response.created) return;
290
+ if (response.canceled) {
291
+ settleAttach(attachFailure(`etcd refused the watch: ${response.cancel_reason || "no reason given"}`));
292
+ return;
293
+ }
294
+ attached = true;
295
+ settleAttach(null);
296
+ return;
297
+ }
298
+ for (const event of response.events ?? []) {
299
+ try {
300
+ if (event.type === "Put") listener.put(event.kv);
301
+ else if (event.type === "Delete") listener.delete(event.kv);
302
+ } catch (handlerError) {
303
+ void watch.close();
304
+ listener.handlerFailed(handlerError);
305
+ listener.lost();
306
+ return;
307
+ }
308
+ }
309
+ if (response.canceled) {
310
+ void watch.close();
311
+ listener.lost();
312
+ }
313
+ }
314
+ stream.on("data", (response) => {
315
+ try {
316
+ receive(response);
317
+ } catch (_) {
318
+ }
319
+ });
320
+ stream.on("error", () => {
321
+ });
322
+ stream.on("status", () => {
323
+ try {
324
+ finish();
325
+ } catch (_) {
326
+ }
327
+ });
328
+ registry.remember(watch);
329
+ if (registry.isClosed()) {
330
+ await watch.close();
331
+ throw shutDownError();
332
+ }
333
+ const timer = setTimeout(
334
+ () => settleAttach(attachFailure(`etcd did not attach the watch within ${ETCD_WATCH_ATTACH_TIMEOUT_MS} ms`)),
335
+ ETCD_WATCH_ATTACH_TIMEOUT_MS
336
+ );
337
+ timer.unref?.();
338
+ const key = Buffer.from(prefix);
339
+ try {
340
+ stream.write({ create_request: { key, range_end: prefixRangeEnd(key), start_revision: startRevision } });
341
+ } catch (_) {
342
+ settleAttach(attachFailure("the etcd watch stream refused the watch request"));
343
+ }
344
+ const failure = await attachedOrFailed;
345
+ clearTimeout(timer);
346
+ if (failure !== null) {
347
+ await watch.close();
348
+ throw registry.isClosed() ? shutDownError() : failure;
349
+ }
350
+ return watch;
351
+ }
352
+ function createWatchStreams() {
353
+ const open = /* @__PURE__ */ new Set();
354
+ const pending = /* @__PURE__ */ new Set();
355
+ let opening = 0;
356
+ let closed = false;
357
+ function track(operation) {
358
+ const followed = operation.then(
359
+ () => {
360
+ },
361
+ () => {
362
+ }
363
+ );
364
+ pending.add(followed);
365
+ void followed.then(() => pending.delete(followed));
366
+ return operation;
367
+ }
368
+ async function openTracked(client, prefix, startRevision, listener) {
369
+ if (closed) throw shutDownError();
370
+ if (open.size + opening >= MAX_WATCHERS) {
371
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Maximum watcher limit (${MAX_WATCHERS}) reached`, {
372
+ currentCount: open.size + opening
373
+ });
374
+ }
375
+ opening += 1;
376
+ let reserved = true;
377
+ const releaseReservation = () => {
378
+ if (!reserved) return;
379
+ reserved = false;
380
+ opening -= 1;
246
381
  };
247
- const onDropped = (error) => {
248
- detach();
249
- reject(error);
382
+ const registry = {
383
+ remember(watch) {
384
+ releaseReservation();
385
+ open.add(watch);
386
+ },
387
+ forget: (watch) => open.delete(watch),
388
+ isClosed: () => closed
250
389
  };
251
- function detach() {
252
- clearTimeout(timer);
253
- watcher.off("connected", onConnected);
254
- watcher.off("disconnected", onDropped);
255
- watcher.off("error", onDropped);
390
+ try {
391
+ return await openOne(client, prefix, startRevision, listener, registry);
392
+ } finally {
393
+ releaseReservation();
256
394
  }
257
- watcher.once("connected", onConnected);
258
- watcher.once("disconnected", onDropped);
259
- watcher.once("error", onDropped);
260
- });
395
+ }
396
+ return {
397
+ open: (client, prefix, startRevision, listener) => track(openTracked(client, prefix, startRevision, listener)),
398
+ track,
399
+ isClosed: () => closed,
400
+ async closeAll() {
401
+ closed = true;
402
+ while (open.size > 0 || pending.size > 0) {
403
+ await Promise.all([...[...open].map((watch) => watch.close()), ...pending]);
404
+ }
405
+ }
406
+ };
407
+ }
408
+
409
+ // src/distribution/coordinator/etcd/prefix-watch.ts
410
+ function blamesTheConnection(error) {
411
+ return error instanceof WatchAttachError || !(error instanceof NarsilError);
261
412
  }
262
- async function watchPrefix(context, prefix, handlers) {
413
+ function watchPrefix(context, prefix, handlers) {
414
+ const { streams } = context;
415
+ if (streams.isClosed()) {
416
+ return Promise.reject(new NarsilError(ErrorCodes.CONFIG_INVALID, "Coordinator has been shut down"));
417
+ }
263
418
  const modRevisionByKey = /* @__PURE__ */ new Map();
264
419
  let current = null;
265
420
  let stopped = false;
266
421
  let reopenTimer = null;
267
422
  let reopenDelayMs = RECONNECT_FIRST_DELAY_MS;
423
+ function inactive() {
424
+ return stopped || streams.isClosed();
425
+ }
268
426
  function recordPut(kv) {
269
427
  modRevisionByKey.set(kv.key.toString(), kv.mod_revision);
270
428
  handlers.put(kv);
@@ -273,6 +431,13 @@ async function watchPrefix(context, prefix, handlers) {
273
431
  modRevisionByKey.delete(fullKey);
274
432
  handlers.delete(fullKey);
275
433
  }
434
+ function reportingHandlerErrors(record) {
435
+ try {
436
+ record();
437
+ } catch (error) {
438
+ context.reportHandlerError(error);
439
+ }
440
+ }
276
441
  async function listMissedChanges(reportMissedChanges) {
277
442
  const listing = await context.client.getAll().prefix(prefix).exec();
278
443
  const listedKeys = /* @__PURE__ */ new Set();
@@ -282,51 +447,54 @@ async function watchPrefix(context, prefix, handlers) {
282
447
  if (!reportMissedChanges) {
283
448
  modRevisionByKey.set(fullKey, kv.mod_revision);
284
449
  } else if (modRevisionByKey.get(fullKey) !== kv.mod_revision) {
285
- recordPut(kv);
450
+ reportingHandlerErrors(() => recordPut(kv));
286
451
  }
287
452
  }
288
453
  if (reportMissedChanges) {
289
454
  for (const fullKey of [...modRevisionByKey.keys()]) {
290
- if (!listedKeys.has(fullKey)) recordDelete(fullKey);
455
+ if (!listedKeys.has(fullKey)) reportingHandlerErrors(() => recordDelete(fullKey));
291
456
  }
292
457
  }
293
458
  return String(BigInt(listing.header.revision) + 1n);
294
459
  }
295
- async function watchFrom(startRevision2) {
460
+ async function watchFrom(startRevision) {
296
461
  const client = context.watchClient.current();
297
- const watcher = client.watch().prefix(prefix).startRevision(startRevision2).watcher();
462
+ let opened = null;
463
+ let lostWhileOpening = false;
464
+ let watch;
298
465
  try {
299
- await connectedOrDropped(watcher);
300
- } catch (error) {
301
- watcher.on("error", () => {
302
- });
303
- watcher.cancel().catch(() => {
466
+ watch = await streams.open(client, prefix, startRevision, {
467
+ put: recordPut,
468
+ delete: (kv) => recordDelete(kv.key.toString()),
469
+ handlerFailed: context.reportHandlerError,
470
+ lost() {
471
+ if (opened === null) lostWhileOpening = true;
472
+ else reopenAfterFailure(opened);
473
+ }
304
474
  });
305
- context.watchClient.replace(client);
475
+ } catch (error) {
476
+ if (!streams.isClosed() && blamesTheConnection(error)) context.watchClient.replace(client);
306
477
  throw error;
307
478
  }
308
- if (stopped || context.isShutdown()) {
309
- watcher.cancel().catch(() => {
310
- });
479
+ if (inactive()) {
480
+ await watch.close();
311
481
  return;
312
482
  }
313
- watcher.on("put", recordPut);
314
- watcher.on("delete", (kv) => recordDelete(kv.key.toString()));
315
- watcher.on("error", () => reopenAfterFailure(watcher));
316
- watcher.on("disconnected", () => reopenAfterFailure(watcher));
317
- context.addWatcher(watcher);
318
- current = watcher;
483
+ opened = watch;
484
+ current = watch;
485
+ if (lostWhileOpening) reopenAfterFailure(watch);
319
486
  }
320
487
  async function listThenWatch(reportMissedChanges) {
321
488
  await watchFrom(await listMissedChanges(reportMissedChanges));
322
489
  }
323
490
  function scheduleReopen() {
324
- if (stopped || context.isShutdown()) return;
491
+ if (inactive()) return;
325
492
  const delayMs = reopenDelayMs;
326
493
  reopenDelayMs = Math.min(reopenDelayMs * 2, RECONNECT_LONGEST_DELAY_MS);
327
494
  reopenTimer = setTimeout(() => {
328
495
  reopenTimer = null;
329
- listThenWatch(true).then(
496
+ if (inactive()) return;
497
+ streams.track(listThenWatch(true)).then(
330
498
  () => {
331
499
  reopenDelayMs = RECONNECT_FIRST_DELAY_MS;
332
500
  },
@@ -338,26 +506,29 @@ async function watchPrefix(context, prefix, handlers) {
338
506
  function reopenAfterFailure(failed) {
339
507
  if (current !== failed) return;
340
508
  current = null;
341
- context.removeWatcher(failed);
509
+ void failed.close();
342
510
  scheduleReopen();
343
511
  }
344
- const startRevision = await listMissedChanges(false);
345
- try {
346
- await watchFrom(startRevision);
347
- } catch (_) {
348
- scheduleReopen();
349
- }
350
- return () => {
351
- stopped = true;
352
- if (reopenTimer !== null) {
353
- clearTimeout(reopenTimer);
354
- reopenTimer = null;
355
- }
356
- if (current !== null) {
357
- context.removeWatcher(current);
358
- current = null;
512
+ async function start() {
513
+ const startRevision = await listMissedChanges(false);
514
+ try {
515
+ await watchFrom(startRevision);
516
+ } catch (_) {
517
+ scheduleReopen();
359
518
  }
360
- };
519
+ return () => {
520
+ stopped = true;
521
+ if (reopenTimer !== null) {
522
+ clearTimeout(reopenTimer);
523
+ reopenTimer = null;
524
+ }
525
+ if (current !== null) {
526
+ void current.close();
527
+ current = null;
528
+ }
529
+ };
530
+ }
531
+ return streams.track(start());
361
532
  }
362
533
  function serializeAllocationTable(table) {
363
534
  const serializable = {
@@ -584,6 +755,17 @@ function createEtcdWriteGuard(etcd3, maxRequestBytes) {
584
755
  };
585
756
  }
586
757
 
758
+ // src/distribution/coordinator/etcd/require-leader.ts
759
+ var ETCD_WATCH_METHOD_PATH = "/etcdserverpb.Watch/Watch";
760
+ var ETCD_REQUIRE_LEADER_METADATA_KEY = "hasleader";
761
+ var ETCD_REQUIRE_LEADER_METADATA_VALUE = "true";
762
+ function requireLeaderOnWatchStreams(properties) {
763
+ if (properties.methodDefinition.path !== ETCD_WATCH_METHOD_PATH) return properties;
764
+ const metadata = properties.metadata.clone();
765
+ metadata.set(ETCD_REQUIRE_LEADER_METADATA_KEY, ETCD_REQUIRE_LEADER_METADATA_VALUE);
766
+ return { ...properties, metadata };
767
+ }
768
+
587
769
  // src/distribution/coordinator/etcd/security.ts
588
770
  var SECURE_SCHEME = "https://";
589
771
  var LOOPBACK_IPV4 = /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/;
@@ -751,11 +933,8 @@ function storedValueMethods(context) {
751
933
  const prefix = keys.schemaPrefix();
752
934
  return watchPrefix2(prefix, {
753
935
  put(kv) {
754
- try {
755
- const schema = deserializeSchema(kv.value);
756
- handler({ type: "schema_created", indexName: extractSuffix(kv.key.toString(), prefix), schema });
757
- } catch (_) {
758
- }
936
+ const schema = deserializeSchema(kv.value);
937
+ handler({ type: "schema_created", indexName: extractSuffix(kv.key.toString(), prefix), schema });
759
938
  },
760
939
  delete(fullKey) {
761
940
  handler({ type: "schema_dropped", indexName: extractSuffix(fullKey, prefix), schema: null });
@@ -845,7 +1024,8 @@ async function createEtcdCoordinator(config) {
845
1024
  grpcOptions: {
846
1025
  "grpc.max_receive_message_length": ETCD_MAX_RESPONSE_BYTES,
847
1026
  "grpc.keepalive_time_ms": ETCD_KEEPALIVE_TIME_MS,
848
- "grpc.keepalive_timeout_ms": ETCD_KEEPALIVE_TIMEOUT_MS
1027
+ "grpc.keepalive_timeout_ms": ETCD_KEEPALIVE_TIMEOUT_MS,
1028
+ callInvocationTransformer: requireLeaderOnWatchStreams
849
1029
  },
850
1030
  defaultCallOptions: (context) => context.isStream ? {} : { deadline: Date.now() + ETCD_CALL_TIMEOUT_MS },
851
1031
  faultHandling: { host: (endpoint) => createEndpointBreaker(endpoint, etcd3Module.isRecoverableError) },
@@ -861,30 +1041,23 @@ async function createEtcdCoordinator(config) {
861
1041
  }
862
1042
  const guard = createEtcdWriteGuard(etcd3Module, resolvedConfig.maxRequestBytes);
863
1043
  const leaseManager = new LeaseManager();
864
- const watchers = /* @__PURE__ */ new Set();
865
1044
  let isShutdown = false;
866
1045
  function assertNotShutdown() {
867
1046
  if (isShutdown) {
868
1047
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Coordinator has been shut down");
869
1048
  }
870
1049
  }
871
- function addWatcher(watcher) {
872
- if (watchers.size >= MAX_WATCHERS) {
873
- watcher.cancel().catch(() => {
874
- });
875
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Maximum watcher limit (${MAX_WATCHERS}) reached`, {
876
- currentCount: watchers.size
877
- });
1050
+ const watchClient = createReplaceableWatchClient(openClient);
1051
+ const streams = createWatchStreams();
1052
+ const { onWatchError } = resolvedConfig;
1053
+ function reportHandlerError(error) {
1054
+ if (onWatchError === void 0) return;
1055
+ try {
1056
+ onWatchError(error instanceof Error ? error : new Error(String(error)));
1057
+ } catch (_) {
878
1058
  }
879
- watchers.add(watcher);
880
- }
881
- function removeWatcher(watcher) {
882
- watcher.cancel().catch(() => {
883
- });
884
- watchers.delete(watcher);
885
1059
  }
886
- const watchClient = createReplaceableWatchClient(openClient);
887
- const watchContext = { client, watchClient, addWatcher, removeWatcher, isShutdown: () => isShutdown };
1060
+ const watchContext = { client, watchClient, streams, reportHandlerError };
888
1061
  const coordinator = {
889
1062
  async registerNode(registration) {
890
1063
  assertNotShutdown();
@@ -947,11 +1120,8 @@ async function createEtcdCoordinator(config) {
947
1120
  const prefix = keys.nodePrefix();
948
1121
  return watchPrefix(watchContext, prefix, {
949
1122
  put(kv) {
950
- try {
951
- const registration = deserializeNodeRegistration(kv.value);
952
- handler({ type: "node_joined", nodeId: registration.nodeId, registration });
953
- } catch (_) {
954
- }
1123
+ const registration = deserializeNodeRegistration(kv.value);
1124
+ handler({ type: "node_joined", nodeId: registration.nodeId, registration });
955
1125
  },
956
1126
  delete(fullKey) {
957
1127
  handler({ type: "node_left", nodeId: extractSuffix(fullKey, prefix), registration: null });
@@ -1008,11 +1178,8 @@ async function createEtcdCoordinator(config) {
1008
1178
  assertNotShutdown();
1009
1179
  return watchPrefix(watchContext, keys.allocationPrefix(), {
1010
1180
  put(kv) {
1011
- try {
1012
- const table = deserializeAllocationTable(kv.value);
1013
- handler({ indexName: table.indexName, table });
1014
- } catch (_) {
1015
- }
1181
+ const table = deserializeAllocationTable(kv.value);
1182
+ handler({ indexName: table.indexName, table });
1016
1183
  },
1017
1184
  delete() {
1018
1185
  }
@@ -1100,13 +1267,7 @@ async function createEtcdCoordinator(config) {
1100
1267
  return;
1101
1268
  }
1102
1269
  isShutdown = true;
1103
- const cancelPromises = [];
1104
- for (const watcher of watchers) {
1105
- cancelPromises.push(watcher.cancel().catch(() => {
1106
- }));
1107
- }
1108
- await Promise.all(cancelPromises);
1109
- watchers.clear();
1270
+ await streams.closeAll();
1110
1271
  await leaseManager.revokeAll();
1111
1272
  watchClient.close();
1112
1273
  client.close();