@delali/narsil 0.1.15 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (251) hide show
  1. package/NOTICE +54 -0
  2. package/README.md +38 -982
  3. package/dist/adapters/filesystem.mjs +1 -1
  4. package/dist/adapters/indexeddb.mjs +1 -1
  5. package/dist/{chunk-WISGW24C.mjs → chunk-AUMKFYTG.mjs} +1 -1
  6. package/dist/{chunk-HRABJLPB.mjs → chunk-BKAI2WXI.mjs} +1 -0
  7. package/dist/chunk-GHJWC2E5.mjs +9 -0
  8. package/dist/chunk-KH3YGMVT.mjs +15 -0
  9. package/dist/{chunk-ALB35PRQ.mjs → chunk-M2EDG55Y.mjs} +8 -0
  10. package/dist/{chunk-HMO6OMZ3.mjs → chunk-S6IAXOBA.mjs} +1073 -506
  11. package/dist/chunk-TGP5M5R7.mjs +880 -0
  12. package/dist/chunk-U2BKYEC7.mjs +254 -0
  13. package/dist/{chunk-MEW7HD75.mjs → chunk-XAKNSWIK.mjs} +2543 -1154
  14. package/dist/{config-CojEPOgS.d.ts → config-P27AtSn4.d.ts} +28 -7
  15. package/dist/direct-executor-2FV3MEDD.mjs +5 -0
  16. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  17. package/dist/distribution/coordinator/etcd.mjs +2 -2
  18. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  19. package/dist/distribution.d.ts +10 -4
  20. package/dist/distribution.mjs +1875 -1772
  21. package/dist/embeddings/openai.mjs +1 -1
  22. package/dist/index.browser.mjs +4566 -1783
  23. package/dist/index.d.ts +14 -5
  24. package/dist/index.mjs +6 -5
  25. package/dist/{language-ONQ3iVDf.d.ts → language-CmJBHf1j.d.ts} +4 -1
  26. package/dist/languages/albanian.d.ts +5 -0
  27. package/dist/languages/albanian.mjs +103 -0
  28. package/dist/languages/amharic.d.ts +5 -0
  29. package/dist/languages/amharic.mjs +240 -0
  30. package/dist/languages/arabic.d.ts +1 -1
  31. package/dist/languages/arabic.mjs +939 -154
  32. package/dist/languages/armenian.d.ts +1 -1
  33. package/dist/languages/armenian.mjs +275 -243
  34. package/dist/languages/azerbaijani.d.ts +5 -0
  35. package/dist/languages/azerbaijani.mjs +152 -0
  36. package/dist/languages/bambara.d.ts +5 -0
  37. package/dist/languages/bambara.mjs +60 -0
  38. package/dist/languages/basque.d.ts +5 -0
  39. package/dist/languages/basque.mjs +726 -0
  40. package/dist/languages/belarusian.d.ts +5 -0
  41. package/dist/languages/belarusian.mjs +100 -0
  42. package/dist/languages/bengali.d.ts +5 -0
  43. package/dist/languages/bengali.mjs +128 -0
  44. package/dist/languages/bosnian.d.ts +5 -0
  45. package/dist/languages/bosnian.mjs +197 -0
  46. package/dist/languages/breton.d.ts +5 -0
  47. package/dist/languages/breton.mjs +96 -0
  48. package/dist/languages/bulgarian.d.ts +1 -1
  49. package/dist/languages/bulgarian.mjs +11 -2
  50. package/dist/languages/burmese.d.ts +5 -0
  51. package/dist/languages/burmese.mjs +57 -0
  52. package/dist/languages/catalan.d.ts +5 -0
  53. package/dist/languages/catalan.mjs +987 -0
  54. package/dist/languages/chinese.d.ts +1 -1
  55. package/dist/languages/chinese.mjs +4 -3
  56. package/dist/languages/croatian.d.ts +5 -0
  57. package/dist/languages/croatian.mjs +191 -0
  58. package/dist/languages/czech.d.ts +5 -0
  59. package/dist/languages/czech.mjs +628 -0
  60. package/dist/languages/dagbani.d.ts +1 -1
  61. package/dist/languages/dagbani.mjs +11 -2
  62. package/dist/languages/danish.d.ts +1 -1
  63. package/dist/languages/danish.mjs +215 -110
  64. package/dist/languages/dutch.d.ts +1 -1
  65. package/dist/languages/dutch.mjs +1248 -163
  66. package/dist/languages/english.d.ts +1 -1
  67. package/dist/languages/english.mjs +2 -1
  68. package/dist/languages/esperanto.d.ts +5 -0
  69. package/dist/languages/esperanto.mjs +689 -0
  70. package/dist/languages/estonian.d.ts +5 -0
  71. package/dist/languages/estonian.mjs +2160 -0
  72. package/dist/languages/ewe.d.ts +1 -1
  73. package/dist/languages/ewe.mjs +11 -2
  74. package/dist/languages/faroese.d.ts +5 -0
  75. package/dist/languages/faroese.mjs +93 -0
  76. package/dist/languages/fijian.d.ts +5 -0
  77. package/dist/languages/fijian.mjs +55 -0
  78. package/dist/languages/finnish.d.ts +1 -1
  79. package/dist/languages/finnish.mjs +528 -263
  80. package/dist/languages/french.d.ts +1 -1
  81. package/dist/languages/french.mjs +950 -437
  82. package/dist/languages/ga.d.ts +1 -1
  83. package/dist/languages/ga.mjs +2 -1
  84. package/dist/languages/galician.d.ts +5 -0
  85. package/dist/languages/galician.mjs +172 -0
  86. package/dist/languages/georgian.d.ts +5 -0
  87. package/dist/languages/georgian.mjs +91 -0
  88. package/dist/languages/german.d.ts +1 -1
  89. package/dist/languages/german.mjs +432 -162
  90. package/dist/languages/greek.d.ts +1 -1
  91. package/dist/languages/greek.mjs +1970 -281
  92. package/dist/languages/guarani.d.ts +5 -0
  93. package/dist/languages/guarani.mjs +132 -0
  94. package/dist/languages/gujarati.d.ts +5 -0
  95. package/dist/languages/gujarati.mjs +234 -0
  96. package/dist/languages/haitian-creole.d.ts +5 -0
  97. package/dist/languages/haitian-creole.mjs +106 -0
  98. package/dist/languages/hausa.d.ts +1 -1
  99. package/dist/languages/hausa.mjs +2 -1
  100. package/dist/languages/hawaiian.d.ts +5 -0
  101. package/dist/languages/hawaiian.mjs +69 -0
  102. package/dist/languages/hebrew.d.ts +5 -0
  103. package/dist/languages/hebrew.mjs +213 -0
  104. package/dist/languages/hindi.d.ts +1 -1
  105. package/dist/languages/hindi.mjs +175 -97
  106. package/dist/languages/hungarian.d.ts +1 -1
  107. package/dist/languages/hungarian.mjs +412 -326
  108. package/dist/languages/icelandic.d.ts +5 -0
  109. package/dist/languages/icelandic.mjs +161 -0
  110. package/dist/languages/igbo.d.ts +1 -1
  111. package/dist/languages/igbo.mjs +2 -1
  112. package/dist/languages/indonesian.d.ts +1 -1
  113. package/dist/languages/indonesian.mjs +329 -137
  114. package/dist/languages/irish.d.ts +1 -1
  115. package/dist/languages/irish.mjs +223 -139
  116. package/dist/languages/italian.d.ts +1 -1
  117. package/dist/languages/italian.mjs +703 -406
  118. package/dist/languages/japanese.d.ts +1 -1
  119. package/dist/languages/japanese.mjs +5 -108
  120. package/dist/languages/kannada.d.ts +5 -0
  121. package/dist/languages/kannada.mjs +94 -0
  122. package/dist/languages/kazakh.d.ts +5 -0
  123. package/dist/languages/kazakh.mjs +90 -0
  124. package/dist/languages/khmer.d.ts +5 -0
  125. package/dist/languages/khmer.mjs +181 -0
  126. package/dist/languages/kinyarwanda.d.ts +5 -0
  127. package/dist/languages/kinyarwanda.mjs +99 -0
  128. package/dist/languages/kirundi.d.ts +5 -0
  129. package/dist/languages/kirundi.mjs +92 -0
  130. package/dist/languages/korean.d.ts +5 -0
  131. package/dist/languages/korean.mjs +252 -0
  132. package/dist/languages/kurmanji.d.ts +5 -0
  133. package/dist/languages/kurmanji.mjs +50 -0
  134. package/dist/languages/kyrgyz.d.ts +5 -0
  135. package/dist/languages/kyrgyz.mjs +108 -0
  136. package/dist/languages/lao.d.ts +5 -0
  137. package/dist/languages/lao.mjs +63 -0
  138. package/dist/languages/latin.d.ts +5 -0
  139. package/dist/languages/latin.mjs +69 -0
  140. package/dist/languages/latvian.d.ts +5 -0
  141. package/dist/languages/latvian.mjs +173 -0
  142. package/dist/languages/lingala.d.ts +5 -0
  143. package/dist/languages/lingala.mjs +61 -0
  144. package/dist/languages/lithuanian.d.ts +5 -0
  145. package/dist/languages/lithuanian.mjs +571 -0
  146. package/dist/languages/luxembourgish.d.ts +5 -0
  147. package/dist/languages/luxembourgish.mjs +214 -0
  148. package/dist/languages/macedonian.d.ts +5 -0
  149. package/dist/languages/macedonian.mjs +808 -0
  150. package/dist/languages/malagasy.d.ts +5 -0
  151. package/dist/languages/malagasy.mjs +67 -0
  152. package/dist/languages/malay.d.ts +5 -0
  153. package/dist/languages/malay.mjs +482 -0
  154. package/dist/languages/malayalam.d.ts +5 -0
  155. package/dist/languages/malayalam.mjs +122 -0
  156. package/dist/languages/maltese.d.ts +5 -0
  157. package/dist/languages/maltese.mjs +139 -0
  158. package/dist/languages/maori.d.ts +5 -0
  159. package/dist/languages/maori.mjs +79 -0
  160. package/dist/languages/marathi.d.ts +5 -0
  161. package/dist/languages/marathi.mjs +111 -0
  162. package/dist/languages/nepali.d.ts +1 -1
  163. package/dist/languages/nepali.mjs +209 -99
  164. package/dist/languages/norwegian.d.ts +1 -1
  165. package/dist/languages/norwegian.mjs +218 -85
  166. package/dist/languages/oromo.d.ts +5 -0
  167. package/dist/languages/oromo.mjs +78 -0
  168. package/dist/languages/persian.d.ts +5 -0
  169. package/dist/languages/persian.mjs +710 -0
  170. package/dist/languages/polish.d.ts +5 -0
  171. package/dist/languages/polish.mjs +614 -0
  172. package/dist/languages/portuguese.d.ts +1 -1
  173. package/dist/languages/portuguese.mjs +613 -380
  174. package/dist/languages/punjabi.d.ts +5 -0
  175. package/dist/languages/punjabi.mjs +197 -0
  176. package/dist/languages/romanian.d.ts +1 -1
  177. package/dist/languages/romanian.mjs +561 -382
  178. package/dist/languages/russian.d.ts +1 -1
  179. package/dist/languages/russian.mjs +373 -244
  180. package/dist/languages/samoan.d.ts +5 -0
  181. package/dist/languages/samoan.mjs +63 -0
  182. package/dist/languages/sanskrit.d.ts +1 -1
  183. package/dist/languages/sanskrit.mjs +2 -2
  184. package/dist/languages/scottish-gaelic.d.ts +5 -0
  185. package/dist/languages/scottish-gaelic.mjs +370 -0
  186. package/dist/languages/serbian.d.ts +1 -1
  187. package/dist/languages/serbian.mjs +3635 -155
  188. package/dist/languages/shona.d.ts +5 -0
  189. package/dist/languages/shona.mjs +91 -0
  190. package/dist/languages/sinhala.d.ts +5 -0
  191. package/dist/languages/sinhala.mjs +182 -0
  192. package/dist/languages/slovak.d.ts +5 -0
  193. package/dist/languages/slovak.mjs +430 -0
  194. package/dist/languages/slovenian.d.ts +1 -1
  195. package/dist/languages/slovenian.mjs +11 -2
  196. package/dist/languages/sorani.d.ts +5 -0
  197. package/dist/languages/sorani.mjs +156 -0
  198. package/dist/languages/spanish.d.ts +1 -1
  199. package/dist/languages/spanish.mjs +676 -374
  200. package/dist/languages/swahili.d.ts +1 -1
  201. package/dist/languages/swahili.mjs +2 -1
  202. package/dist/languages/swedish.d.ts +1 -1
  203. package/dist/languages/swedish.mjs +237 -95
  204. package/dist/languages/tagalog.d.ts +5 -0
  205. package/dist/languages/tagalog.mjs +159 -0
  206. package/dist/languages/tamil.d.ts +1 -1
  207. package/dist/languages/tamil.mjs +836 -64
  208. package/dist/languages/tatar.d.ts +5 -0
  209. package/dist/languages/tatar.mjs +1017 -0
  210. package/dist/languages/telugu.d.ts +5 -0
  211. package/dist/languages/telugu.mjs +62 -0
  212. package/dist/languages/thai.d.ts +5 -0
  213. package/dist/languages/thai.mjs +65 -0
  214. package/dist/languages/tibetan.d.ts +5 -0
  215. package/dist/languages/tibetan.mjs +119 -0
  216. package/dist/languages/tigrinya.d.ts +5 -0
  217. package/dist/languages/tigrinya.mjs +194 -0
  218. package/dist/languages/tongan.d.ts +5 -0
  219. package/dist/languages/tongan.mjs +56 -0
  220. package/dist/languages/turkish.d.ts +1 -1
  221. package/dist/languages/turkish.mjs +1246 -393
  222. package/dist/languages/twi.d.ts +1 -1
  223. package/dist/languages/twi.mjs +13 -2
  224. package/dist/languages/ukrainian.d.ts +1 -1
  225. package/dist/languages/ukrainian.mjs +125 -47
  226. package/dist/languages/urdu.d.ts +5 -0
  227. package/dist/languages/urdu.mjs +528 -0
  228. package/dist/languages/vietnamese.d.ts +5 -0
  229. package/dist/languages/vietnamese.mjs +277 -0
  230. package/dist/languages/welsh.d.ts +5 -0
  231. package/dist/languages/welsh.mjs +446 -0
  232. package/dist/languages/wolof.d.ts +5 -0
  233. package/dist/languages/wolof.mjs +79 -0
  234. package/dist/languages/xhosa.d.ts +5 -0
  235. package/dist/languages/xhosa.mjs +95 -0
  236. package/dist/languages/yoruba.d.ts +1 -1
  237. package/dist/languages/yoruba.mjs +1 -0
  238. package/dist/languages/zulu.d.ts +1 -1
  239. package/dist/languages/zulu.mjs +2 -1
  240. package/dist/narsil-CMZRPDlQ.d.ts +159 -0
  241. package/dist/persistence/durability/checkpoint-worker.d.ts +13 -0
  242. package/dist/persistence/durability/checkpoint-worker.mjs +6 -5
  243. package/dist/{schema-DCgkO1j4.d.ts → schema-BnvLvjRp.d.ts} +5 -3
  244. package/dist/server.d.ts +4 -3
  245. package/dist/server.mjs +13 -2
  246. package/dist/{types-DHX6zGeD.d.ts → types-BRTcBgAp.d.ts} +1 -1
  247. package/dist/workers/entry.mjs +2 -2
  248. package/package.json +10 -3
  249. package/dist/chunk-TW6BXTOL.mjs +0 -338
  250. package/dist/direct-executor-E52FFGDE.mjs +0 -4
  251. package/dist/narsil-D-84Nvm4.d.ts +0 -72
@@ -1,12 +1,13 @@
1
1
  export { createInMemoryCoordinator } from './chunk-RQLXC2YS.mjs';
2
- import './chunk-WISGW24C.mjs';
3
- import { ReplicationMessageTypes, ClusterMessageTypes, DEFAULT_TRANSPORT_CONFIG, TransportError, TransportErrorCodes, MAX_MESSAGE_SIZE_BYTES, createEngineCore, createNarsilFromCore, QueryMessageTypes, applyIndexEntry, applyDeleteEntry, validateEntryPayload, createAckMessage, validateReplicationEntry, MAX_PARTITION_COUNT, MAX_INDEX_NAME_LENGTH, validateIndexName, computeEntryChecksum, createInsyncConfirmMessage, createEntryMessage, getIndexMetadata, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, putIndexMetadata, createForwardMessage } from './chunk-MEW7HD75.mjs';
4
- import { deserializePayloadV2, createPartitionIndex, getLanguage, fnv1a, mergePartitionStats, MAX_NESTING_DEPTH } from './chunk-HMO6OMZ3.mjs';
2
+ import './chunk-AUMKFYTG.mjs';
3
+ import { ReplicationMessageTypes, ClusterMessageTypes, DEFAULT_TRANSPORT_CONFIG, TransportError, TransportErrorCodes, MAX_MESSAGE_SIZE_BYTES, createEngineCore, createNarsilFromCore, QueryMessageTypes, applyIndexEntry, applyDeleteEntry, validateEntryPayload, createAckMessage, validateReplicationEntry, MAX_PARTITION_COUNT, MAX_INDEX_NAME_LENGTH, validateIndexName, computeEntryChecksum, createInsyncConfirmMessage, createEntryMessage, getIndexMetadata, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, putIndexMetadata, createForwardMessage } from './chunk-XAKNSWIK.mjs';
4
+ import { deserializePayloadV2, createPartitionIndex, getLanguage, fnv1a, mergePartitionStats, MAX_NESTING_DEPTH } from './chunk-S6IAXOBA.mjs';
5
+ import './chunk-TGP5M5R7.mjs';
5
6
  import './chunk-S7RJJMHI.mjs';
6
7
  import { crc32 } from './chunk-KMMJMDUA.mjs';
7
- import './chunk-TW6BXTOL.mjs';
8
- import { ErrorCodes, NarsilError } from './chunk-ALB35PRQ.mjs';
9
- import { generateId } from './chunk-HRABJLPB.mjs';
8
+ import './chunk-U2BKYEC7.mjs';
9
+ import { ErrorCodes, NarsilError } from './chunk-M2EDG55Y.mjs';
10
+ import { generateId } from './chunk-BKAI2WXI.mjs';
10
11
  import { encode, decode } from '@msgpack/msgpack';
11
12
 
12
13
  // src/distribution/cluster/controller/types.ts
@@ -1474,6 +1475,90 @@ function createDataNodeLifecycle(config) {
1474
1475
  return handle;
1475
1476
  }
1476
1477
 
1478
+ // src/distribution/query/cursor.ts
1479
+ var MAX_CURSOR_LENGTH = 4096;
1480
+ var MAX_CURSOR_DOC_ID_LENGTH = 512;
1481
+ function truncateCursorForError(encoded) {
1482
+ if (encoded.length > 100) {
1483
+ return `${encoded.slice(0, 100)}...`;
1484
+ }
1485
+ return encoded;
1486
+ }
1487
+ function encodeDistributedCursor(score, docId) {
1488
+ const json = JSON.stringify({ s: score, d: docId });
1489
+ if (typeof Buffer !== "undefined") {
1490
+ return Buffer.from(json).toString("base64");
1491
+ }
1492
+ return btoa(json);
1493
+ }
1494
+ function decodeDistributedCursor(encoded) {
1495
+ if (encoded.length > MAX_CURSOR_LENGTH) {
1496
+ throw new NarsilError(
1497
+ ErrorCodes.SEARCH_INVALID_CURSOR,
1498
+ `Distributed cursor exceeds maximum length of ${MAX_CURSOR_LENGTH}`,
1499
+ { cursor: truncateCursorForError(encoded) }
1500
+ );
1501
+ }
1502
+ let json;
1503
+ try {
1504
+ if (typeof Buffer !== "undefined") {
1505
+ json = Buffer.from(encoded, "base64").toString("utf-8");
1506
+ } else {
1507
+ json = atob(encoded);
1508
+ }
1509
+ } catch {
1510
+ throw new NarsilError(
1511
+ ErrorCodes.SEARCH_INVALID_CURSOR,
1512
+ "Failed to decode distributed cursor: invalid base64 encoding",
1513
+ { cursor: truncateCursorForError(encoded) }
1514
+ );
1515
+ }
1516
+ let parsed;
1517
+ try {
1518
+ parsed = JSON.parse(json);
1519
+ } catch {
1520
+ throw new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, "Failed to decode distributed cursor: invalid JSON", {
1521
+ cursor: truncateCursorForError(encoded)
1522
+ });
1523
+ }
1524
+ if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
1525
+ throw new NarsilError(
1526
+ ErrorCodes.SEARCH_INVALID_CURSOR,
1527
+ 'Invalid distributed cursor structure: expected an object with "s" and "d" fields',
1528
+ { cursor: truncateCursorForError(encoded) }
1529
+ );
1530
+ }
1531
+ const record = parsed;
1532
+ const { s, d } = record;
1533
+ if (typeof s !== "number" || !Number.isFinite(s)) {
1534
+ throw new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, 'Invalid distributed cursor: "s" must be a finite number', {
1535
+ cursor: truncateCursorForError(encoded)
1536
+ });
1537
+ }
1538
+ if (typeof d !== "string" || d.length === 0) {
1539
+ throw new NarsilError(
1540
+ ErrorCodes.SEARCH_INVALID_CURSOR,
1541
+ 'Invalid distributed cursor: "d" must be a non-empty string',
1542
+ { cursor: truncateCursorForError(encoded) }
1543
+ );
1544
+ }
1545
+ if (d.length > MAX_CURSOR_DOC_ID_LENGTH) {
1546
+ throw new NarsilError(
1547
+ ErrorCodes.SEARCH_INVALID_CURSOR,
1548
+ `Invalid distributed cursor: "d" exceeds maximum length of ${MAX_CURSOR_DOC_ID_LENGTH}`,
1549
+ { cursor: truncateCursorForError(encoded) }
1550
+ );
1551
+ }
1552
+ if (d.includes("\0")) {
1553
+ throw new NarsilError(
1554
+ ErrorCodes.SEARCH_INVALID_CURSOR,
1555
+ 'Invalid distributed cursor: "d" must not contain null bytes',
1556
+ { cursor: truncateCursorForError(encoded) }
1557
+ );
1558
+ }
1559
+ return { s, d };
1560
+ }
1561
+
1477
1562
  // src/distribution/query/validators/common.ts
1478
1563
  var MAX_PARTITION_IDS = MAX_PARTITION_COUNT;
1479
1564
  var MAX_TERM_LENGTH = 1024;
@@ -1682,90 +1767,6 @@ function validateFetchResultPayload(decoded) {
1682
1767
  return decoded;
1683
1768
  }
1684
1769
 
1685
- // src/distribution/query/cursor.ts
1686
- var MAX_CURSOR_LENGTH = 4096;
1687
- var MAX_CURSOR_DOC_ID_LENGTH = 512;
1688
- function truncateCursorForError(encoded) {
1689
- if (encoded.length > 100) {
1690
- return `${encoded.slice(0, 100)}...`;
1691
- }
1692
- return encoded;
1693
- }
1694
- function encodeDistributedCursor(score, docId) {
1695
- const json = JSON.stringify({ s: score, d: docId });
1696
- if (typeof Buffer !== "undefined") {
1697
- return Buffer.from(json).toString("base64");
1698
- }
1699
- return btoa(json);
1700
- }
1701
- function decodeDistributedCursor(encoded) {
1702
- if (encoded.length > MAX_CURSOR_LENGTH) {
1703
- throw new NarsilError(
1704
- ErrorCodes.SEARCH_INVALID_CURSOR,
1705
- `Distributed cursor exceeds maximum length of ${MAX_CURSOR_LENGTH}`,
1706
- { cursor: truncateCursorForError(encoded) }
1707
- );
1708
- }
1709
- let json;
1710
- try {
1711
- if (typeof Buffer !== "undefined") {
1712
- json = Buffer.from(encoded, "base64").toString("utf-8");
1713
- } else {
1714
- json = atob(encoded);
1715
- }
1716
- } catch {
1717
- throw new NarsilError(
1718
- ErrorCodes.SEARCH_INVALID_CURSOR,
1719
- "Failed to decode distributed cursor: invalid base64 encoding",
1720
- { cursor: truncateCursorForError(encoded) }
1721
- );
1722
- }
1723
- let parsed;
1724
- try {
1725
- parsed = JSON.parse(json);
1726
- } catch {
1727
- throw new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, "Failed to decode distributed cursor: invalid JSON", {
1728
- cursor: truncateCursorForError(encoded)
1729
- });
1730
- }
1731
- if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
1732
- throw new NarsilError(
1733
- ErrorCodes.SEARCH_INVALID_CURSOR,
1734
- 'Invalid distributed cursor structure: expected an object with "s" and "d" fields',
1735
- { cursor: truncateCursorForError(encoded) }
1736
- );
1737
- }
1738
- const record = parsed;
1739
- const { s, d } = record;
1740
- if (typeof s !== "number" || !Number.isFinite(s)) {
1741
- throw new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, 'Invalid distributed cursor: "s" must be a finite number', {
1742
- cursor: truncateCursorForError(encoded)
1743
- });
1744
- }
1745
- if (typeof d !== "string" || d.length === 0) {
1746
- throw new NarsilError(
1747
- ErrorCodes.SEARCH_INVALID_CURSOR,
1748
- 'Invalid distributed cursor: "d" must be a non-empty string',
1749
- { cursor: truncateCursorForError(encoded) }
1750
- );
1751
- }
1752
- if (d.length > MAX_CURSOR_DOC_ID_LENGTH) {
1753
- throw new NarsilError(
1754
- ErrorCodes.SEARCH_INVALID_CURSOR,
1755
- `Invalid distributed cursor: "d" exceeds maximum length of ${MAX_CURSOR_DOC_ID_LENGTH}`,
1756
- { cursor: truncateCursorForError(encoded) }
1757
- );
1758
- }
1759
- if (d.includes("\0")) {
1760
- throw new NarsilError(
1761
- ErrorCodes.SEARCH_INVALID_CURSOR,
1762
- 'Invalid distributed cursor: "d" must not contain null bytes',
1763
- { cursor: truncateCursorForError(encoded) }
1764
- );
1765
- }
1766
- return { s, d };
1767
- }
1768
-
1769
1770
  // src/distribution/query/validators/filters.ts
1770
1771
  var COMPARISON_NUMERIC_OPS = ["gt", "lt", "gte", "lte"];
1771
1772
  var COMPARISON_EQ_OPS = ["eq", "ne"];
@@ -2493,6 +2494,8 @@ function createStatsMessage(payload, sourceId) {
2493
2494
  payload: encode(payload)
2494
2495
  };
2495
2496
  }
2497
+
2498
+ // src/distribution/query/fan-out.ts
2496
2499
  async function collectDistributedStats(indexName, params, nodeToPartitions, deps, config) {
2497
2500
  const terms = params.term !== null ? params.term.split(/\s+/).filter((t) => t.length > 0) : [];
2498
2501
  const statsPromises = [];
@@ -2849,8 +2852,8 @@ function mergeDistributedFacets(allFacets, maxBuckets = DEFAULT_MAX_FACET_BUCKET
2849
2852
  }
2850
2853
 
2851
2854
  // src/distribution/query/selection.ts
2852
- function hashBasedSelector(candidates, partitionId) {
2853
- return candidates[partitionId % candidates.length];
2855
+ function randomSelector(candidates, _partitionId) {
2856
+ return candidates[Math.floor(Math.random() * candidates.length)];
2854
2857
  }
2855
2858
  function collectActiveCandidates(assignment) {
2856
2859
  if (assignment.state !== "ACTIVE") {
@@ -2868,7 +2871,7 @@ function collectActiveCandidates(assignment) {
2868
2871
  candidates.sort();
2869
2872
  return candidates;
2870
2873
  }
2871
- function selectReplica(assignment, localNodeId, selector = hashBasedSelector, partitionId = 0) {
2874
+ function selectReplica(assignment, localNodeId, selector = randomSelector, partitionId = 0) {
2872
2875
  const candidates = collectActiveCandidates(assignment);
2873
2876
  if (candidates.length === 0) {
2874
2877
  return null;
@@ -2878,7 +2881,7 @@ function selectReplica(assignment, localNodeId, selector = hashBasedSelector, pa
2878
2881
  }
2879
2882
  return selector(candidates, partitionId);
2880
2883
  }
2881
- function selectReplicasForQuery(allocationTable, localNodeId, selector = hashBasedSelector) {
2884
+ function selectReplicasForQuery(allocationTable, localNodeId, selector = randomSelector) {
2882
2885
  const nodeToPartitions = /* @__PURE__ */ new Map();
2883
2886
  const unavailablePartitions = [];
2884
2887
  for (const [partitionId, assignment] of allocationTable.assignments) {
@@ -2936,7 +2939,7 @@ async function distributedQuery(indexName, params, deps, config, selector) {
2936
2939
  coverage: { totalPartitions: 0, queriedPartitions: 0, timedOutPartitions: 0, failedPartitions: 0 }
2937
2940
  };
2938
2941
  }
2939
- const routing = selectReplicasForQuery(allocationTable, deps.sourceNodeId, hashBasedSelector);
2942
+ const routing = selectReplicasForQuery(allocationTable, deps.sourceNodeId, randomSelector);
2940
2943
  const totalPartitions = allocationTable.assignments.size;
2941
2944
  if (routing.unavailablePartitions.length > 0 && !resolvedConfig.allowPartialResults) {
2942
2945
  throw new NarsilError("QUERY_NO_ACTIVE_REPLICA", "No active replica for one or more partitions", {
@@ -3100,242 +3103,53 @@ function worstCaseCoverage(a, b) {
3100
3103
  };
3101
3104
  }
3102
3105
 
3103
- // src/distribution/replication/types.ts
3104
- var DEFAULT_LOG_RETENTION_BYTES = 268435456;
3105
-
3106
- // src/distribution/replication/log.ts
3107
- var ENTRY_FIXED_OVERHEAD_BYTES = 40;
3108
- function estimateEntrySize(entry) {
3109
- return ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
3106
+ // src/distribution/cluster-node/bootstrap-cleanup.ts
3107
+ async function cleanupRemovedPartition(indexName, partitionId, deps) {
3108
+ try {
3109
+ const existing = deps.engine.listIndexes().find((idx) => idx.name === indexName);
3110
+ if (existing === void 0) {
3111
+ return;
3112
+ }
3113
+ if (await nodeHasOtherPartitions(indexName, partitionId, deps)) {
3114
+ return;
3115
+ }
3116
+ await deps.engine.dropIndex(indexName);
3117
+ } catch (err) {
3118
+ if (deps.onError === void 0) {
3119
+ return;
3120
+ }
3121
+ if (err instanceof NarsilError && err.code === ErrorCodes.INDEX_NOT_FOUND) {
3122
+ return;
3123
+ }
3124
+ const cause = err instanceof Error ? err.message : String(err);
3125
+ deps.onError(
3126
+ new NarsilError(
3127
+ ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED,
3128
+ `failed to drop local index '${indexName}' after partition removal: ${cause}`,
3129
+ { indexName, partitionId, cause }
3130
+ )
3131
+ );
3132
+ }
3110
3133
  }
3111
- var computeChecksum = computeEntryChecksum;
3112
- function createReplicationLog(partitionId, config) {
3113
- const retentionBytes = config?.logRetentionBytes ?? DEFAULT_LOG_RETENTION_BYTES;
3114
- let nextSeqNo = config?.startSeqNo ?? 1;
3115
- let committedSeqNo = nextSeqNo - 1;
3116
- let committedPrimaryTerm = config?.lastPrimaryTerm ?? 0;
3117
- let entries = [];
3118
- let headIndex = 0;
3119
- let totalSizeBytes = 0;
3120
- function liveCount() {
3121
- return entries.length - headIndex;
3134
+ async function nodeHasOtherPartitions(indexName, removedPartitionId, deps) {
3135
+ let allocation;
3136
+ try {
3137
+ allocation = await deps.coordinator.getAllocation(indexName);
3138
+ } catch (_) {
3139
+ return true;
3122
3140
  }
3123
- function compact() {
3124
- if (headIndex > 0) {
3125
- entries = entries.slice(headIndex);
3126
- headIndex = 0;
3127
- }
3141
+ if (allocation === null) {
3142
+ return false;
3128
3143
  }
3129
- function findIndexBySeqNo(seqNo) {
3130
- let low = headIndex;
3131
- let high = entries.length - 1;
3132
- while (low <= high) {
3133
- const mid = low + high >>> 1;
3134
- const midSeqNo = entries[mid].seqNo;
3135
- if (midSeqNo < seqNo) {
3136
- low = mid + 1;
3137
- } else if (midSeqNo > seqNo) {
3138
- high = mid - 1;
3139
- } else {
3140
- return mid;
3141
- }
3144
+ for (const [partitionId, assignment] of allocation.assignments) {
3145
+ if (partitionId === removedPartitionId) {
3146
+ continue;
3142
3147
  }
3143
- return low;
3144
- }
3145
- function evictOldEntries() {
3146
- while (liveCount() > 0 && totalSizeBytes > retentionBytes) {
3147
- totalSizeBytes -= estimateEntrySize(entries[headIndex]);
3148
- entries[headIndex] = null;
3149
- headIndex += 1;
3148
+ if (assignment.primary === deps.nodeId) {
3149
+ return true;
3150
3150
  }
3151
- if (totalSizeBytes < 0) {
3152
- totalSizeBytes = 0;
3153
- }
3154
- if (headIndex >= 256 || headIndex > liveCount()) {
3155
- compact();
3156
- }
3157
- }
3158
- function storeEntry(entry) {
3159
- entries.push(entry);
3160
- if (entry.seqNo >= committedSeqNo) {
3161
- committedSeqNo = entry.seqNo;
3162
- committedPrimaryTerm = entry.primaryTerm;
3163
- }
3164
- totalSizeBytes += estimateEntrySize(entry);
3165
- evictOldEntries();
3166
- return { ...entry };
3167
- }
3168
- function readEntry(seqNo) {
3169
- if (liveCount() === 0) return void 0;
3170
- const idx = findIndexBySeqNo(seqNo);
3171
- if (idx < entries.length && entries[idx].seqNo === seqNo) {
3172
- return { ...entries[idx] };
3173
- }
3174
- return void 0;
3175
- }
3176
- function verifyEntryChecksum(entry) {
3177
- const expected = computeChecksum({
3178
- seqNo: entry.seqNo,
3179
- primaryTerm: entry.primaryTerm,
3180
- operation: entry.operation,
3181
- partitionId: entry.partitionId,
3182
- indexName: entry.indexName,
3183
- documentId: entry.documentId,
3184
- document: entry.document
3185
- });
3186
- return expected === entry.checksum;
3187
- }
3188
- return {
3189
- append(partial) {
3190
- const seqNo = nextSeqNo;
3191
- nextSeqNo += 1;
3192
- const entryWithoutChecksum = {
3193
- seqNo,
3194
- primaryTerm: partial.primaryTerm,
3195
- operation: partial.operation,
3196
- partitionId,
3197
- indexName: partial.indexName,
3198
- documentId: partial.documentId,
3199
- document: partial.document
3200
- };
3201
- const checksum = computeChecksum(entryWithoutChecksum);
3202
- const entry = { ...entryWithoutChecksum, checksum };
3203
- return storeEntry(entry);
3204
- },
3205
- appendCommitted(entry) {
3206
- if (!Number.isSafeInteger(entry.seqNo) || entry.seqNo < 1) {
3207
- throw new NarsilError(
3208
- ErrorCodes.REPLICATION_ENTRY_INVALID,
3209
- `Invalid replication sequence number ${entry.seqNo}`,
3210
- {
3211
- seqNo: entry.seqNo,
3212
- partitionId
3213
- }
3214
- );
3215
- }
3216
- if (entry.partitionId !== partitionId) {
3217
- throw new NarsilError(
3218
- ErrorCodes.REPLICATION_ENTRY_INVALID,
3219
- `Replication entry partition ${entry.partitionId} does not match log partition ${partitionId}`,
3220
- { entryPartitionId: entry.partitionId, logPartitionId: partitionId }
3221
- );
3222
- }
3223
- const existing = readEntry(entry.seqNo);
3224
- if (existing !== void 0) {
3225
- if (existing.checksum !== entry.checksum) {
3226
- throw new NarsilError(
3227
- ErrorCodes.REPLICATION_ENTRY_INVALID,
3228
- `Conflicting replication entry at sequence number ${entry.seqNo}`,
3229
- { seqNo: entry.seqNo, partitionId }
3230
- );
3231
- }
3232
- return existing;
3233
- }
3234
- if (entry.seqNo !== nextSeqNo) {
3235
- throw new NarsilError(
3236
- ErrorCodes.REPLICATION_ENTRY_INVALID,
3237
- `Out-of-order replication entry ${entry.seqNo}; expected ${nextSeqNo}`,
3238
- { seqNo: entry.seqNo, expectedSeqNo: nextSeqNo, partitionId }
3239
- );
3240
- }
3241
- if (!verifyEntryChecksum(entry)) {
3242
- throw new NarsilError(
3243
- ErrorCodes.REPLICATION_ENTRY_INVALID,
3244
- `Invalid checksum for replication entry ${entry.seqNo}`,
3245
- { seqNo: entry.seqNo, partitionId }
3246
- );
3247
- }
3248
- nextSeqNo = entry.seqNo + 1;
3249
- return storeEntry({ ...entry });
3250
- },
3251
- getEntriesFrom(fromSeqNo) {
3252
- if (liveCount() === 0) return [];
3253
- const idx = findIndexBySeqNo(fromSeqNo);
3254
- if (idx >= entries.length) return [];
3255
- return entries.slice(idx).map((e) => ({ ...e }));
3256
- },
3257
- getEntry(seqNo) {
3258
- return readEntry(seqNo);
3259
- },
3260
- verifyChecksum(entry) {
3261
- return verifyEntryChecksum(entry);
3262
- },
3263
- get committedSeqNo() {
3264
- return committedSeqNo;
3265
- },
3266
- get committedPrimaryTerm() {
3267
- return committedPrimaryTerm;
3268
- },
3269
- get oldestSeqNo() {
3270
- if (liveCount() === 0) return void 0;
3271
- return entries[headIndex].seqNo;
3272
- },
3273
- get newestSeqNo() {
3274
- if (liveCount() === 0) return void 0;
3275
- return entries[entries.length - 1].seqNo;
3276
- },
3277
- get entryCount() {
3278
- return liveCount();
3279
- },
3280
- get sizeBytes() {
3281
- return totalSizeBytes;
3282
- },
3283
- clear() {
3284
- entries = [];
3285
- headIndex = 0;
3286
- totalSizeBytes = 0;
3287
- committedSeqNo = nextSeqNo - 1;
3288
- }
3289
- };
3290
- }
3291
-
3292
- // src/distribution/cluster-node/bootstrap-cleanup.ts
3293
- async function cleanupRemovedPartition(indexName, partitionId, deps) {
3294
- try {
3295
- const existing = deps.engine.listIndexes().find((idx) => idx.name === indexName);
3296
- if (existing === void 0) {
3297
- return;
3298
- }
3299
- if (await nodeHasOtherPartitions(indexName, partitionId, deps)) {
3300
- return;
3301
- }
3302
- await deps.engine.dropIndex(indexName);
3303
- } catch (err) {
3304
- if (deps.onError === void 0) {
3305
- return;
3306
- }
3307
- if (err instanceof NarsilError && err.code === ErrorCodes.INDEX_NOT_FOUND) {
3308
- return;
3309
- }
3310
- const cause = err instanceof Error ? err.message : String(err);
3311
- deps.onError(
3312
- new NarsilError(
3313
- ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED,
3314
- `failed to drop local index '${indexName}' after partition removal: ${cause}`,
3315
- { indexName, partitionId, cause }
3316
- )
3317
- );
3318
- }
3319
- }
3320
- async function nodeHasOtherPartitions(indexName, removedPartitionId, deps) {
3321
- let allocation;
3322
- try {
3323
- allocation = await deps.coordinator.getAllocation(indexName);
3324
- } catch (_) {
3325
- return true;
3326
- }
3327
- if (allocation === null) {
3328
- return false;
3329
- }
3330
- for (const [partitionId, assignment] of allocation.assignments) {
3331
- if (partitionId === removedPartitionId) {
3332
- continue;
3333
- }
3334
- if (assignment.primary === deps.nodeId) {
3335
- return true;
3336
- }
3337
- if (assignment.replicas.includes(deps.nodeId)) {
3338
- return true;
3151
+ if (assignment.replicas.includes(deps.nodeId)) {
3152
+ return true;
3339
3153
  }
3340
3154
  }
3341
3155
  return false;
@@ -3662,6 +3476,8 @@ function toStreamFailure(err, defaultCode, defaultMessage) {
3662
3476
  details: { cause: message }
3663
3477
  };
3664
3478
  }
3479
+
3480
+ // src/distribution/cluster-node/bootstrap-fetch.ts
3665
3481
  var CAPACITY_EXHAUSTED_BACKOFF_BASE_MS = 100;
3666
3482
  var CAPACITY_EXHAUSTED_BACKOFF_MAX_MS = 500;
3667
3483
  var RETRY_ANY_TARGET_CODES = /* @__PURE__ */ new Set([
@@ -3963,10 +3779,10 @@ function validateArguments(indexName, partitionId, primaryNodeId) {
3963
3779
  }
3964
3780
  return null;
3965
3781
  }
3966
- async function resolveTransportTargets(indexName, primaryNodeId, resolveNodeTargets2) {
3782
+ async function resolveTransportTargets(indexName, primaryNodeId, resolveNodeTargets3) {
3967
3783
  let targets;
3968
3784
  try {
3969
- targets = await resolveNodeTargets2(primaryNodeId);
3785
+ targets = await resolveNodeTargets3(primaryNodeId);
3970
3786
  } catch (err) {
3971
3787
  const cause = err instanceof Error ? err.message : String(err);
3972
3788
  return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_NO_TARGETS, `resolveNodeTargets failed: ${cause}`, {
@@ -4128,7 +3944,7 @@ async function dropExistingIndex(engine, indexName, primaryNodeId) {
4128
3944
  }
4129
3945
  }
4130
3946
 
4131
- // src/distribution/cluster-node/bootstrap-sync.ts
3947
+ // src/distribution/cluster-node/bootstrap-sync/state.ts
4132
3948
  var DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS = 6e5;
4133
3949
  function createBootstrapSyncState() {
4134
3950
  return {
@@ -4160,40 +3976,6 @@ function clearBootstrapSyncIndex(state, indexName, partitionId) {
4160
3976
  function hasCompletedBootstrapSync(state, indexName, partitionId) {
4161
3977
  return state.completed.has(entryKey(indexName, partitionId));
4162
3978
  }
4163
- async function runBootstrapSync(state, indexName, partitionId, primaryNodeId, deps) {
4164
- const validationError = validateArguments(indexName, partitionId, primaryNodeId);
4165
- if (validationError !== null) {
4166
- surfaceError(deps, indexName, primaryNodeId, validationError);
4167
- return false;
4168
- }
4169
- const key = entryKey(indexName, partitionId);
4170
- if (state.completed.has(key)) {
4171
- return true;
4172
- }
4173
- const existing = state.inFlight.get(key);
4174
- if (existing !== void 0) {
4175
- const result = await Promise.race([existing.promise, existing.abortPromise]);
4176
- if (result === ABORT_SENTINEL) {
4177
- return false;
4178
- }
4179
- return result;
4180
- }
4181
- const entry = createEntry(state, indexName, partitionId, key);
4182
- entry.promise = executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps).finally(() => {
4183
- const current = state.inFlight.get(key);
4184
- if (current === entry) {
4185
- state.inFlight.delete(key);
4186
- }
4187
- if (!state.completed.has(key) && !state.inFlight.has(key)) {
4188
- const currentGen = state.generations.get(key);
4189
- if (currentGen !== void 0 && currentGen !== entry.generation) {
4190
- state.generations.delete(key);
4191
- }
4192
- }
4193
- });
4194
- state.inFlight.set(key, entry);
4195
- return entry.promise;
4196
- }
4197
3979
  function createEntry(state, indexName, partitionId, key) {
4198
3980
  const startGeneration = state.generations.get(key) ?? 0;
4199
3981
  let abortResolve = () => {
@@ -4211,401 +3993,100 @@ function createEntry(state, indexName, partitionId, key) {
4211
3993
  abortPromise
4212
3994
  };
4213
3995
  }
4214
- async function executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps) {
4215
- const deadlineMs = deps.deadlineMs ?? DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS;
4216
- const deadline = Date.now() + deadlineMs;
4217
- const key = entryKey(indexName, partitionId);
4218
- const abortCheck = () => {
4219
- const currentGeneration = state.generations.get(key) ?? 0;
4220
- if (currentGeneration !== entry.generation) {
4221
- entry.aborted = true;
4222
- }
4223
- return entry.aborted;
4224
- };
4225
- if (hasLiveBootstrapSyncDeps(deps)) {
4226
- return executeLiveBootstrapSync(
4227
- state,
4228
- entry,
4229
- indexName,
4230
- partitionId,
4231
- primaryNodeId,
4232
- deps,
4233
- deadlineMs,
4234
- deadline,
4235
- abortCheck
4236
- );
4237
- }
4238
- const schemaResult = await fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps);
4239
- if (schemaResult === "aborted") {
4240
- surfaceAborted(deps, indexName, primaryNodeId);
4241
- return false;
4242
- }
4243
- if (schemaResult === null) {
3996
+ function anotherBootstrapOwnsKey(state, key, self) {
3997
+ const current = state.inFlight.get(key);
3998
+ if (current === void 0) {
4244
3999
  return false;
4245
4000
  }
4246
- if (abortCheck()) {
4247
- surfaceAborted(deps, indexName, primaryNodeId);
4248
- return false;
4249
- }
4250
- if (Date.now() >= deadline) {
4251
- surfaceError(
4252
- deps,
4253
- indexName,
4254
- primaryNodeId,
4255
- new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before fetching snapshot", {
4256
- indexName,
4257
- deadlineMs
4258
- })
4259
- );
4260
- return false;
4261
- }
4262
- const targetsResult = await resolveTransportTargets(indexName, primaryNodeId, deps.resolveNodeTargets);
4263
- if (targetsResult instanceof NarsilError) {
4264
- surfaceError(deps, indexName, primaryNodeId, targetsResult);
4265
- return false;
4266
- }
4267
- const fetchDeps = { transport: deps.transport, sourceNodeId: deps.sourceNodeId };
4268
- const fetchResult = await fetchSnapshotFromAnyTarget(
4269
- indexName,
4270
- primaryNodeId,
4271
- targetsResult,
4272
- deadline,
4273
- fetchDeps,
4274
- abortCheck,
4275
- partitionId
4276
- );
4277
- if (!fetchResult.ok) {
4278
- if (fetchResult.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
4279
- surfaceAborted(deps, indexName, primaryNodeId);
4280
- return false;
4281
- }
4282
- surfaceError(
4283
- deps,
4284
- indexName,
4285
- primaryNodeId,
4286
- new NarsilError(fetchResult.code, fetchResult.message, {
4287
- indexName,
4288
- primaryNodeId,
4289
- ...fetchResult.details
4290
- })
4291
- );
4292
- return false;
4293
- }
4294
- if (abortCheck()) {
4295
- surfaceAborted(deps, indexName, primaryNodeId);
4296
- return false;
4297
- }
4298
- const restoreSucceeded = await applyRestore(
4299
- state,
4300
- entry,
4301
- indexName,
4302
- primaryNodeId,
4303
- fetchResult.bytes,
4304
- schemaResult,
4305
- deadline,
4306
- deps
4307
- );
4308
- if (!restoreSucceeded) {
4309
- return false;
4310
- }
4311
- if (abortCheck()) {
4312
- if (!anotherBootstrapOwnsKey(state, key, entry)) {
4313
- await dropRestoredIndexQuietly(deps.engine, indexName, deps);
4314
- }
4315
- surfaceAborted(deps, indexName, primaryNodeId);
4316
- return false;
4317
- }
4318
- deps.onSnapshotApplied?.(indexName, partitionId, fetchResult.header);
4319
- state.completed.add(key);
4320
- return true;
4321
- }
4322
- async function resolveLivePartitionCount(coordinator, indexName, partitionId, primaryNodeId, abortPromise) {
4323
- try {
4324
- const winner = await Promise.race([coordinator.getAllocation(indexName), abortPromise]);
4325
- if (winner === ABORT_SENTINEL) {
4326
- return "aborted";
4327
- }
4328
- const allocation = winner;
4329
- if (allocation === null) {
4330
- return {
4331
- error: new NarsilError(
4332
- ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
4333
- `coordinator has no allocation for index '${indexName}'`,
4334
- { indexName, primaryNodeId }
4335
- )
4336
- };
4337
- }
4338
- const assignment = allocation.assignments.get(partitionId);
4339
- if (assignment === void 0) {
4340
- return {
4341
- error: new NarsilError(
4342
- ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
4343
- `No assignment exists for partition ${partitionId} of index '${indexName}'`,
4344
- { indexName, partitionId, primaryNodeId }
4345
- )
4346
- };
4347
- }
4348
- if (assignment.primary !== primaryNodeId) {
4349
- return {
4350
- error: new NarsilError(
4351
- ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
4352
- `Partition ${partitionId} of index '${indexName}' is no longer primary on '${primaryNodeId}'`,
4353
- { indexName, partitionId, primaryNodeId, currentPrimary: assignment.primary }
4354
- )
4355
- };
4356
- }
4357
- return { partitionCount: allocation.assignments.size };
4358
- } catch (err) {
4359
- const cause = err instanceof Error ? err.message : String(err);
4360
- return {
4361
- error: new NarsilError(
4362
- ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
4363
- `coordinator.getAllocation failed: ${cause}`,
4364
- {
4365
- indexName,
4366
- partitionId,
4367
- primaryNodeId,
4368
- cause
4369
- }
4370
- )
4371
- };
4372
- }
4001
+ return current !== self;
4373
4002
  }
4374
- async function executeLiveBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps, deadlineMs, deadline, abortCheck) {
4375
- const schemaResult = await loadCoordinatorSchema(deps.coordinator, indexName, primaryNodeId, entry.abortPromise);
4376
- if (schemaResult === "aborted") {
4377
- surfaceAborted(deps, indexName, primaryNodeId);
4378
- return false;
4379
- }
4380
- if ("error" in schemaResult) {
4381
- surfaceError(deps, indexName, primaryNodeId, schemaResult.error);
4382
- return false;
4383
- }
4384
- if (abortCheck()) {
4385
- surfaceAborted(deps, indexName, primaryNodeId);
4386
- return false;
4387
- }
4388
- const allocationResult = await resolveLivePartitionCount(
4389
- deps.coordinator,
4390
- indexName,
4391
- partitionId,
4392
- primaryNodeId,
4393
- entry.abortPromise
4394
- );
4395
- if (allocationResult === "aborted") {
4396
- surfaceAborted(deps, indexName, primaryNodeId);
4397
- return false;
4398
- }
4399
- if ("error" in allocationResult) {
4400
- surfaceError(deps, indexName, primaryNodeId, allocationResult.error);
4401
- return false;
4402
- }
4403
- if (Date.now() >= deadline) {
4404
- surfaceError(
4405
- deps,
4406
- indexName,
4407
- primaryNodeId,
4408
- new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before starting live sync", {
4409
- indexName,
4410
- deadlineMs
4411
- })
4412
- );
4413
- return false;
4414
- }
4415
- const targetsResult = await resolveTransportTargets(indexName, primaryNodeId, deps.resolveNodeTargets);
4416
- if (targetsResult instanceof NarsilError) {
4417
- surfaceError(deps, indexName, primaryNodeId, targetsResult);
4418
- return false;
4419
- }
4420
- const syncResult = await syncFromAnyTarget(
4421
- state,
4422
- entry,
4423
- indexName,
4424
- partitionId,
4425
- primaryNodeId,
4426
- targetsResult,
4427
- schemaResult.schema,
4428
- allocationResult.partitionCount,
4429
- deadline,
4430
- deps,
4431
- abortCheck
4432
- );
4433
- if (!syncResult.ok) {
4434
- if (syncResult.error.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
4435
- surfaceAborted(deps, indexName, primaryNodeId);
4436
- return false;
4437
- }
4438
- if (syncResult.error.details.alreadySurfaced !== true) {
4439
- surfaceError(deps, indexName, primaryNodeId, syncResult.error);
4440
- }
4441
- return false;
4442
- }
4443
- if (abortCheck()) {
4444
- surfaceAborted(deps, indexName, primaryNodeId);
4445
- return false;
4446
- }
4447
- if (syncResult.snapshotHeader !== null) {
4448
- deps.onSnapshotApplied?.(indexName, partitionId, syncResult.snapshotHeader);
4003
+
4004
+ // src/distribution/cluster-node/bootstrap-sync/entries.ts
4005
+ async function applySyncEntries(indexName, partitionId, entriesPayloads, expectedFirstSeqNo, localPrimaryTerm, deps) {
4006
+ const terminalError = validateSyncEntryBatchTermination(entriesPayloads);
4007
+ if (terminalError !== null) {
4008
+ return { ok: false, error: terminalError };
4449
4009
  }
4450
- state.completed.add(entryKey(indexName, partitionId));
4451
- return true;
4452
- }
4453
- async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNodeId, targets, coordinatorSchema, partitionCount, deadline, deps, abortCheck) {
4454
- let lastError = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED, "no live sync attempts were made", {
4455
- indexName,
4456
- partitionId,
4457
- primaryNodeId
4458
- });
4459
- const attempted = /* @__PURE__ */ new Set();
4460
- for (const target of targets) {
4461
- if (attempted.has(target)) {
4462
- continue;
4463
- }
4464
- attempted.add(target);
4465
- if (abortCheck()) {
4466
- return {
4467
- ok: false,
4468
- error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_ABORTED, "bootstrap sync aborted before next live target", {
4469
- indexName,
4470
- partitionId,
4471
- primaryNodeId,
4472
- target
4473
- })
4474
- };
4475
- }
4476
- if (Date.now() >= deadline) {
4477
- return {
4478
- ok: false,
4479
- error: new NarsilError(
4480
- ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
4481
- "bootstrap sync exceeded deadline while iterating live targets",
4482
- { indexName, partitionId, primaryNodeId, targets }
4483
- )
4484
- };
4485
- }
4486
- const attempt = await syncFromTarget(
4487
- state,
4488
- entry,
4489
- indexName,
4490
- partitionId,
4491
- target,
4492
- coordinatorSchema,
4493
- partitionCount,
4494
- deadline,
4495
- deps,
4496
- abortCheck
4497
- );
4498
- if (attempt.ok) {
4499
- return attempt;
4500
- }
4501
- lastError = new NarsilError(attempt.error.code, attempt.error.message, {
4502
- ...attempt.error.details,
4503
- target
4504
- });
4505
- if (!isTransientFailure(attempt.error.code, attempt.error.details)) {
4506
- return { ok: false, error: lastError };
4010
+ const log = deps.getReplicationLog(indexName, partitionId);
4011
+ let expectedSeqNo = expectedFirstSeqNo;
4012
+ let entriesApplied = 0;
4013
+ for (const payload of entriesPayloads) {
4014
+ for (const entry of payload.entries) {
4015
+ const entryError = validateSyncEntryForApply(entry, indexName, partitionId, expectedSeqNo, localPrimaryTerm, log);
4016
+ if (entryError !== null) {
4017
+ return { ok: false, error: entryError };
4018
+ }
4019
+ try {
4020
+ await deps.applyReplicationEntry(entry);
4021
+ log.appendCommitted(entry);
4022
+ } catch (err) {
4023
+ const cause = err instanceof Error ? err.message : String(err);
4024
+ return {
4025
+ ok: false,
4026
+ error: new NarsilError(ErrorCodes.REPLICATION_ENTRY_INVALID, `failed to apply sync entry: ${cause}`, {
4027
+ indexName,
4028
+ partitionId,
4029
+ seqNo: entry.seqNo,
4030
+ cause
4031
+ })
4032
+ };
4033
+ }
4034
+ expectedSeqNo = entry.seqNo + 1;
4035
+ entriesApplied += 1;
4507
4036
  }
4508
4037
  }
4509
- return { ok: false, error: lastError };
4510
- }
4511
- async function syncFromTarget(state, entry, indexName, partitionId, target, coordinatorSchema, partitionCount, deadline, deps, abortCheck) {
4512
- const logState = readLocalLogState(indexName, partitionId, deps);
4513
- const requestPayload = {
4514
- indexName,
4515
- partitionId,
4516
- lastSeqNo: logState.lastSeqNo,
4517
- lastPrimaryTerm: logState.lastPrimaryTerm
4518
- };
4519
- const request = {
4520
- type: ReplicationMessageTypes.SYNC_REQUEST,
4521
- sourceId: deps.sourceNodeId,
4522
- requestId: generateId(),
4523
- payload: encode(requestPayload)
4038
+ return {
4039
+ ok: true,
4040
+ entriesApplied,
4041
+ newSeqNo: expectedSeqNo - 1
4524
4042
  };
4525
- const frameState = createLiveSyncFrameState(indexName, partitionId);
4526
- const remainingMs = deadline - Date.now();
4527
- if (remainingMs <= 0) {
4528
- return {
4529
- ok: false,
4530
- error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "deadline expired before live sync stream", {
4531
- indexName,
4532
- partitionId,
4533
- target
4534
- })
4535
- };
4043
+ }
4044
+ function validateSyncEntryBatchTermination(entriesPayloads) {
4045
+ if (entriesPayloads.length === 0) {
4046
+ return null;
4536
4047
  }
4537
- try {
4538
- const streamWork = deps.transport.stream(target, request, (chunk) => {
4539
- handleLiveSyncFrame(frameState, chunk);
4540
- });
4541
- await withDeadline(streamWork, remainingMs, indexName, "live-sync");
4542
- } catch (err) {
4543
- const cause = err instanceof Error ? err.message : String(err);
4544
- return {
4545
- ok: false,
4546
- error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED, `live sync transport failed: ${cause}`, {
4547
- indexName,
4548
- partitionId,
4549
- target,
4550
- cause
4551
- })
4552
- };
4048
+ for (let i = 0; i < entriesPayloads.length - 1; i += 1) {
4049
+ if (entriesPayloads[i].isLast) {
4050
+ return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "sync_entries marked isLast before final batch");
4051
+ }
4553
4052
  }
4554
- if (frameState.error !== null) {
4555
- return { ok: false, error: frameState.error };
4053
+ if (!entriesPayloads[entriesPayloads.length - 1].isLast) {
4054
+ return new NarsilError(
4055
+ ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
4056
+ "live sync stream ended before final sync_entries batch"
4057
+ );
4556
4058
  }
4557
- if (abortCheck()) {
4558
- return {
4559
- ok: false,
4560
- error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_ABORTED, "bootstrap sync aborted after live stream", {
4561
- indexName,
4562
- partitionId,
4563
- target
4564
- })
4565
- };
4059
+ return null;
4060
+ }
4061
+ function validateSyncEntryForApply(entry, indexName, partitionId, expectedSeqNo, localPrimaryTerm, log) {
4062
+ if (entry.indexName !== indexName || entry.partitionId !== partitionId) {
4063
+ return new NarsilError(
4064
+ ErrorCodes.REPLICATION_ENTRY_INVALID,
4065
+ "replication entry scope does not match bootstrap target",
4066
+ {
4067
+ expectedIndexName: indexName,
4068
+ expectedPartitionId: partitionId,
4069
+ entryIndexName: entry.indexName,
4070
+ entryPartitionId: entry.partitionId
4071
+ }
4072
+ );
4566
4073
  }
4567
- if (frameState.sawSnapshotFrame) {
4568
- return applyLiveSnapshotSync(
4569
- state,
4570
- entry,
4571
- indexName,
4572
- partitionId,
4573
- target,
4574
- frameState,
4575
- coordinatorSchema,
4576
- partitionCount,
4577
- deadline,
4578
- deps
4074
+ if (entry.seqNo !== expectedSeqNo) {
4075
+ return new NarsilError(
4076
+ ErrorCodes.REPLICATION_ENTRY_INVALID,
4077
+ `Out-of-order sync entry ${entry.seqNo}; expected ${expectedSeqNo}`,
4078
+ { indexName, partitionId, seqNo: entry.seqNo, expectedSeqNo }
4579
4079
  );
4580
4080
  }
4581
- if (frameState.syncEntries.length > 0) {
4582
- return applyLiveIncrementalSync(
4583
- indexName,
4584
- partitionId,
4585
- target,
4586
- frameState.syncEntries,
4587
- logState.lastSeqNo + 1,
4588
- logState.lastPrimaryTerm,
4589
- coordinatorSchema,
4590
- partitionCount,
4591
- deps
4081
+ const validation = validateReplicationEntry(entry, localPrimaryTerm, log);
4082
+ if (!validation.valid) {
4083
+ return new NarsilError(
4084
+ validation.error ?? ErrorCodes.REPLICATION_ENTRY_INVALID,
4085
+ `Invalid sync entry ${entry.seqNo}`,
4086
+ { indexName, partitionId, seqNo: entry.seqNo }
4592
4087
  );
4593
4088
  }
4594
- return {
4595
- ok: false,
4596
- error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_CHUNK_MISSING, "live sync stream ended without sync frames", {
4597
- indexName,
4598
- partitionId,
4599
- target
4600
- })
4601
- };
4602
- }
4603
- function readLocalLogState(indexName, partitionId, deps) {
4604
- const log = deps.getReplicationLog(indexName, partitionId);
4605
- const lastSeqNo = log.committedSeqNo;
4606
- const lastPrimaryTerm = log.committedPrimaryTerm;
4607
- const newestEntry = log.getEntry(lastSeqNo);
4608
- return { lastSeqNo, lastPrimaryTerm: newestEntry?.primaryTerm ?? lastPrimaryTerm };
4089
+ return null;
4609
4090
  }
4610
4091
  function createLiveSyncFrameState(indexName, partitionId) {
4611
4092
  return {
@@ -4679,6 +4160,11 @@ function decodeSyncEntriesRecord(record) {
4679
4160
  }
4680
4161
  return { entries, isLast: record.isLast };
4681
4162
  }
4163
+ function streamFailureToError(failure) {
4164
+ return new NarsilError(failure.code, failure.message, failure.details);
4165
+ }
4166
+
4167
+ // src/distribution/cluster-node/bootstrap-sync/apply.ts
4682
4168
  async function applyLiveIncrementalSync(indexName, partitionId, target, entriesPayloads, expectedFirstSeqNo, localPrimaryTerm, coordinatorSchema, partitionCount, deps) {
4683
4169
  const ensureResult = await ensureLocalIndexForIncremental(indexName, target, coordinatorSchema, partitionCount, deps);
4684
4170
  if (ensureResult instanceof NarsilError) {
@@ -4838,136 +4324,334 @@ async function ensureLocalIndexForIncremental(indexName, primaryNodeId, coordina
4838
4324
  const cause = err instanceof Error ? err.message : String(err);
4839
4325
  return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, `engine.createIndex failed: ${cause}`, {
4840
4326
  indexName,
4841
- primaryNodeId,
4842
- cause
4843
- });
4327
+ primaryNodeId,
4328
+ cause
4329
+ });
4330
+ }
4331
+ return { created: true };
4332
+ }
4333
+ function validateLocalSchema(engine, indexName, primaryNodeId, coordinatorSchema) {
4334
+ const schemaError = validateRestoredSchema(engine, indexName, primaryNodeId, coordinatorSchema);
4335
+ if (schemaError !== null) {
4336
+ return schemaError;
4337
+ }
4338
+ return { created: false };
4339
+ }
4340
+ function validateLocalPartitionCount(engine, indexName, primaryNodeId, partitionCount) {
4341
+ let localPartitionCount;
4342
+ try {
4343
+ localPartitionCount = engine.getStats(indexName).partitionCount;
4344
+ } catch (err) {
4345
+ const cause = err instanceof Error ? err.message : String(err);
4346
+ return new NarsilError(
4347
+ ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED,
4348
+ `failed to read local partition count for '${indexName}': ${cause}`,
4349
+ { indexName, primaryNodeId, cause }
4350
+ );
4351
+ }
4352
+ if (localPartitionCount !== partitionCount) {
4353
+ return new NarsilError(
4354
+ ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED,
4355
+ `local partition count ${localPartitionCount} disagrees with coordinator partition count ${partitionCount}`,
4356
+ { indexName, primaryNodeId, localPartitionCount, partitionCount }
4357
+ );
4358
+ }
4359
+ return { created: false };
4360
+ }
4361
+
4362
+ // src/distribution/cluster-node/bootstrap-sync/targets.ts
4363
+ function readLocalLogState(indexName, partitionId, deps) {
4364
+ const log = deps.getReplicationLog(indexName, partitionId);
4365
+ const lastSeqNo = log.committedSeqNo;
4366
+ const lastPrimaryTerm = log.committedPrimaryTerm;
4367
+ const newestEntry = log.getEntry(lastSeqNo);
4368
+ return { lastSeqNo, lastPrimaryTerm: newestEntry?.primaryTerm ?? lastPrimaryTerm };
4369
+ }
4370
+ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNodeId, targets, coordinatorSchema, partitionCount, deadline, deps, abortCheck) {
4371
+ let lastError = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED, "no live sync attempts were made", {
4372
+ indexName,
4373
+ partitionId,
4374
+ primaryNodeId
4375
+ });
4376
+ const attempted = /* @__PURE__ */ new Set();
4377
+ for (const target of targets) {
4378
+ if (attempted.has(target)) {
4379
+ continue;
4380
+ }
4381
+ attempted.add(target);
4382
+ if (abortCheck()) {
4383
+ return {
4384
+ ok: false,
4385
+ error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_ABORTED, "bootstrap sync aborted before next live target", {
4386
+ indexName,
4387
+ partitionId,
4388
+ primaryNodeId,
4389
+ target
4390
+ })
4391
+ };
4392
+ }
4393
+ if (Date.now() >= deadline) {
4394
+ return {
4395
+ ok: false,
4396
+ error: new NarsilError(
4397
+ ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
4398
+ "bootstrap sync exceeded deadline while iterating live targets",
4399
+ { indexName, partitionId, primaryNodeId, targets }
4400
+ )
4401
+ };
4402
+ }
4403
+ const attempt = await syncFromTarget(
4404
+ state,
4405
+ entry,
4406
+ indexName,
4407
+ partitionId,
4408
+ target,
4409
+ coordinatorSchema,
4410
+ partitionCount,
4411
+ deadline,
4412
+ deps,
4413
+ abortCheck
4414
+ );
4415
+ if (attempt.ok) {
4416
+ return attempt;
4417
+ }
4418
+ lastError = new NarsilError(attempt.error.code, attempt.error.message, {
4419
+ ...attempt.error.details,
4420
+ target
4421
+ });
4422
+ if (!isTransientFailure(attempt.error.code, attempt.error.details)) {
4423
+ return { ok: false, error: lastError };
4424
+ }
4425
+ }
4426
+ return { ok: false, error: lastError };
4427
+ }
4428
+ async function syncFromTarget(state, entry, indexName, partitionId, target, coordinatorSchema, partitionCount, deadline, deps, abortCheck) {
4429
+ const logState = readLocalLogState(indexName, partitionId, deps);
4430
+ const requestPayload = {
4431
+ indexName,
4432
+ partitionId,
4433
+ lastSeqNo: logState.lastSeqNo,
4434
+ lastPrimaryTerm: logState.lastPrimaryTerm
4435
+ };
4436
+ const request = {
4437
+ type: ReplicationMessageTypes.SYNC_REQUEST,
4438
+ sourceId: deps.sourceNodeId,
4439
+ requestId: generateId(),
4440
+ payload: encode(requestPayload)
4441
+ };
4442
+ const frameState = createLiveSyncFrameState(indexName, partitionId);
4443
+ const remainingMs = deadline - Date.now();
4444
+ if (remainingMs <= 0) {
4445
+ return {
4446
+ ok: false,
4447
+ error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "deadline expired before live sync stream", {
4448
+ indexName,
4449
+ partitionId,
4450
+ target
4451
+ })
4452
+ };
4453
+ }
4454
+ try {
4455
+ const streamWork = deps.transport.stream(target, request, (chunk) => {
4456
+ handleLiveSyncFrame(frameState, chunk);
4457
+ });
4458
+ await withDeadline(streamWork, remainingMs, indexName, "live-sync");
4459
+ } catch (err) {
4460
+ const cause = err instanceof Error ? err.message : String(err);
4461
+ return {
4462
+ ok: false,
4463
+ error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED, `live sync transport failed: ${cause}`, {
4464
+ indexName,
4465
+ partitionId,
4466
+ target,
4467
+ cause
4468
+ })
4469
+ };
4470
+ }
4471
+ if (frameState.error !== null) {
4472
+ return { ok: false, error: frameState.error };
4473
+ }
4474
+ if (abortCheck()) {
4475
+ return {
4476
+ ok: false,
4477
+ error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_ABORTED, "bootstrap sync aborted after live stream", {
4478
+ indexName,
4479
+ partitionId,
4480
+ target
4481
+ })
4482
+ };
4483
+ }
4484
+ if (frameState.sawSnapshotFrame) {
4485
+ return applyLiveSnapshotSync(
4486
+ state,
4487
+ entry,
4488
+ indexName,
4489
+ partitionId,
4490
+ target,
4491
+ frameState,
4492
+ coordinatorSchema,
4493
+ partitionCount,
4494
+ deadline,
4495
+ deps
4496
+ );
4844
4497
  }
4845
- return { created: true };
4846
- }
4847
- function validateLocalSchema(engine, indexName, primaryNodeId, coordinatorSchema) {
4848
- const schemaError = validateRestoredSchema(engine, indexName, primaryNodeId, coordinatorSchema);
4849
- if (schemaError !== null) {
4850
- return schemaError;
4498
+ if (frameState.syncEntries.length > 0) {
4499
+ return applyLiveIncrementalSync(
4500
+ indexName,
4501
+ partitionId,
4502
+ target,
4503
+ frameState.syncEntries,
4504
+ logState.lastSeqNo + 1,
4505
+ logState.lastPrimaryTerm,
4506
+ coordinatorSchema,
4507
+ partitionCount,
4508
+ deps
4509
+ );
4851
4510
  }
4852
- return { created: false };
4511
+ return {
4512
+ ok: false,
4513
+ error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_CHUNK_MISSING, "live sync stream ended without sync frames", {
4514
+ indexName,
4515
+ partitionId,
4516
+ target
4517
+ })
4518
+ };
4853
4519
  }
4854
- function validateLocalPartitionCount(engine, indexName, primaryNodeId, partitionCount) {
4855
- let localPartitionCount;
4520
+
4521
+ // src/distribution/cluster-node/bootstrap-sync/live.ts
4522
+ async function resolveLivePartitionCount(coordinator, indexName, partitionId, primaryNodeId, abortPromise) {
4856
4523
  try {
4857
- localPartitionCount = engine.getStats(indexName).partitionCount;
4524
+ const winner = await Promise.race([coordinator.getAllocation(indexName), abortPromise]);
4525
+ if (winner === ABORT_SENTINEL) {
4526
+ return "aborted";
4527
+ }
4528
+ const allocation = winner;
4529
+ if (allocation === null) {
4530
+ return {
4531
+ error: new NarsilError(
4532
+ ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
4533
+ `coordinator has no allocation for index '${indexName}'`,
4534
+ { indexName, primaryNodeId }
4535
+ )
4536
+ };
4537
+ }
4538
+ const assignment = allocation.assignments.get(partitionId);
4539
+ if (assignment === void 0) {
4540
+ return {
4541
+ error: new NarsilError(
4542
+ ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
4543
+ `No assignment exists for partition ${partitionId} of index '${indexName}'`,
4544
+ { indexName, partitionId, primaryNodeId }
4545
+ )
4546
+ };
4547
+ }
4548
+ if (assignment.primary !== primaryNodeId) {
4549
+ return {
4550
+ error: new NarsilError(
4551
+ ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
4552
+ `Partition ${partitionId} of index '${indexName}' is no longer primary on '${primaryNodeId}'`,
4553
+ { indexName, partitionId, primaryNodeId, currentPrimary: assignment.primary }
4554
+ )
4555
+ };
4556
+ }
4557
+ return { partitionCount: allocation.assignments.size };
4858
4558
  } catch (err) {
4859
4559
  const cause = err instanceof Error ? err.message : String(err);
4860
- return new NarsilError(
4861
- ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED,
4862
- `failed to read local partition count for '${indexName}': ${cause}`,
4863
- { indexName, primaryNodeId, cause }
4864
- );
4865
- }
4866
- if (localPartitionCount !== partitionCount) {
4867
- return new NarsilError(
4868
- ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED,
4869
- `local partition count ${localPartitionCount} disagrees with coordinator partition count ${partitionCount}`,
4870
- { indexName, primaryNodeId, localPartitionCount, partitionCount }
4871
- );
4560
+ return {
4561
+ error: new NarsilError(
4562
+ ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
4563
+ `coordinator.getAllocation failed: ${cause}`,
4564
+ {
4565
+ indexName,
4566
+ partitionId,
4567
+ primaryNodeId,
4568
+ cause
4569
+ }
4570
+ )
4571
+ };
4872
4572
  }
4873
- return { created: false };
4874
4573
  }
4875
- async function applySyncEntries(indexName, partitionId, entriesPayloads, expectedFirstSeqNo, localPrimaryTerm, deps) {
4876
- const terminalError = validateSyncEntryBatchTermination(entriesPayloads);
4877
- if (terminalError !== null) {
4878
- return { ok: false, error: terminalError };
4574
+ async function executeLiveBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps, deadlineMs, deadline, abortCheck) {
4575
+ const schemaResult = await loadCoordinatorSchema(deps.coordinator, indexName, primaryNodeId, entry.abortPromise);
4576
+ if (schemaResult === "aborted") {
4577
+ surfaceAborted(deps, indexName, primaryNodeId);
4578
+ return false;
4879
4579
  }
4880
- const log = deps.getReplicationLog(indexName, partitionId);
4881
- let expectedSeqNo = expectedFirstSeqNo;
4882
- let entriesApplied = 0;
4883
- for (const payload of entriesPayloads) {
4884
- for (const entry of payload.entries) {
4885
- const entryError = validateSyncEntryForApply(entry, indexName, partitionId, expectedSeqNo, localPrimaryTerm, log);
4886
- if (entryError !== null) {
4887
- return { ok: false, error: entryError };
4888
- }
4889
- try {
4890
- await deps.applyReplicationEntry(entry);
4891
- log.appendCommitted(entry);
4892
- } catch (err) {
4893
- const cause = err instanceof Error ? err.message : String(err);
4894
- return {
4895
- ok: false,
4896
- error: new NarsilError(ErrorCodes.REPLICATION_ENTRY_INVALID, `failed to apply sync entry: ${cause}`, {
4897
- indexName,
4898
- partitionId,
4899
- seqNo: entry.seqNo,
4900
- cause
4901
- })
4902
- };
4903
- }
4904
- expectedSeqNo = entry.seqNo + 1;
4905
- entriesApplied += 1;
4906
- }
4580
+ if ("error" in schemaResult) {
4581
+ surfaceError(deps, indexName, primaryNodeId, schemaResult.error);
4582
+ return false;
4907
4583
  }
4908
- return {
4909
- ok: true,
4910
- entriesApplied,
4911
- newSeqNo: expectedSeqNo - 1
4912
- };
4913
- }
4914
- function validateSyncEntryBatchTermination(entriesPayloads) {
4915
- if (entriesPayloads.length === 0) {
4916
- return null;
4584
+ if (abortCheck()) {
4585
+ surfaceAborted(deps, indexName, primaryNodeId);
4586
+ return false;
4917
4587
  }
4918
- for (let i = 0; i < entriesPayloads.length - 1; i += 1) {
4919
- if (entriesPayloads[i].isLast) {
4920
- return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "sync_entries marked isLast before final batch");
4921
- }
4588
+ const allocationResult = await resolveLivePartitionCount(
4589
+ deps.coordinator,
4590
+ indexName,
4591
+ partitionId,
4592
+ primaryNodeId,
4593
+ entry.abortPromise
4594
+ );
4595
+ if (allocationResult === "aborted") {
4596
+ surfaceAborted(deps, indexName, primaryNodeId);
4597
+ return false;
4922
4598
  }
4923
- if (!entriesPayloads[entriesPayloads.length - 1].isLast) {
4924
- return new NarsilError(
4925
- ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
4926
- "live sync stream ended before final sync_entries batch"
4927
- );
4599
+ if ("error" in allocationResult) {
4600
+ surfaceError(deps, indexName, primaryNodeId, allocationResult.error);
4601
+ return false;
4928
4602
  }
4929
- return null;
4930
- }
4931
- function validateSyncEntryForApply(entry, indexName, partitionId, expectedSeqNo, localPrimaryTerm, log) {
4932
- if (entry.indexName !== indexName || entry.partitionId !== partitionId) {
4933
- return new NarsilError(
4934
- ErrorCodes.REPLICATION_ENTRY_INVALID,
4935
- "replication entry scope does not match bootstrap target",
4936
- {
4937
- expectedIndexName: indexName,
4938
- expectedPartitionId: partitionId,
4939
- entryIndexName: entry.indexName,
4940
- entryPartitionId: entry.partitionId
4941
- }
4603
+ if (Date.now() >= deadline) {
4604
+ surfaceError(
4605
+ deps,
4606
+ indexName,
4607
+ primaryNodeId,
4608
+ new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before starting live sync", {
4609
+ indexName,
4610
+ deadlineMs
4611
+ })
4942
4612
  );
4613
+ return false;
4943
4614
  }
4944
- if (entry.seqNo !== expectedSeqNo) {
4945
- return new NarsilError(
4946
- ErrorCodes.REPLICATION_ENTRY_INVALID,
4947
- `Out-of-order sync entry ${entry.seqNo}; expected ${expectedSeqNo}`,
4948
- { indexName, partitionId, seqNo: entry.seqNo, expectedSeqNo }
4949
- );
4615
+ const targetsResult = await resolveTransportTargets(indexName, primaryNodeId, deps.resolveNodeTargets);
4616
+ if (targetsResult instanceof NarsilError) {
4617
+ surfaceError(deps, indexName, primaryNodeId, targetsResult);
4618
+ return false;
4950
4619
  }
4951
- const validation = validateReplicationEntry(entry, localPrimaryTerm, log);
4952
- if (!validation.valid) {
4953
- return new NarsilError(
4954
- ErrorCodes.REPLICATION_ENTRY_INVALID,
4955
- `Invalid sync entry ${entry.seqNo}: ${validation.error ?? "unknown validation error"}`,
4956
- { indexName, partitionId, seqNo: entry.seqNo }
4957
- );
4620
+ const syncResult = await syncFromAnyTarget(
4621
+ state,
4622
+ entry,
4623
+ indexName,
4624
+ partitionId,
4625
+ primaryNodeId,
4626
+ targetsResult,
4627
+ schemaResult.schema,
4628
+ allocationResult.partitionCount,
4629
+ deadline,
4630
+ deps,
4631
+ abortCheck
4632
+ );
4633
+ if (!syncResult.ok) {
4634
+ if (syncResult.error.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
4635
+ surfaceAborted(deps, indexName, primaryNodeId);
4636
+ return false;
4637
+ }
4638
+ if (syncResult.error.details.alreadySurfaced !== true) {
4639
+ surfaceError(deps, indexName, primaryNodeId, syncResult.error);
4640
+ }
4641
+ return false;
4958
4642
  }
4959
- return null;
4960
- }
4961
- function streamFailureToError(failure) {
4962
- return new NarsilError(failure.code, failure.message, failure.details);
4963
- }
4964
- function anotherBootstrapOwnsKey(state, key, self) {
4965
- const current = state.inFlight.get(key);
4966
- if (current === void 0) {
4643
+ if (abortCheck()) {
4644
+ surfaceAborted(deps, indexName, primaryNodeId);
4967
4645
  return false;
4968
4646
  }
4969
- return current !== self;
4647
+ if (syncResult.snapshotHeader !== null) {
4648
+ deps.onSnapshotApplied?.(indexName, partitionId, syncResult.snapshotHeader);
4649
+ }
4650
+ state.completed.add(entryKey(indexName, partitionId));
4651
+ return true;
4970
4652
  }
4653
+
4654
+ // src/distribution/cluster-node/bootstrap-sync/snapshot-restore.ts
4971
4655
  async function fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps) {
4972
4656
  const dropError = await dropExistingIndex(deps.engine, indexName, primaryNodeId);
4973
4657
  if (dropError !== null) {
@@ -5028,6 +4712,150 @@ async function applyRestore(state, entry, indexName, primaryNodeId, bytes, coord
5028
4712
  return true;
5029
4713
  }
5030
4714
 
4715
+ // src/distribution/cluster-node/bootstrap-sync/index.ts
4716
+ async function runBootstrapSync(state, indexName, partitionId, primaryNodeId, deps) {
4717
+ const validationError = validateArguments(indexName, partitionId, primaryNodeId);
4718
+ if (validationError !== null) {
4719
+ surfaceError(deps, indexName, primaryNodeId, validationError);
4720
+ return false;
4721
+ }
4722
+ const key = entryKey(indexName, partitionId);
4723
+ if (state.completed.has(key)) {
4724
+ return true;
4725
+ }
4726
+ const existing = state.inFlight.get(key);
4727
+ if (existing !== void 0) {
4728
+ const result = await Promise.race([existing.promise, existing.abortPromise]);
4729
+ if (result === ABORT_SENTINEL) {
4730
+ return false;
4731
+ }
4732
+ return result;
4733
+ }
4734
+ const entry = createEntry(state, indexName, partitionId, key);
4735
+ entry.promise = executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps).finally(() => {
4736
+ const current = state.inFlight.get(key);
4737
+ if (current === entry) {
4738
+ state.inFlight.delete(key);
4739
+ }
4740
+ if (!state.completed.has(key) && !state.inFlight.has(key)) {
4741
+ const currentGen = state.generations.get(key);
4742
+ if (currentGen !== void 0 && currentGen !== entry.generation) {
4743
+ state.generations.delete(key);
4744
+ }
4745
+ }
4746
+ });
4747
+ state.inFlight.set(key, entry);
4748
+ return entry.promise;
4749
+ }
4750
+ async function executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps) {
4751
+ const deadlineMs = deps.deadlineMs ?? DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS;
4752
+ const deadline = Date.now() + deadlineMs;
4753
+ const key = entryKey(indexName, partitionId);
4754
+ const abortCheck = () => {
4755
+ const currentGeneration = state.generations.get(key) ?? 0;
4756
+ if (currentGeneration !== entry.generation) {
4757
+ entry.aborted = true;
4758
+ }
4759
+ return entry.aborted;
4760
+ };
4761
+ if (hasLiveBootstrapSyncDeps(deps)) {
4762
+ return executeLiveBootstrapSync(
4763
+ state,
4764
+ entry,
4765
+ indexName,
4766
+ partitionId,
4767
+ primaryNodeId,
4768
+ deps,
4769
+ deadlineMs,
4770
+ deadline,
4771
+ abortCheck
4772
+ );
4773
+ }
4774
+ const schemaResult = await fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps);
4775
+ if (schemaResult === "aborted") {
4776
+ surfaceAborted(deps, indexName, primaryNodeId);
4777
+ return false;
4778
+ }
4779
+ if (schemaResult === null) {
4780
+ return false;
4781
+ }
4782
+ if (abortCheck()) {
4783
+ surfaceAborted(deps, indexName, primaryNodeId);
4784
+ return false;
4785
+ }
4786
+ if (Date.now() >= deadline) {
4787
+ surfaceError(
4788
+ deps,
4789
+ indexName,
4790
+ primaryNodeId,
4791
+ new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before fetching snapshot", {
4792
+ indexName,
4793
+ deadlineMs
4794
+ })
4795
+ );
4796
+ return false;
4797
+ }
4798
+ const targetsResult = await resolveTransportTargets(indexName, primaryNodeId, deps.resolveNodeTargets);
4799
+ if (targetsResult instanceof NarsilError) {
4800
+ surfaceError(deps, indexName, primaryNodeId, targetsResult);
4801
+ return false;
4802
+ }
4803
+ const fetchDeps = { transport: deps.transport, sourceNodeId: deps.sourceNodeId };
4804
+ const fetchResult = await fetchSnapshotFromAnyTarget(
4805
+ indexName,
4806
+ primaryNodeId,
4807
+ targetsResult,
4808
+ deadline,
4809
+ fetchDeps,
4810
+ abortCheck,
4811
+ partitionId
4812
+ );
4813
+ if (!fetchResult.ok) {
4814
+ if (fetchResult.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
4815
+ surfaceAborted(deps, indexName, primaryNodeId);
4816
+ return false;
4817
+ }
4818
+ surfaceError(
4819
+ deps,
4820
+ indexName,
4821
+ primaryNodeId,
4822
+ new NarsilError(fetchResult.code, fetchResult.message, {
4823
+ indexName,
4824
+ primaryNodeId,
4825
+ ...fetchResult.details
4826
+ })
4827
+ );
4828
+ return false;
4829
+ }
4830
+ if (abortCheck()) {
4831
+ surfaceAborted(deps, indexName, primaryNodeId);
4832
+ return false;
4833
+ }
4834
+ const restoreSucceeded = await applyRestore(
4835
+ state,
4836
+ entry,
4837
+ indexName,
4838
+ primaryNodeId,
4839
+ fetchResult.bytes,
4840
+ schemaResult,
4841
+ deadline,
4842
+ deps
4843
+ );
4844
+ if (!restoreSucceeded) {
4845
+ return false;
4846
+ }
4847
+ if (abortCheck()) {
4848
+ if (!anotherBootstrapOwnsKey(state, key, entry)) {
4849
+ await dropRestoredIndexQuietly(deps.engine, indexName, deps);
4850
+ }
4851
+ surfaceAborted(deps, indexName, primaryNodeId);
4852
+ return false;
4853
+ }
4854
+ deps.onSnapshotApplied?.(indexName, partitionId, fetchResult.header);
4855
+ state.completed.add(key);
4856
+ return true;
4857
+ }
4858
+
5031
4859
  // src/distribution/cluster-node/local-engine.ts
5032
4860
  async function createClusterLocalEngine(config) {
5033
4861
  const core = createEngineCore(config);
@@ -5257,38 +5085,6 @@ async function applyReplicationEntry(core, entry) {
5257
5085
  requestId: `replicate-entry-remove-${entry.indexName}-${entry.partitionId}-${entry.seqNo}`
5258
5086
  });
5259
5087
  }
5260
- function decideSyncTier(log, replicaLastSeqNo) {
5261
- const oldest = log.oldestSeqNo;
5262
- if (oldest === void 0) {
5263
- if (log.committedSeqNo === 0) {
5264
- return replicaLastSeqNo === 0 ? "incremental" : "snapshot";
5265
- }
5266
- return replicaLastSeqNo >= log.committedSeqNo ? "incremental" : "snapshot";
5267
- }
5268
- if (oldest <= replicaLastSeqNo + 1) {
5269
- return "incremental";
5270
- }
5271
- return "snapshot";
5272
- }
5273
- function validateSyncRequest(decoded) {
5274
- if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
5275
- throw new Error("Invalid SyncRequestPayload: expected an object");
5276
- }
5277
- const record = decoded;
5278
- if (typeof record.indexName !== "string") {
5279
- throw new Error('Invalid SyncRequestPayload: "indexName" must be a string');
5280
- }
5281
- if (typeof record.partitionId !== "number" || !Number.isInteger(record.partitionId) || record.partitionId < 0) {
5282
- throw new Error('Invalid SyncRequestPayload: "partitionId" must be a non-negative integer');
5283
- }
5284
- if (typeof record.lastSeqNo !== "number" || !Number.isInteger(record.lastSeqNo) || record.lastSeqNo < 0) {
5285
- throw new Error('Invalid SyncRequestPayload: "lastSeqNo" must be a non-negative integer');
5286
- }
5287
- if (typeof record.lastPrimaryTerm !== "number" || !Number.isInteger(record.lastPrimaryTerm) || record.lastPrimaryTerm < 0) {
5288
- throw new Error('Invalid SyncRequestPayload: "lastPrimaryTerm" must be a non-negative integer');
5289
- }
5290
- return record;
5291
- }
5292
5088
 
5293
5089
  // src/distribution/cluster-node/snapshot-auth.ts
5294
5090
  var BOOTSTRAPPABLE_STATES = /* @__PURE__ */ new Set(["ACTIVE", "INITIALISING", "MIGRATING"]);
@@ -5319,133 +5115,35 @@ async function authorizeSnapshotRequest(coordinator, indexName, sourceId) {
5319
5115
  continue;
5320
5116
  }
5321
5117
  sawAssignment = true;
5322
- if (!BOOTSTRAPPABLE_STATES.has(assignment.state)) {
5323
- continue;
5324
- }
5325
- if (isPrimary) {
5326
- return { outcome: "authorized" };
5327
- }
5328
- if (!assignmentAllowsReplicaBootstrap(assignment, sourceId)) {
5329
- continue;
5330
- }
5331
- return { outcome: "authorized" };
5332
- }
5333
- if (sawAssignment) {
5334
- return {
5335
- outcome: "denied",
5336
- code: ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
5337
- reason: `node '${sourceId}' is listed for index '${indexName}' but not in a bootstrappable state`
5338
- };
5339
- }
5340
- return {
5341
- outcome: "denied",
5342
- code: ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
5343
- reason: `node '${sourceId}' is not an assigned replica for index '${indexName}'`
5344
- };
5345
- }
5346
- function assignmentAllowsReplicaBootstrap(assignment, sourceId) {
5347
- if (assignment.state === "INITIALISING" || assignment.state === "MIGRATING") {
5348
- return true;
5349
- }
5350
- return assignment.inSyncSet.includes(sourceId);
5351
- }
5352
- var SNAPSHOT_SYNC_ERROR_TYPE = `${ReplicationMessageTypes.SNAPSHOT_SYNC_REQUEST}.error`;
5353
- var CHUNK_YIELD_INTERVAL_MS = 8;
5354
- function createSingleResponseSink(respond) {
5355
- const sink = ((response) => {
5356
- if (sink.closed) {
5357
- return;
5358
- }
5359
- respond(response);
5360
- });
5361
- sink.closed = false;
5362
- return sink;
5363
- }
5364
- async function streamSnapshotToReplica(sink, nodeId, requestId, indexName, build, metadata, options = {}) {
5365
- const snapshotBytes = build.bytes;
5366
- const totalBytes = snapshotBytes.byteLength;
5367
- const checksum = build.checksum;
5368
- const header = {
5369
- lastSeqNo: metadata.lastSeqNo,
5370
- primaryTerm: metadata.primaryTerm,
5371
- partitionId: metadata.partitionId,
5372
- indexName,
5373
- checksum
5374
- };
5375
- const startPayload = { header, totalBytes };
5376
- const startBytes = encode(startPayload);
5377
- assertMessageWithinLimit(startBytes, "SNAPSHOT_START");
5378
- respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_START, nodeId, requestId, startBytes);
5379
- await yieldToEventLoop();
5380
- let offset = 0;
5381
- let lastYieldAt = now();
5382
- while (offset < totalBytes) {
5383
- const end = Math.min(offset + SNAPSHOT_CHUNK_SIZE, totalBytes);
5384
- const chunk = snapshotBytes.subarray(offset, end);
5385
- const chunkPayload = {
5386
- partitionId: metadata.partitionId,
5387
- indexName,
5388
- offset,
5389
- data: chunk
5390
- };
5391
- const chunkBytes = encode(chunkPayload);
5392
- assertMessageWithinLimit(chunkBytes, "SNAPSHOT_CHUNK");
5393
- respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_CHUNK, nodeId, requestId, chunkBytes);
5394
- offset = end;
5395
- if (offset < totalBytes && now() - lastYieldAt >= CHUNK_YIELD_INTERVAL_MS) {
5396
- await yieldToEventLoop();
5397
- lastYieldAt = now();
5118
+ if (!BOOTSTRAPPABLE_STATES.has(assignment.state)) {
5119
+ continue;
5398
5120
  }
5121
+ if (isPrimary) {
5122
+ return { outcome: "authorized" };
5123
+ }
5124
+ if (!assignmentAllowsReplicaBootstrap(assignment, sourceId)) {
5125
+ continue;
5126
+ }
5127
+ return { outcome: "authorized" };
5399
5128
  }
5400
- await yieldToEventLoop();
5401
- const endPayload = {
5402
- partitionId: metadata.partitionId,
5403
- indexName,
5404
- totalBytes,
5405
- checksum
5406
- };
5407
- const endBytes = encode(endPayload);
5408
- assertMessageWithinLimit(endBytes, "SNAPSHOT_END");
5409
- respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_END, nodeId, requestId, endBytes);
5410
- if (options.closeOnEnd !== false) {
5411
- sink.closed = true;
5412
- }
5413
- }
5414
- function now() {
5415
- if (typeof performance !== "undefined" && typeof performance.now === "function") {
5416
- return performance.now();
5129
+ if (sawAssignment) {
5130
+ return {
5131
+ outcome: "denied",
5132
+ code: ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
5133
+ reason: `node '${sourceId}' is listed for index '${indexName}' but not in a bootstrappable state`
5134
+ };
5417
5135
  }
5418
- return Date.now();
5419
- }
5420
- function yieldToEventLoop() {
5421
- return new Promise((resolve) => {
5422
- if (typeof setImmediate === "function") {
5423
- setImmediate(resolve);
5424
- return;
5425
- }
5426
- setTimeout(resolve, 0);
5427
- });
5428
- }
5429
- function respondMessage(sink, type, sourceId, requestId, payload) {
5430
- sink({ type, sourceId, requestId, payload });
5136
+ return {
5137
+ outcome: "denied",
5138
+ code: ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
5139
+ reason: `node '${sourceId}' is not an assigned replica for index '${indexName}'`
5140
+ };
5431
5141
  }
5432
- function assertMessageWithinLimit(bytes, label) {
5433
- if (bytes.byteLength > MAX_MESSAGE_SIZE_BYTES) {
5434
- throw new NarsilError(
5435
- ErrorCodes.CONFIG_INVALID,
5436
- `${label} payload (${bytes.byteLength} bytes) exceeds transport message limit (${MAX_MESSAGE_SIZE_BYTES})`
5437
- );
5142
+ function assignmentAllowsReplicaBootstrap(assignment, sourceId) {
5143
+ if (assignment.state === "INITIALISING" || assignment.state === "MIGRATING") {
5144
+ return true;
5438
5145
  }
5439
- }
5440
- function respondError(sink, sourceId, requestId, code, message) {
5441
- const response = {
5442
- type: SNAPSHOT_SYNC_ERROR_TYPE,
5443
- sourceId,
5444
- requestId,
5445
- payload: encode({ error: true, code, message })
5446
- };
5447
- sink(response);
5448
- sink.closed = true;
5146
+ return assignment.inSyncSet.includes(sourceId);
5449
5147
  }
5450
5148
 
5451
5149
  // src/distribution/cluster-node/snapshot-cache.ts
@@ -5576,37 +5274,106 @@ async function acquireSnapshotBuild(state, indexName, builder) {
5576
5274
  state.cache.set(indexName, entry);
5577
5275
  return entry.promise;
5578
5276
  }
5579
-
5580
- // src/distribution/cluster-node/snapshot-sync-handler.ts
5581
- var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
5582
- var MAX_SOURCE_ID_LENGTH = 256;
5583
- function containsControlCharacter(value) {
5584
- for (let i = 0; i < value.length; i++) {
5585
- const code = value.charCodeAt(i);
5586
- if (code < 32) {
5587
- return true;
5277
+ var SNAPSHOT_SYNC_ERROR_TYPE = `${ReplicationMessageTypes.SNAPSHOT_SYNC_REQUEST}.error`;
5278
+ var CHUNK_YIELD_INTERVAL_MS = 8;
5279
+ function createSingleResponseSink(respond) {
5280
+ const sink = ((response) => {
5281
+ if (sink.closed) {
5282
+ return;
5588
5283
  }
5589
- if (code >= 127 && code <= 159) {
5590
- return true;
5284
+ respond(response);
5285
+ });
5286
+ sink.closed = false;
5287
+ return sink;
5288
+ }
5289
+ async function streamSnapshotToReplica(sink, nodeId, requestId, indexName, build, metadata, options = {}) {
5290
+ const snapshotBytes = build.bytes;
5291
+ const totalBytes = snapshotBytes.byteLength;
5292
+ const checksum = build.checksum;
5293
+ const header = {
5294
+ lastSeqNo: metadata.lastSeqNo,
5295
+ primaryTerm: metadata.primaryTerm,
5296
+ partitionId: metadata.partitionId,
5297
+ indexName,
5298
+ checksum
5299
+ };
5300
+ const startPayload = { header, totalBytes };
5301
+ const startBytes = encode(startPayload);
5302
+ assertMessageWithinLimit(startBytes, "SNAPSHOT_START");
5303
+ respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_START, nodeId, requestId, startBytes);
5304
+ await yieldToEventLoop();
5305
+ let offset = 0;
5306
+ let lastYieldAt = now();
5307
+ while (offset < totalBytes) {
5308
+ const end = Math.min(offset + SNAPSHOT_CHUNK_SIZE, totalBytes);
5309
+ const chunk = snapshotBytes.subarray(offset, end);
5310
+ const chunkPayload = {
5311
+ partitionId: metadata.partitionId,
5312
+ indexName,
5313
+ offset,
5314
+ data: chunk
5315
+ };
5316
+ const chunkBytes = encode(chunkPayload);
5317
+ assertMessageWithinLimit(chunkBytes, "SNAPSHOT_CHUNK");
5318
+ respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_CHUNK, nodeId, requestId, chunkBytes);
5319
+ offset = end;
5320
+ if (offset < totalBytes && now() - lastYieldAt >= CHUNK_YIELD_INTERVAL_MS) {
5321
+ await yieldToEventLoop();
5322
+ lastYieldAt = now();
5591
5323
  }
5592
- if (code === 8232 || code === 8233 || code === 65279) {
5593
- return true;
5324
+ }
5325
+ await yieldToEventLoop();
5326
+ const endPayload = {
5327
+ partitionId: metadata.partitionId,
5328
+ indexName,
5329
+ totalBytes,
5330
+ checksum
5331
+ };
5332
+ const endBytes = encode(endPayload);
5333
+ assertMessageWithinLimit(endBytes, "SNAPSHOT_END");
5334
+ respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_END, nodeId, requestId, endBytes);
5335
+ if (options.closeOnEnd !== false) {
5336
+ sink.closed = true;
5337
+ }
5338
+ }
5339
+ function now() {
5340
+ if (typeof performance !== "undefined" && typeof performance.now === "function") {
5341
+ return performance.now();
5342
+ }
5343
+ return Date.now();
5344
+ }
5345
+ function yieldToEventLoop() {
5346
+ return new Promise((resolve) => {
5347
+ if (typeof setImmediate === "function") {
5348
+ setImmediate(resolve);
5349
+ return;
5594
5350
  }
5351
+ setTimeout(resolve, 0);
5352
+ });
5353
+ }
5354
+ function respondMessage(sink, type, sourceId, requestId, payload) {
5355
+ sink({ type, sourceId, requestId, payload });
5356
+ }
5357
+ function assertMessageWithinLimit(bytes, label) {
5358
+ if (bytes.byteLength > MAX_MESSAGE_SIZE_BYTES) {
5359
+ throw new NarsilError(
5360
+ ErrorCodes.CONFIG_INVALID,
5361
+ `${label} payload (${bytes.byteLength} bytes) exceeds transport message limit (${MAX_MESSAGE_SIZE_BYTES})`
5362
+ );
5595
5363
  }
5596
- return false;
5597
5364
  }
5598
- var MAX_SNAPSHOT_SYNC_REQUEST_BYTES = 4096;
5599
- var REQUEST_DECODE_OPTIONS = {
5600
- maxMapLength: 16,
5601
- maxArrayLength: 16,
5602
- maxStrLength: MAX_INDEX_NAME_LENGTH,
5603
- maxBinLength: 0,
5604
- maxExtLength: 0,
5605
- keyDecoder: null
5606
- };
5607
- function createSnapshotSyncHandlerState(maxConcurrent = DEFAULT_MAX_CONCURRENT_SNAPSHOTS, maxPerSource = DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, maxStreamsPerIndex = DEFAULT_MAX_STREAMS_PER_INDEX) {
5608
- return createSnapshotCacheState(maxConcurrent, maxPerSource, maxStreamsPerIndex);
5365
+ function respondError(sink, sourceId, requestId, code, message) {
5366
+ const response = {
5367
+ type: SNAPSHOT_SYNC_ERROR_TYPE,
5368
+ sourceId,
5369
+ requestId,
5370
+ payload: encode({ error: true, code, message })
5371
+ };
5372
+ sink(response);
5373
+ sink.closed = true;
5609
5374
  }
5375
+
5376
+ // src/distribution/cluster-node/snapshot-sync-handler/metadata.ts
5610
5377
  async function defaultSnapshotHeaderMetadataProvider(coordinator, indexName) {
5611
5378
  try {
5612
5379
  const allocation = await coordinator.getAllocation(indexName);
@@ -5628,81 +5395,24 @@ async function defaultSnapshotHeaderMetadataProvider(coordinator, indexName) {
5628
5395
  primaryTerm: 0,
5629
5396
  lastSeqNo: SNAPSHOT_HEADER_SENTINEL_SEQNO
5630
5397
  };
5631
- }
5632
- async function handleSnapshotSyncRequest(message, respond, deps) {
5633
- const sink = createSingleResponseSink(respond);
5634
- try {
5635
- if (message.payload.byteLength > MAX_SNAPSHOT_SYNC_REQUEST_BYTES) {
5636
- respondError(
5637
- sink,
5638
- deps.nodeId,
5639
- message.requestId,
5640
- ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
5641
- `SNAPSHOT_SYNC_REQUEST payload (${message.payload.byteLength} bytes) exceeds the ${MAX_SNAPSHOT_SYNC_REQUEST_BYTES} byte limit`
5642
- );
5643
- return;
5644
- }
5645
- await runSnapshotSyncRequest(message, sink, deps);
5646
- } catch (err) {
5647
- const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SNAPSHOT_FAILED;
5648
- const errMessage = err instanceof Error ? err.message : String(err);
5649
- respondError(sink, deps.nodeId, message.requestId, code, errMessage);
5650
- }
5651
- }
5652
- async function runSnapshotSyncRequest(message, sink, deps) {
5653
- const request = decodeRequest(message, sink, deps);
5654
- if (request === null) {
5655
- return;
5656
- }
5657
- await streamValidatedSnapshotRequest(message, sink, request, deps);
5658
- }
5659
- async function streamValidatedSnapshotRequest(message, sink, request, deps, options = {}) {
5660
- const sourceIdError = validateSourceId(message.sourceId);
5661
- if (sourceIdError !== null) {
5662
- respondError(sink, deps.nodeId, message.requestId, ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID, sourceIdError);
5663
- return;
5664
- }
5665
- const authResult = await authorizeSnapshotRequest(deps.coordinator, request.indexName, message.sourceId);
5666
- if (authResult.outcome === "denied") {
5667
- respondError(sink, deps.nodeId, message.requestId, authResult.code, authResult.reason);
5668
- return;
5669
- }
5670
- const existingIndex = deps.engine.listIndexes().find((idx) => idx.name === request.indexName);
5671
- if (existingIndex === void 0) {
5672
- respondError(
5673
- sink,
5674
- deps.nodeId,
5675
- message.requestId,
5676
- ErrorCodes.SNAPSHOT_SYNC_INDEX_NOT_FOUND,
5677
- `Index '${request.indexName}' is not hosted on this node`
5678
- );
5679
- return;
5680
- }
5681
- await acquireAndStream(message, sink, request, deps, options);
5682
- }
5683
- function decodeRequest(message, sink, deps) {
5684
- try {
5685
- const decoded = decode(message.payload, REQUEST_DECODE_OPTIONS);
5686
- return validateSnapshotSyncRequestPayload(decoded);
5687
- } catch (err) {
5688
- const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID;
5689
- const errMessage = err instanceof Error ? err.message : String(err);
5690
- respondError(sink, deps.nodeId, message.requestId, code, errMessage);
5691
- return null;
5692
- }
5693
- }
5694
- function validateSourceId(sourceId) {
5695
- if (typeof sourceId !== "string" || sourceId.length === 0) {
5696
- return "request sourceId is missing";
5697
- }
5698
- if (sourceId.length > MAX_SOURCE_ID_LENGTH) {
5699
- return `request sourceId exceeds ${MAX_SOURCE_ID_LENGTH} characters`;
5398
+ }
5399
+ async function resolveHeaderMetadata(deps, indexName, partitionId) {
5400
+ const provider = deps.resolveHeaderMetadata;
5401
+ if (provider === void 0) {
5402
+ return defaultSnapshotHeaderMetadataProvider(deps.coordinator, indexName);
5700
5403
  }
5701
- if (containsControlCharacter(sourceId)) {
5702
- return "request sourceId contains control characters";
5404
+ try {
5405
+ return await provider(indexName, partitionId);
5406
+ } catch (_) {
5407
+ return {
5408
+ partitionId: SNAPSHOT_HEADER_SENTINEL_PARTITION_ID,
5409
+ primaryTerm: 0,
5410
+ lastSeqNo: SNAPSHOT_HEADER_SENTINEL_SEQNO
5411
+ };
5703
5412
  }
5704
- return null;
5705
5413
  }
5414
+
5415
+ // src/distribution/cluster-node/snapshot-sync-handler/stream.ts
5706
5416
  async function acquireAndStream(message, sink, request, deps, options) {
5707
5417
  const { indexName } = request;
5708
5418
  let sourceHandle;
@@ -5783,241 +5493,411 @@ function buildSnapshotForRequest(deps, indexName, options) {
5783
5493
  }
5784
5494
  return buildSnapshot(deps, indexName);
5785
5495
  }
5786
- async function resolveHeaderMetadata(deps, indexName, partitionId) {
5787
- const provider = deps.resolveHeaderMetadata;
5788
- if (provider === void 0) {
5789
- return defaultSnapshotHeaderMetadataProvider(deps.coordinator, indexName);
5496
+ var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
5497
+ var MAX_SOURCE_ID_LENGTH = 256;
5498
+ function containsControlCharacter(value) {
5499
+ for (let i = 0; i < value.length; i++) {
5500
+ const code = value.charCodeAt(i);
5501
+ if (code < 32) {
5502
+ return true;
5503
+ }
5504
+ if (code >= 127 && code <= 159) {
5505
+ return true;
5506
+ }
5507
+ if (code === 8232 || code === 8233 || code === 65279) {
5508
+ return true;
5509
+ }
5790
5510
  }
5511
+ return false;
5512
+ }
5513
+ var MAX_SNAPSHOT_SYNC_REQUEST_BYTES = 4096;
5514
+ var REQUEST_DECODE_OPTIONS = {
5515
+ maxMapLength: 16,
5516
+ maxArrayLength: 16,
5517
+ maxStrLength: MAX_INDEX_NAME_LENGTH,
5518
+ maxBinLength: 0,
5519
+ maxExtLength: 0,
5520
+ keyDecoder: null
5521
+ };
5522
+ function decodeRequest(message, sink, deps) {
5791
5523
  try {
5792
- return await provider(indexName, partitionId);
5793
- } catch (_) {
5794
- return {
5795
- partitionId: SNAPSHOT_HEADER_SENTINEL_PARTITION_ID,
5796
- primaryTerm: 0,
5797
- lastSeqNo: SNAPSHOT_HEADER_SENTINEL_SEQNO
5798
- };
5524
+ const decoded = decode(message.payload, REQUEST_DECODE_OPTIONS);
5525
+ return validateSnapshotSyncRequestPayload(decoded);
5526
+ } catch (err) {
5527
+ const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID;
5528
+ const errMessage = err instanceof Error ? err.message : String(err);
5529
+ respondError(sink, deps.nodeId, message.requestId, code, errMessage);
5530
+ return null;
5531
+ }
5532
+ }
5533
+ function validateSourceId(sourceId) {
5534
+ if (typeof sourceId !== "string" || sourceId.length === 0) {
5535
+ return "request sourceId is missing";
5536
+ }
5537
+ if (sourceId.length > MAX_SOURCE_ID_LENGTH) {
5538
+ return `request sourceId exceeds ${MAX_SOURCE_ID_LENGTH} characters`;
5539
+ }
5540
+ if (containsControlCharacter(sourceId)) {
5541
+ return "request sourceId contains control characters";
5542
+ }
5543
+ return null;
5544
+ }
5545
+ function validateSnapshotSyncRequestPayload(decoded) {
5546
+ if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
5547
+ throw new NarsilError(
5548
+ ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
5549
+ "Invalid SnapshotSyncRequestPayload: expected an object"
5550
+ );
5551
+ }
5552
+ const record = decoded;
5553
+ const indexName = record.indexName;
5554
+ if (typeof indexName !== "string" || indexName.length === 0) {
5555
+ throw new NarsilError(
5556
+ ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
5557
+ 'Invalid SnapshotSyncRequestPayload: "indexName" must be a non-empty string'
5558
+ );
5559
+ }
5560
+ if (indexName.length > MAX_INDEX_NAME_LENGTH) {
5561
+ throw new NarsilError(
5562
+ ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
5563
+ `Invalid SnapshotSyncRequestPayload: "indexName" must be at most ${MAX_INDEX_NAME_LENGTH} characters`,
5564
+ { length: indexName.length, limit: MAX_INDEX_NAME_LENGTH }
5565
+ );
5566
+ }
5567
+ if (!INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
5568
+ throw new NarsilError(
5569
+ ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
5570
+ 'Invalid SnapshotSyncRequestPayload: "indexName" contains invalid characters'
5571
+ );
5572
+ }
5573
+ const partitionId = record.partitionId;
5574
+ if (partitionId !== void 0 && partitionId !== null) {
5575
+ if (typeof partitionId !== "number" || !Number.isInteger(partitionId) || partitionId < 0) {
5576
+ throw new NarsilError(
5577
+ ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
5578
+ 'Invalid SnapshotSyncRequestPayload: "partitionId" must be a non-negative integer when provided'
5579
+ );
5580
+ }
5581
+ return { indexName, partitionId };
5582
+ }
5583
+ return { indexName, partitionId: null };
5584
+ }
5585
+
5586
+ // src/distribution/cluster-node/snapshot-sync-handler/index.ts
5587
+ function createSnapshotSyncHandlerState(maxConcurrent = DEFAULT_MAX_CONCURRENT_SNAPSHOTS, maxPerSource = DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, maxStreamsPerIndex = DEFAULT_MAX_STREAMS_PER_INDEX) {
5588
+ return createSnapshotCacheState(maxConcurrent, maxPerSource, maxStreamsPerIndex);
5589
+ }
5590
+ async function handleSnapshotSyncRequest(message, respond, deps) {
5591
+ const sink = createSingleResponseSink(respond);
5592
+ try {
5593
+ if (message.payload.byteLength > MAX_SNAPSHOT_SYNC_REQUEST_BYTES) {
5594
+ respondError(
5595
+ sink,
5596
+ deps.nodeId,
5597
+ message.requestId,
5598
+ ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
5599
+ `SNAPSHOT_SYNC_REQUEST payload (${message.payload.byteLength} bytes) exceeds the ${MAX_SNAPSHOT_SYNC_REQUEST_BYTES} byte limit`
5600
+ );
5601
+ return;
5602
+ }
5603
+ await runSnapshotSyncRequest(message, sink, deps);
5604
+ } catch (err) {
5605
+ const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SNAPSHOT_FAILED;
5606
+ const errMessage = err instanceof Error ? err.message : String(err);
5607
+ respondError(sink, deps.nodeId, message.requestId, code, errMessage);
5608
+ }
5609
+ }
5610
+ async function runSnapshotSyncRequest(message, sink, deps) {
5611
+ const request = decodeRequest(message, sink, deps);
5612
+ if (request === null) {
5613
+ return;
5614
+ }
5615
+ await streamValidatedSnapshotRequest(message, sink, request, deps);
5616
+ }
5617
+ async function streamValidatedSnapshotRequest(message, sink, request, deps, options = {}) {
5618
+ const sourceIdError = validateSourceId(message.sourceId);
5619
+ if (sourceIdError !== null) {
5620
+ respondError(sink, deps.nodeId, message.requestId, ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID, sourceIdError);
5621
+ return;
5622
+ }
5623
+ const authResult = await authorizeSnapshotRequest(deps.coordinator, request.indexName, message.sourceId);
5624
+ if (authResult.outcome === "denied") {
5625
+ respondError(sink, deps.nodeId, message.requestId, authResult.code, authResult.reason);
5626
+ return;
5627
+ }
5628
+ const existingIndex = deps.engine.listIndexes().find((idx) => idx.name === request.indexName);
5629
+ if (existingIndex === void 0) {
5630
+ respondError(
5631
+ sink,
5632
+ deps.nodeId,
5633
+ message.requestId,
5634
+ ErrorCodes.SNAPSHOT_SYNC_INDEX_NOT_FOUND,
5635
+ `Index '${request.indexName}' is not hosted on this node`
5636
+ );
5637
+ return;
5638
+ }
5639
+ await acquireAndStream(message, sink, request, deps, options);
5640
+ }
5641
+ async function handleSearch(message, respond, deps) {
5642
+ const decoded = decode(message.payload);
5643
+ const payload = validateSearchPayload(decoded);
5644
+ const queryParams = {
5645
+ term: payload.params.term ?? void 0,
5646
+ fields: payload.params.fields ?? void 0,
5647
+ filters: payload.params.filters ?? void 0,
5648
+ boost: payload.params.boost ?? void 0,
5649
+ scoring: payload.params.scoring,
5650
+ tolerance: payload.params.tolerance ?? void 0,
5651
+ minScore: payload.params.threshold ?? void 0,
5652
+ limit: payload.params.limit,
5653
+ offset: payload.params.offset,
5654
+ searchAfter: payload.params.searchAfter ?? void 0,
5655
+ sort: convertWireSortToLocal(payload.params.sort),
5656
+ group: payload.params.group !== null ? { fields: [payload.params.group.field], maxPerGroup: payload.params.group.maxPerGroup } : void 0,
5657
+ facets: convertWireFacetsToLocal(payload.params.facets, payload.facetShardSize ?? payload.params.facetSize),
5658
+ vector: payload.params.vector !== null ? {
5659
+ field: payload.params.vector.field,
5660
+ value: payload.params.vector.value ?? void 0,
5661
+ text: payload.params.vector.text ?? void 0,
5662
+ similarity: payload.params.vector.similarity ?? void 0
5663
+ } : void 0,
5664
+ hybrid: payload.params.hybrid !== null ? { strategy: payload.params.hybrid.strategy, k: payload.params.hybrid.k, alpha: payload.params.hybrid.alpha } : void 0
5665
+ };
5666
+ const queryResult = await deps.engine.query(payload.indexName, queryParams);
5667
+ const scored = queryResult.hits.map((hit) => ({
5668
+ docId: hit.id,
5669
+ score: hit.score,
5670
+ sortValues: null
5671
+ }));
5672
+ const results = payload.partitionIds.map((partitionId) => ({
5673
+ partitionId,
5674
+ scored: partitionId === payload.partitionIds[0] ? scored : [],
5675
+ totalHits: partitionId === payload.partitionIds[0] ? queryResult.count : 0
5676
+ }));
5677
+ const resultPayload = { results, facets: convertLocalFacetsToWire(queryResult.facets) };
5678
+ respond({
5679
+ type: QueryMessageTypes.SEARCH_RESULT,
5680
+ sourceId: deps.nodeId,
5681
+ requestId: message.requestId,
5682
+ payload: encode(resultPayload)
5683
+ });
5684
+ }
5685
+ async function handleFetch(message, respond, deps) {
5686
+ const decoded = decode(message.payload);
5687
+ const payload = validateFetchPayload(decoded);
5688
+ const documents = [];
5689
+ for (const docRef of payload.documentIds) {
5690
+ const doc = await deps.engine.get(payload.indexName, docRef.docId);
5691
+ if (doc !== void 0) {
5692
+ documents.push({
5693
+ docId: docRef.docId,
5694
+ document: doc,
5695
+ highlights: null
5696
+ });
5697
+ }
5698
+ }
5699
+ const resultPayload = { documents };
5700
+ respond({
5701
+ type: QueryMessageTypes.FETCH_RESULT,
5702
+ sourceId: deps.nodeId,
5703
+ requestId: message.requestId,
5704
+ payload: encode(resultPayload)
5705
+ });
5706
+ }
5707
+ async function handleStats(message, respond, deps) {
5708
+ const decoded = decode(message.payload);
5709
+ const payload = validateStatsPayload(decoded);
5710
+ const stats = deps.engine.getStats(payload.indexName);
5711
+ const resultPayload = {
5712
+ totalDocuments: stats.documentCount,
5713
+ docFrequencies: {},
5714
+ totalFieldLengths: {}
5715
+ };
5716
+ respond({
5717
+ type: QueryMessageTypes.STATS_RESULT,
5718
+ sourceId: deps.nodeId,
5719
+ requestId: message.requestId,
5720
+ payload: encode(resultPayload)
5721
+ });
5722
+ }
5723
+ function convertWireSortToLocal(wireSort) {
5724
+ if (wireSort === null || wireSort.length === 0) {
5725
+ return void 0;
5726
+ }
5727
+ const result = {};
5728
+ for (const entry of wireSort) {
5729
+ result[entry.field] = entry.direction;
5799
5730
  }
5731
+ return result;
5800
5732
  }
5801
- function validateSnapshotSyncRequestPayload(decoded) {
5802
- if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
5803
- throw new NarsilError(
5804
- ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
5805
- "Invalid SnapshotSyncRequestPayload: expected an object"
5806
- );
5733
+ function convertWireFacetsToLocal(facets, limit) {
5734
+ if (facets === null || facets.length === 0) {
5735
+ return void 0;
5807
5736
  }
5808
- const record = decoded;
5809
- const indexName = record.indexName;
5810
- if (typeof indexName !== "string" || indexName.length === 0) {
5811
- throw new NarsilError(
5812
- ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
5813
- 'Invalid SnapshotSyncRequestPayload: "indexName" must be a non-empty string'
5814
- );
5737
+ const result = {};
5738
+ for (const field of facets) {
5739
+ result[field] = limit !== null ? { limit } : {};
5815
5740
  }
5816
- if (indexName.length > MAX_INDEX_NAME_LENGTH) {
5817
- throw new NarsilError(
5818
- ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
5819
- `Invalid SnapshotSyncRequestPayload: "indexName" must be at most ${MAX_INDEX_NAME_LENGTH} characters`,
5820
- { length: indexName.length, limit: MAX_INDEX_NAME_LENGTH }
5821
- );
5741
+ return result;
5742
+ }
5743
+ function convertLocalFacetsToWire(facets) {
5744
+ if (facets === void 0) {
5745
+ return null;
5822
5746
  }
5823
- if (!INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
5824
- throw new NarsilError(
5825
- ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
5826
- 'Invalid SnapshotSyncRequestPayload: "indexName" contains invalid characters'
5827
- );
5747
+ const result = {};
5748
+ for (const [field, facet] of Object.entries(facets)) {
5749
+ result[field] = Object.entries(facet.values).map(([value, count]) => ({ value, count }));
5828
5750
  }
5829
- const partitionId = record.partitionId;
5830
- if (partitionId !== void 0 && partitionId !== null) {
5831
- if (typeof partitionId !== "number" || !Number.isInteger(partitionId) || partitionId < 0) {
5832
- throw new NarsilError(
5833
- ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
5834
- 'Invalid SnapshotSyncRequestPayload: "partitionId" must be a non-negative integer when provided'
5835
- );
5751
+ return result;
5752
+ }
5753
+ function decideSyncTier(log, replicaLastSeqNo) {
5754
+ const oldest = log.oldestSeqNo;
5755
+ if (oldest === void 0) {
5756
+ if (log.committedSeqNo === 0) {
5757
+ return replicaLastSeqNo === 0 ? "incremental" : "snapshot";
5836
5758
  }
5837
- return { indexName, partitionId };
5759
+ return replicaLastSeqNo >= log.committedSeqNo ? "incremental" : "snapshot";
5838
5760
  }
5839
- return { indexName, partitionId: null };
5761
+ if (oldest <= replicaLastSeqNo + 1) {
5762
+ return "incremental";
5763
+ }
5764
+ return "snapshot";
5840
5765
  }
5841
- async function replicateToReplicas(entry, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets2) {
5842
- const uniqueReplicas = [...new Set(inSyncReplicas)];
5843
- if (uniqueReplicas.length === 0) {
5844
- return { acknowledged: [], failed: [] };
5766
+ function validateSyncRequest(decoded) {
5767
+ if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
5768
+ throw new Error("Invalid SyncRequestPayload: expected an object");
5845
5769
  }
5846
- const message = createEntryMessage(entry, sourceNodeId);
5847
- const sendResults = await Promise.allSettled(
5848
- uniqueReplicas.map((replicaNodeId) => sendToReplica(transport, replicaNodeId, message, resolveNodeTargets2))
5849
- );
5850
- const acknowledged = [];
5851
- const failed = [];
5852
- for (let i = 0; i < sendResults.length; i++) {
5853
- const result = sendResults[i];
5854
- const replicaNodeId = uniqueReplicas[i];
5855
- if (result.status === "rejected") {
5856
- if (result.reason instanceof TransportError) {
5857
- failed.push(replicaNodeId);
5858
- continue;
5859
- }
5860
- throw result.reason;
5861
- }
5862
- const response = result.value;
5863
- if (isMatchingAck(response, entry)) {
5864
- acknowledged.push(replicaNodeId);
5865
- } else {
5866
- failed.push(replicaNodeId);
5867
- }
5770
+ const record = decoded;
5771
+ if (typeof record.indexName !== "string") {
5772
+ throw new Error('Invalid SyncRequestPayload: "indexName" must be a string');
5868
5773
  }
5869
- return { acknowledged, failed };
5870
- }
5871
- async function sendToReplica(transport, replicaNodeId, message, resolveNodeTargets2) {
5872
- const targets = resolveNodeTargets2 === void 0 ? [replicaNodeId] : await resolveNodeTargets2(replicaNodeId);
5873
- const resolvedTargets = targets.length > 0 ? targets : [replicaNodeId];
5874
- let lastError;
5875
- for (const target of resolvedTargets) {
5876
- try {
5877
- return await transport.send(target, message);
5878
- } catch (error) {
5879
- lastError = error;
5880
- }
5774
+ if (typeof record.partitionId !== "number" || !Number.isInteger(record.partitionId) || record.partitionId < 0) {
5775
+ throw new Error('Invalid SyncRequestPayload: "partitionId" must be a non-negative integer');
5881
5776
  }
5882
- throw lastError instanceof Error ? lastError : new Error(String(lastError));
5883
- }
5884
- function isMatchingAck(response, entry) {
5885
- if (response.type !== ReplicationMessageTypes.ACK) {
5886
- return false;
5777
+ if (typeof record.lastSeqNo !== "number" || !Number.isInteger(record.lastSeqNo) || record.lastSeqNo < 0) {
5778
+ throw new Error('Invalid SyncRequestPayload: "lastSeqNo" must be a non-negative integer');
5887
5779
  }
5888
- try {
5889
- const payload = validateAckPayload(decode(response.payload));
5890
- return payload.seqNo === entry.seqNo && payload.partitionId === entry.partitionId && payload.indexName === entry.indexName;
5891
- } catch {
5892
- return false;
5780
+ if (typeof record.lastPrimaryTerm !== "number" || !Number.isInteger(record.lastPrimaryTerm) || record.lastPrimaryTerm < 0) {
5781
+ throw new Error('Invalid SyncRequestPayload: "lastPrimaryTerm" must be a non-negative integer');
5893
5782
  }
5783
+ return record;
5894
5784
  }
5895
5785
 
5896
- // src/distribution/cluster-node/types.ts
5897
- var DEFAULT_PARTITION_COUNT = 5;
5898
- var DEFAULT_REPLICATION_FACTOR = 1;
5899
- var DEFAULT_CAPACITY = {
5900
- memoryBytes: 8e9,
5901
- cpuCores: 4,
5902
- diskBytes: null
5903
- };
5904
-
5905
- // src/distribution/cluster-node/write-routing.ts
5906
- var MAX_REPLICATION_FACTOR = 255;
5907
- function validateCreateIndexOptions(partitionCount, replicationFactor) {
5908
- if (!Number.isInteger(partitionCount) || partitionCount < 1 || partitionCount > MAX_PARTITION_COUNT) {
5786
+ // src/distribution/cluster-node/message-handler/sync.ts
5787
+ async function handleSyncRequestMessage(message, respond, deps) {
5788
+ const request = validateSyncRequest(decode(message.payload));
5789
+ const authResult = await authorizeSnapshotRequest(deps.coordinator, request.indexName, message.sourceId);
5790
+ if (authResult.outcome === "denied") {
5791
+ throw new NarsilError(authResult.code, authResult.reason, {
5792
+ indexName: request.indexName,
5793
+ partitionId: request.partitionId,
5794
+ sourceNodeId: message.sourceId
5795
+ });
5796
+ }
5797
+ const table = await deps.coordinator.getAllocation(request.indexName);
5798
+ const assignment = table?.assignments.get(request.partitionId);
5799
+ if (assignment === void 0) {
5909
5800
  throw new NarsilError(
5910
- ErrorCodes.CONFIG_INVALID,
5911
- `partitionCount must be an integer between 1 and ${MAX_PARTITION_COUNT}, received ${partitionCount}`,
5912
- { partitionCount }
5801
+ ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
5802
+ `No assignment exists for sync request partition ${request.partitionId} of index '${request.indexName}'`,
5803
+ { indexName: request.indexName, partitionId: request.partitionId }
5913
5804
  );
5914
5805
  }
5915
- if (!Number.isInteger(replicationFactor) || replicationFactor < 0 || replicationFactor > MAX_REPLICATION_FACTOR) {
5806
+ if (assignment.primary !== deps.nodeId) {
5916
5807
  throw new NarsilError(
5917
- ErrorCodes.CONFIG_INVALID,
5918
- `replicationFactor must be an integer between 0 and ${MAX_REPLICATION_FACTOR}, received ${replicationFactor}`,
5919
- { replicationFactor }
5808
+ ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
5809
+ `Node '${deps.nodeId}' is not primary for sync request partition ${request.partitionId} of index '${request.indexName}'`,
5810
+ {
5811
+ indexName: request.indexName,
5812
+ partitionId: request.partitionId,
5813
+ localNodeId: deps.nodeId,
5814
+ primaryNodeId: assignment.primary
5815
+ }
5920
5816
  );
5921
5817
  }
5922
- }
5923
- async function routeCreateIndex(name, config, options, coordinator, engine) {
5924
- validateIndexName(name);
5925
- const partitionCount = options?.partitionCount ?? DEFAULT_PARTITION_COUNT;
5926
- const replicationFactor = options?.replicationFactor ?? DEFAULT_REPLICATION_FACTOR;
5927
- validateCreateIndexOptions(partitionCount, replicationFactor);
5928
- const constraints = {
5929
- zoneAwareness: false,
5930
- zoneAttribute: "zone",
5931
- maxShardsPerNode: null
5932
- };
5933
- const metadata = {
5934
- indexName: name,
5935
- partitionCount,
5936
- replicationFactor,
5937
- constraints
5938
- };
5939
- const stored = await putIndexMetadata(coordinator, metadata);
5940
- if (!stored) {
5818
+ const sourceAssigned = assignment.primary === message.sourceId || assignment.replicas.includes(message.sourceId);
5819
+ if (!sourceAssigned) {
5941
5820
  throw new NarsilError(
5942
- ErrorCodes.INDEX_ALREADY_EXISTS,
5943
- `Index '${name}' already exists in the cluster or a partial creation is pending`,
5944
- { indexName: name }
5821
+ ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
5822
+ `Node '${message.sourceId}' is not assigned to partition ${request.partitionId} of index '${request.indexName}'`,
5823
+ { indexName: request.indexName, partitionId: request.partitionId, sourceNodeId: message.sourceId }
5945
5824
  );
5946
5825
  }
5947
- try {
5948
- await coordinator.putSchema(name, config.schema);
5949
- await engine.createIndex(name, config);
5950
- } catch (createErr) {
5951
- let cleanupFailed = false;
5952
- let cleanupError;
5953
- try {
5954
- const currentBytes = await coordinator.get(`_narsil/index/${name}/config`);
5955
- if (currentBytes !== null) {
5956
- await coordinator.compareAndSet(`_narsil/index/${name}/config`, currentBytes, new Uint8Array(0));
5826
+ if (request.lastPrimaryTerm > assignment.primaryTerm) {
5827
+ throw new NarsilError(
5828
+ ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
5829
+ `Replica term ${request.lastPrimaryTerm} is newer than primary term ${assignment.primaryTerm}`,
5830
+ {
5831
+ indexName: request.indexName,
5832
+ partitionId: request.partitionId,
5833
+ lastPrimaryTerm: request.lastPrimaryTerm,
5834
+ primaryTerm: assignment.primaryTerm
5835
+ }
5836
+ );
5837
+ }
5838
+ const log = deps.writeDeps.getReplicationLog(request.indexName, request.partitionId);
5839
+ const tier = decideSyncTier(log, request.lastSeqNo);
5840
+ if (tier === "incremental") {
5841
+ sendSyncEntriesResponse(message, respond, deps, log.getEntriesFrom(request.lastSeqNo + 1));
5842
+ return;
5843
+ }
5844
+ const snapshotSeqNo = log.committedSeqNo;
5845
+ const sink = createSingleResponseSink(respond);
5846
+ await streamValidatedSnapshotRequest(
5847
+ message,
5848
+ sink,
5849
+ { indexName: request.indexName, partitionId: request.partitionId },
5850
+ {
5851
+ nodeId: deps.nodeId,
5852
+ engine: deps.engine,
5853
+ coordinator: deps.coordinator,
5854
+ state: deps.snapshotSyncState,
5855
+ resolveHeaderMetadata: deps.resolveHeaderMetadata
5856
+ },
5857
+ {
5858
+ metadata: {
5859
+ partitionId: request.partitionId,
5860
+ lastSeqNo: snapshotSeqNo,
5861
+ primaryTerm: assignment.primaryTerm
5862
+ },
5863
+ closeOnEnd: false,
5864
+ disableBuildCache: true,
5865
+ buildSnapshot: async () => {
5866
+ const bytes = await deps.engine.serializeReplicationPartition(request.indexName, request.partitionId);
5867
+ return { bytes, checksum: crc32(bytes) };
5868
+ },
5869
+ afterSnapshot: (trailingSink) => {
5870
+ const entries = log.getEntriesFrom(snapshotSeqNo + 1);
5871
+ sendSyncEntriesResponse(message, trailingSink, deps, entries);
5957
5872
  }
5958
- } catch (cleanErr) {
5959
- cleanupFailed = true;
5960
- cleanupError = cleanErr;
5961
- }
5962
- if (cleanupFailed) {
5963
- throw new NarsilError(
5964
- ErrorCodes.CONFIG_INVALID,
5965
- `Index creation for '${name}' failed and metadata cleanup also failed. The index may be in a partial state and require manual intervention.`,
5966
- {
5967
- indexName: name,
5968
- createError: createErr instanceof Error ? createErr.message : String(createErr),
5969
- cleanupError: cleanupError instanceof Error ? cleanupError.message : String(cleanupError)
5970
- }
5971
- );
5972
5873
  }
5973
- throw createErr;
5974
- }
5975
- }
5976
- function resolvePartitionId(docId, partitionCount) {
5977
- return fnv1a(docId) % partitionCount;
5978
- }
5979
- async function forwardInsertToRemote(indexName, document, docId, primaryNodeId, deps) {
5980
- const payload = {
5981
- indexName,
5982
- documentId: docId,
5983
- operation: "insert",
5984
- document: encode(document),
5985
- updateFields: null
5986
- };
5987
- const message = createForwardMessage(payload, deps.nodeId);
5988
- const response = await sendToNode(primaryNodeId, message, deps);
5989
- const decoded = decode(response.payload);
5990
- if (typeof decoded.documentId !== "string") {
5991
- throw new NarsilError(
5992
- ErrorCodes.QUERY_ROUTING_FAILED,
5993
- `Remote primary returned invalid forward response for index '${indexName}'`,
5994
- { indexName, primaryNodeId }
5995
- );
5996
- }
5997
- return decoded.documentId;
5874
+ );
5998
5875
  }
5999
- async function forwardRemoveToRemote(indexName, docId, primaryNodeId, deps) {
5876
+ function sendSyncEntriesResponse(message, respond, deps, entries) {
6000
5877
  const payload = {
6001
- indexName,
6002
- documentId: docId,
6003
- operation: "remove",
6004
- document: null,
6005
- updateFields: null
5878
+ entries,
5879
+ isLast: true
6006
5880
  };
6007
- const message = createForwardMessage(payload, deps.nodeId);
6008
- await sendToNode(primaryNodeId, message, deps);
5881
+ respond({
5882
+ type: ReplicationMessageTypes.SYNC_ENTRIES,
5883
+ sourceId: deps.nodeId,
5884
+ requestId: message.requestId,
5885
+ payload: encode(payload)
5886
+ });
6009
5887
  }
6010
- async function sendToNode(nodeId, message, deps) {
6011
- const targets = await resolveNodeTargets(nodeId, deps);
6012
- let lastError;
6013
- for (const target of targets) {
6014
- try {
6015
- return await deps.transport.send(target, message);
6016
- } catch (error) {
6017
- lastError = error;
6018
- }
6019
- }
6020
- throw lastError instanceof Error ? lastError : new Error(String(lastError));
5888
+
5889
+ // src/distribution/cluster-node/types.ts
5890
+ var DEFAULT_PARTITION_COUNT = 5;
5891
+ var DEFAULT_REPLICATION_FACTOR = 1;
5892
+ var DEFAULT_CAPACITY = {
5893
+ memoryBytes: 8e9,
5894
+ cpuCores: 4,
5895
+ diskBytes: null
5896
+ };
5897
+
5898
+ // src/distribution/cluster-node/write-routing/assignment.ts
5899
+ function resolvePartitionId(docId, partitionCount) {
5900
+ return fnv1a(docId) % partitionCount;
6021
5901
  }
6022
5902
  async function resolveNodeTargets(nodeId, deps) {
6023
5903
  if (deps.resolveNodeTargets === void 0) {
@@ -6029,7 +5909,7 @@ async function resolveNodeTargets(nodeId, deps) {
6029
5909
  function requireAssignedPrimary(assignment, indexName, partitionId) {
6030
5910
  if (assignment === void 0 || assignment.primary === null) {
6031
5911
  throw new NarsilError(
6032
- ErrorCodes.QUERY_ROUTING_FAILED,
5912
+ ErrorCodes.PARTITION_UNASSIGNED,
6033
5913
  `No primary assigned for partition ${partitionId} of index '${indexName}'`,
6034
5914
  { indexName, partitionId }
6035
5915
  );
@@ -6053,13 +5933,27 @@ async function resolvePrimaryAssignment(indexName, docId, deps, requireLocalPrim
6053
5933
  const assignment = requireAssignedPrimary(table.assignments.get(partitionId), indexName, partitionId);
6054
5934
  if (requireLocalPrimary && assignment.primary !== deps.nodeId) {
6055
5935
  throw new NarsilError(
6056
- ErrorCodes.QUERY_ROUTING_FAILED,
5936
+ ErrorCodes.PARTITION_NOT_PRIMARY,
6057
5937
  `Node '${deps.nodeId}' is not primary for partition ${partitionId} of index '${indexName}'`,
6058
5938
  { indexName, partitionId, primaryNodeId: assignment.primary, localNodeId: deps.nodeId }
6059
5939
  );
6060
5940
  }
6061
5941
  return { partitionId, assignment };
6062
5942
  }
5943
+ function assertSufficientActiveReplicas(indexName, partitionId, assignment, deps) {
5944
+ const requiredActiveCopies = deps.waitForActiveReplicas ?? 1;
5945
+ if (requiredActiveCopies <= 1) {
5946
+ return;
5947
+ }
5948
+ const activeCopies = 1 + getInSyncReplicaTargets(assignment, deps.nodeId).length;
5949
+ if (activeCopies < requiredActiveCopies) {
5950
+ throw new NarsilError(
5951
+ ErrorCodes.INSUFFICIENT_REPLICAS,
5952
+ `Write to partition ${partitionId} of index '${indexName}' requires ${requiredActiveCopies} active copies but only ${activeCopies} are available`,
5953
+ { indexName, partitionId, requiredActiveCopies, activeCopies }
5954
+ );
5955
+ }
5956
+ }
6063
5957
  function getInSyncReplicaTargets(assignment, localNodeId) {
6064
5958
  const configuredReplicas = new Set(assignment.replicas);
6065
5959
  const targets = [];
@@ -6070,6 +5964,51 @@ function getInSyncReplicaTargets(assignment, localNodeId) {
6070
5964
  }
6071
5965
  return targets;
6072
5966
  }
5967
+ async function forwardInsertToRemote(indexName, document, docId, primaryNodeId, deps) {
5968
+ const payload = {
5969
+ indexName,
5970
+ documentId: docId,
5971
+ operation: "insert",
5972
+ document: encode(document),
5973
+ updateFields: null
5974
+ };
5975
+ const message = createForwardMessage(payload, deps.nodeId);
5976
+ const response = await sendToNode(primaryNodeId, message, deps);
5977
+ const decoded = decode(response.payload);
5978
+ if (typeof decoded.documentId !== "string") {
5979
+ throw new NarsilError(
5980
+ ErrorCodes.QUERY_ROUTING_FAILED,
5981
+ `Remote primary returned invalid forward response for index '${indexName}'`,
5982
+ { indexName, primaryNodeId }
5983
+ );
5984
+ }
5985
+ return decoded.documentId;
5986
+ }
5987
+ async function forwardRemoveToRemote(indexName, docId, primaryNodeId, deps) {
5988
+ const payload = {
5989
+ indexName,
5990
+ documentId: docId,
5991
+ operation: "remove",
5992
+ document: null,
5993
+ updateFields: null
5994
+ };
5995
+ const message = createForwardMessage(payload, deps.nodeId);
5996
+ await sendToNode(primaryNodeId, message, deps);
5997
+ }
5998
+ async function sendToNode(nodeId, message, deps) {
5999
+ const targets = await resolveNodeTargets(nodeId, deps);
6000
+ let lastError;
6001
+ for (const target of targets) {
6002
+ try {
6003
+ return await deps.transport.send(target, message);
6004
+ } catch (error) {
6005
+ lastError = error;
6006
+ }
6007
+ }
6008
+ throw lastError instanceof Error ? lastError : new Error(String(lastError));
6009
+ }
6010
+
6011
+ // src/distribution/cluster-node/write-routing/failures.ts
6073
6012
  function errorMessage(error) {
6074
6013
  return error instanceof Error ? error.message : String(error);
6075
6014
  }
@@ -6121,6 +6060,62 @@ async function rollbackPrimaryRemove(indexName, partitionId, documentId, previou
6121
6060
  }
6122
6061
  throwWriteFailure(originalError);
6123
6062
  }
6063
+ async function replicateToReplicas(entry, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
6064
+ const uniqueReplicas = [...new Set(inSyncReplicas)];
6065
+ if (uniqueReplicas.length === 0) {
6066
+ return { acknowledged: [], failed: [] };
6067
+ }
6068
+ const message = createEntryMessage(entry, sourceNodeId);
6069
+ const sendResults = await Promise.allSettled(
6070
+ uniqueReplicas.map((replicaNodeId) => sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3))
6071
+ );
6072
+ const acknowledged = [];
6073
+ const failed = [];
6074
+ for (let i = 0; i < sendResults.length; i++) {
6075
+ const result = sendResults[i];
6076
+ const replicaNodeId = uniqueReplicas[i];
6077
+ if (result.status === "rejected") {
6078
+ if (result.reason instanceof TransportError) {
6079
+ failed.push(replicaNodeId);
6080
+ continue;
6081
+ }
6082
+ throw result.reason;
6083
+ }
6084
+ const response = result.value;
6085
+ if (isMatchingAck(response, entry)) {
6086
+ acknowledged.push(replicaNodeId);
6087
+ } else {
6088
+ failed.push(replicaNodeId);
6089
+ }
6090
+ }
6091
+ return { acknowledged, failed };
6092
+ }
6093
+ async function sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3) {
6094
+ const targets = resolveNodeTargets3 === void 0 ? [replicaNodeId] : await resolveNodeTargets3(replicaNodeId);
6095
+ const resolvedTargets = targets.length > 0 ? targets : [replicaNodeId];
6096
+ let lastError;
6097
+ for (const target of resolvedTargets) {
6098
+ try {
6099
+ return await transport.send(target, message);
6100
+ } catch (error) {
6101
+ lastError = error;
6102
+ }
6103
+ }
6104
+ throw lastError instanceof Error ? lastError : new Error(String(lastError));
6105
+ }
6106
+ function isMatchingAck(response, entry) {
6107
+ if (response.type !== ReplicationMessageTypes.ACK) {
6108
+ return false;
6109
+ }
6110
+ try {
6111
+ const payload = validateAckPayload(decode(response.payload));
6112
+ return payload.seqNo === entry.seqNo && payload.partitionId === entry.partitionId && payload.indexName === entry.indexName;
6113
+ } catch {
6114
+ return false;
6115
+ }
6116
+ }
6117
+
6118
+ // src/distribution/cluster-node/write-routing/replication.ts
6124
6119
  async function assertPrimaryWriteAuthority(entry, deps) {
6125
6120
  const table = await deps.coordinator.getAllocation(entry.indexName);
6126
6121
  const currentAssignment = table?.assignments.get(entry.partitionId);
@@ -6130,7 +6125,7 @@ async function assertPrimaryWriteAuthority(entry, deps) {
6130
6125
  return;
6131
6126
  }
6132
6127
  throw new NarsilError(
6133
- ErrorCodes.QUERY_ROUTING_FAILED,
6128
+ ErrorCodes.PARTITION_NOT_PRIMARY,
6134
6129
  `Primary authority changed before acknowledging write for index '${entry.indexName}' partition ${entry.partitionId}`,
6135
6130
  {
6136
6131
  indexName: entry.indexName,
@@ -6221,7 +6216,10 @@ async function replicateEntry(entry, assignment, deps) {
6221
6216
  await removeFailedReplicasFromInsync(entry, result.failed, deps);
6222
6217
  await assertPrimaryWriteAuthority(entry, deps);
6223
6218
  }
6219
+
6220
+ // src/distribution/cluster-node/write-routing/primary-writes.ts
6224
6221
  async function applyPrimaryInsert(indexName, document, docId, partitionId, assignment, deps) {
6222
+ assertSufficientActiveReplicas(indexName, partitionId, assignment, deps);
6225
6223
  const insertedDocId = await deps.engine.insert(indexName, document, docId);
6226
6224
  const storedDocument = await deps.engine.get(indexName, insertedDocId);
6227
6225
  if (storedDocument === void 0) {
@@ -6240,6 +6238,7 @@ async function applyPrimaryInsert(indexName, document, docId, partitionId, assig
6240
6238
  return insertedDocId;
6241
6239
  }
6242
6240
  async function applyPrimaryRemove(indexName, docId, partitionId, assignment, deps) {
6241
+ assertSufficientActiveReplicas(indexName, partitionId, assignment, deps);
6243
6242
  const previousDocument = await deps.engine.get(indexName, docId);
6244
6243
  await deps.engine.remove(indexName, docId);
6245
6244
  try {
@@ -6262,21 +6261,93 @@ async function applyForwardedWrite(payload, deps) {
6262
6261
  if (payload.document === null) {
6263
6262
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Invalid ForwardPayload: insert requires a document");
6264
6263
  }
6265
- const document = decode(payload.document);
6266
- return applyPrimaryInsert(
6267
- payload.indexName,
6268
- document,
6269
- payload.documentId,
6270
- resolution.partitionId,
6271
- resolution.assignment,
6272
- deps
6273
- );
6274
- }
6275
- if (payload.operation === "remove") {
6276
- await applyPrimaryRemove(payload.indexName, payload.documentId, resolution.partitionId, resolution.assignment, deps);
6277
- return payload.documentId;
6264
+ const document = decode(payload.document);
6265
+ return applyPrimaryInsert(
6266
+ payload.indexName,
6267
+ document,
6268
+ payload.documentId,
6269
+ resolution.partitionId,
6270
+ resolution.assignment,
6271
+ deps
6272
+ );
6273
+ }
6274
+ if (payload.operation === "remove") {
6275
+ await applyPrimaryRemove(payload.indexName, payload.documentId, resolution.partitionId, resolution.assignment, deps);
6276
+ return payload.documentId;
6277
+ }
6278
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Forward update operations are not supported yet");
6279
+ }
6280
+
6281
+ // src/distribution/cluster-node/write-routing/index.ts
6282
+ var MAX_REPLICATION_FACTOR = 255;
6283
+ function validateCreateIndexOptions(partitionCount, replicationFactor) {
6284
+ if (!Number.isInteger(partitionCount) || partitionCount < 1 || partitionCount > MAX_PARTITION_COUNT) {
6285
+ throw new NarsilError(
6286
+ ErrorCodes.CONFIG_INVALID,
6287
+ `partitionCount must be an integer between 1 and ${MAX_PARTITION_COUNT}, received ${partitionCount}`,
6288
+ { partitionCount }
6289
+ );
6290
+ }
6291
+ if (!Number.isInteger(replicationFactor) || replicationFactor < 0 || replicationFactor > MAX_REPLICATION_FACTOR) {
6292
+ throw new NarsilError(
6293
+ ErrorCodes.CONFIG_INVALID,
6294
+ `replicationFactor must be an integer between 0 and ${MAX_REPLICATION_FACTOR}, received ${replicationFactor}`,
6295
+ { replicationFactor }
6296
+ );
6297
+ }
6298
+ }
6299
+ async function routeCreateIndex(name, config, options, coordinator, engine) {
6300
+ validateIndexName(name);
6301
+ const partitionCount = options?.partitionCount ?? DEFAULT_PARTITION_COUNT;
6302
+ const replicationFactor = options?.replicationFactor ?? DEFAULT_REPLICATION_FACTOR;
6303
+ validateCreateIndexOptions(partitionCount, replicationFactor);
6304
+ const constraints = {
6305
+ zoneAwareness: false,
6306
+ zoneAttribute: "zone",
6307
+ maxShardsPerNode: null
6308
+ };
6309
+ const metadata = {
6310
+ indexName: name,
6311
+ partitionCount,
6312
+ replicationFactor,
6313
+ constraints
6314
+ };
6315
+ const stored = await putIndexMetadata(coordinator, metadata);
6316
+ if (!stored) {
6317
+ throw new NarsilError(
6318
+ ErrorCodes.INDEX_ALREADY_EXISTS,
6319
+ `Index '${name}' already exists in the cluster or a partial creation is pending`,
6320
+ { indexName: name }
6321
+ );
6322
+ }
6323
+ try {
6324
+ await coordinator.putSchema(name, config.schema);
6325
+ await engine.createIndex(name, config);
6326
+ } catch (createErr) {
6327
+ let cleanupFailed = false;
6328
+ let cleanupError;
6329
+ try {
6330
+ const currentBytes = await coordinator.get(`_narsil/index/${name}/config`);
6331
+ if (currentBytes !== null) {
6332
+ await coordinator.compareAndSet(`_narsil/index/${name}/config`, currentBytes, new Uint8Array(0));
6333
+ }
6334
+ } catch (cleanErr) {
6335
+ cleanupFailed = true;
6336
+ cleanupError = cleanErr;
6337
+ }
6338
+ if (cleanupFailed) {
6339
+ throw new NarsilError(
6340
+ ErrorCodes.CONFIG_INVALID,
6341
+ `Index creation for '${name}' failed and metadata cleanup also failed. The index may be in a partial state and require manual intervention.`,
6342
+ {
6343
+ indexName: name,
6344
+ createError: createErr instanceof Error ? createErr.message : String(createErr),
6345
+ cleanupError: cleanupError instanceof Error ? cleanupError.message : String(cleanupError)
6346
+ }
6347
+ );
6348
+ }
6349
+ throw createErr;
6278
6350
  }
6279
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Forward update operations are not supported yet");
6280
6351
  }
6281
6352
  async function routeInsert(indexName, document, docId, deps) {
6282
6353
  const resolvedDocId = docId ?? generateId();
@@ -6376,158 +6447,7 @@ async function routeRemoveBatch(indexName, docIds, deps) {
6376
6447
  return { succeeded, failed };
6377
6448
  }
6378
6449
 
6379
- // src/distribution/cluster-node/message-handler.ts
6380
- function createDataNodeHandler(deps) {
6381
- return async (message, respond) => {
6382
- if (message.type === ReplicationMessageTypes.SNAPSHOT_SYNC_REQUEST) {
6383
- await handleSnapshotSyncRequest(message, respond, {
6384
- nodeId: deps.nodeId,
6385
- engine: deps.engine,
6386
- coordinator: deps.coordinator,
6387
- state: deps.snapshotSyncState,
6388
- resolveHeaderMetadata: deps.resolveHeaderMetadata
6389
- });
6390
- return;
6391
- }
6392
- try {
6393
- switch (message.type) {
6394
- case ReplicationMessageTypes.SYNC_REQUEST:
6395
- await handleSyncRequestMessage(message, respond, deps);
6396
- return;
6397
- case ReplicationMessageTypes.FORWARD:
6398
- await handleForward(message, respond, deps);
6399
- return;
6400
- case ReplicationMessageTypes.ENTRY:
6401
- await handleReplicationEntry(message, respond, deps);
6402
- return;
6403
- case QueryMessageTypes.SEARCH:
6404
- await handleSearch(message, respond, deps);
6405
- return;
6406
- case QueryMessageTypes.FETCH:
6407
- await handleFetch(message, respond, deps);
6408
- return;
6409
- case QueryMessageTypes.STATS:
6410
- await handleStats(message, respond, deps);
6411
- return;
6412
- default:
6413
- return;
6414
- }
6415
- } catch (err) {
6416
- const errorPayload = encode({
6417
- error: true,
6418
- code: err instanceof NarsilError ? err.code : "INTERNAL_ERROR",
6419
- message: err instanceof Error ? err.message : String(err)
6420
- });
6421
- respond({
6422
- type: `${message.type}.error`,
6423
- sourceId: deps.nodeId,
6424
- requestId: message.requestId,
6425
- payload: errorPayload
6426
- });
6427
- }
6428
- };
6429
- }
6430
- async function handleSyncRequestMessage(message, respond, deps) {
6431
- const request = validateSyncRequest(decode(message.payload));
6432
- const authResult = await authorizeSnapshotRequest(deps.coordinator, request.indexName, message.sourceId);
6433
- if (authResult.outcome === "denied") {
6434
- throw new NarsilError(authResult.code, authResult.reason, {
6435
- indexName: request.indexName,
6436
- partitionId: request.partitionId,
6437
- sourceNodeId: message.sourceId
6438
- });
6439
- }
6440
- const table = await deps.coordinator.getAllocation(request.indexName);
6441
- const assignment = table?.assignments.get(request.partitionId);
6442
- if (assignment === void 0) {
6443
- throw new NarsilError(
6444
- ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
6445
- `No assignment exists for sync request partition ${request.partitionId} of index '${request.indexName}'`,
6446
- { indexName: request.indexName, partitionId: request.partitionId }
6447
- );
6448
- }
6449
- if (assignment.primary !== deps.nodeId) {
6450
- throw new NarsilError(
6451
- ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
6452
- `Node '${deps.nodeId}' is not primary for sync request partition ${request.partitionId} of index '${request.indexName}'`,
6453
- {
6454
- indexName: request.indexName,
6455
- partitionId: request.partitionId,
6456
- localNodeId: deps.nodeId,
6457
- primaryNodeId: assignment.primary
6458
- }
6459
- );
6460
- }
6461
- const sourceAssigned = assignment.primary === message.sourceId || assignment.replicas.includes(message.sourceId);
6462
- if (!sourceAssigned) {
6463
- throw new NarsilError(
6464
- ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
6465
- `Node '${message.sourceId}' is not assigned to partition ${request.partitionId} of index '${request.indexName}'`,
6466
- { indexName: request.indexName, partitionId: request.partitionId, sourceNodeId: message.sourceId }
6467
- );
6468
- }
6469
- if (request.lastPrimaryTerm > assignment.primaryTerm) {
6470
- throw new NarsilError(
6471
- ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
6472
- `Replica term ${request.lastPrimaryTerm} is newer than primary term ${assignment.primaryTerm}`,
6473
- {
6474
- indexName: request.indexName,
6475
- partitionId: request.partitionId,
6476
- lastPrimaryTerm: request.lastPrimaryTerm,
6477
- primaryTerm: assignment.primaryTerm
6478
- }
6479
- );
6480
- }
6481
- const log = deps.writeDeps.getReplicationLog(request.indexName, request.partitionId);
6482
- const tier = decideSyncTier(log, request.lastSeqNo);
6483
- if (tier === "incremental") {
6484
- sendSyncEntriesResponse(message, respond, deps, log.getEntriesFrom(request.lastSeqNo + 1));
6485
- return;
6486
- }
6487
- const snapshotSeqNo = log.committedSeqNo;
6488
- const sink = createSingleResponseSink(respond);
6489
- await streamValidatedSnapshotRequest(
6490
- message,
6491
- sink,
6492
- { indexName: request.indexName, partitionId: request.partitionId },
6493
- {
6494
- nodeId: deps.nodeId,
6495
- engine: deps.engine,
6496
- coordinator: deps.coordinator,
6497
- state: deps.snapshotSyncState,
6498
- resolveHeaderMetadata: deps.resolveHeaderMetadata
6499
- },
6500
- {
6501
- metadata: {
6502
- partitionId: request.partitionId,
6503
- lastSeqNo: snapshotSeqNo,
6504
- primaryTerm: assignment.primaryTerm
6505
- },
6506
- closeOnEnd: false,
6507
- disableBuildCache: true,
6508
- buildSnapshot: async () => {
6509
- const bytes = await deps.engine.serializeReplicationPartition(request.indexName, request.partitionId);
6510
- return { bytes, checksum: crc32(bytes) };
6511
- },
6512
- afterSnapshot: (trailingSink) => {
6513
- const entries = log.getEntriesFrom(snapshotSeqNo + 1);
6514
- sendSyncEntriesResponse(message, trailingSink, deps, entries);
6515
- }
6516
- }
6517
- );
6518
- }
6519
- function sendSyncEntriesResponse(message, respond, deps, entries) {
6520
- const payload = {
6521
- entries,
6522
- isLast: true
6523
- };
6524
- respond({
6525
- type: ReplicationMessageTypes.SYNC_ENTRIES,
6526
- sourceId: deps.nodeId,
6527
- requestId: message.requestId,
6528
- payload: encode(payload)
6529
- });
6530
- }
6450
+ // src/distribution/cluster-node/message-handler/writes.ts
6531
6451
  async function handleForward(message, respond, deps) {
6532
6452
  const decoded = decode(message.payload);
6533
6453
  const payload = validateForwardPayload(decoded);
@@ -6572,7 +6492,7 @@ async function handleReplicationEntry(message, respond, deps) {
6572
6492
  }
6573
6493
  if (assignment.primaryTerm !== entry.primaryTerm) {
6574
6494
  throw new NarsilError(
6575
- ErrorCodes.REPLICATION_ENTRY_INVALID,
6495
+ ErrorCodes.REPLICATION_TERM_MISMATCH,
6576
6496
  `Replication entry term ${entry.primaryTerm} does not match allocation term ${assignment.primaryTerm}`,
6577
6497
  { indexName: entry.indexName, partitionId: entry.partitionId }
6578
6498
  );
@@ -6591,118 +6511,36 @@ async function handleReplicationEntry(message, respond, deps) {
6591
6511
  throw new NarsilError(
6592
6512
  ErrorCodes.REPLICATION_ENTRY_INVALID,
6593
6513
  `Conflicting replication entry at sequence number ${entry.seqNo}`,
6594
- { indexName: entry.indexName, partitionId: entry.partitionId, seqNo: entry.seqNo }
6595
- );
6596
- }
6597
- respond(createAckMessage(entry.seqNo, entry.partitionId, entry.indexName, deps.nodeId, message.requestId));
6598
- return;
6599
- }
6600
- const expectedSeqNo = (log.newestSeqNo ?? 0) + 1;
6601
- if (entry.seqNo !== expectedSeqNo) {
6602
- const canSeedFromCompletedBootstrap = log.entryCount === 0 && entry.seqNo > expectedSeqNo && deps.isBootstrapSynced?.(entry.indexName, entry.partitionId) === true;
6603
- if (!canSeedFromCompletedBootstrap) {
6604
- throw new NarsilError(
6605
- ErrorCodes.REPLICATION_ENTRY_INVALID,
6606
- `Out-of-order replication entry ${entry.seqNo}; expected ${expectedSeqNo}`,
6607
- { indexName: entry.indexName, partitionId: entry.partitionId, seqNo: entry.seqNo, expectedSeqNo }
6608
- );
6609
- }
6610
- deps.writeDeps.resetReplicationLog(entry.indexName, entry.partitionId, entry.seqNo, entry.primaryTerm);
6611
- log = deps.writeDeps.getReplicationLog(entry.indexName, entry.partitionId);
6612
- }
6613
- const validation = validateReplicationEntry(entry, assignment.primaryTerm, log);
6614
- if (!validation.valid) {
6615
- throw new NarsilError(
6616
- ErrorCodes.REPLICATION_ENTRY_INVALID,
6617
- `Invalid replication entry ${entry.seqNo}: ${validation.error ?? "unknown validation error"}`,
6618
- { indexName: entry.indexName, partitionId: entry.partitionId, seqNo: entry.seqNo }
6619
- );
6620
- }
6621
- await deps.engine.applyReplicationEntry(entry);
6622
- log.appendCommitted(entry);
6623
- respond(createAckMessage(entry.seqNo, entry.partitionId, entry.indexName, deps.nodeId, message.requestId));
6624
- }
6625
- async function handleSearch(message, respond, deps) {
6626
- const decoded = decode(message.payload);
6627
- const payload = validateSearchPayload(decoded);
6628
- const queryParams = {
6629
- term: payload.params.term ?? void 0,
6630
- fields: payload.params.fields ?? void 0,
6631
- filters: payload.params.filters ?? void 0,
6632
- boost: payload.params.boost ?? void 0,
6633
- scoring: payload.params.scoring,
6634
- tolerance: payload.params.tolerance ?? void 0,
6635
- minScore: payload.params.threshold ?? void 0,
6636
- limit: payload.params.limit,
6637
- offset: payload.params.offset,
6638
- searchAfter: payload.params.searchAfter ?? void 0,
6639
- sort: convertWireSortToLocal(payload.params.sort),
6640
- group: payload.params.group !== null ? { fields: [payload.params.group.field], maxPerGroup: payload.params.group.maxPerGroup } : void 0,
6641
- facets: convertWireFacetsToLocal(payload.params.facets, payload.facetShardSize ?? payload.params.facetSize),
6642
- vector: payload.params.vector !== null ? {
6643
- field: payload.params.vector.field,
6644
- value: payload.params.vector.value ?? void 0,
6645
- text: payload.params.vector.text ?? void 0,
6646
- similarity: payload.params.vector.similarity ?? void 0
6647
- } : void 0,
6648
- hybrid: payload.params.hybrid !== null ? { strategy: payload.params.hybrid.strategy, k: payload.params.hybrid.k, alpha: payload.params.hybrid.alpha } : void 0
6649
- };
6650
- const queryResult = await deps.engine.query(payload.indexName, queryParams);
6651
- const scored = queryResult.hits.map((hit) => ({
6652
- docId: hit.id,
6653
- score: hit.score,
6654
- sortValues: null
6655
- }));
6656
- const results = payload.partitionIds.map((partitionId) => ({
6657
- partitionId,
6658
- scored: partitionId === payload.partitionIds[0] ? scored : [],
6659
- totalHits: partitionId === payload.partitionIds[0] ? queryResult.count : 0
6660
- }));
6661
- const resultPayload = { results, facets: convertLocalFacetsToWire(queryResult.facets) };
6662
- respond({
6663
- type: QueryMessageTypes.SEARCH_RESULT,
6664
- sourceId: deps.nodeId,
6665
- requestId: message.requestId,
6666
- payload: encode(resultPayload)
6667
- });
6668
- }
6669
- async function handleFetch(message, respond, deps) {
6670
- const decoded = decode(message.payload);
6671
- const payload = validateFetchPayload(decoded);
6672
- const documents = [];
6673
- for (const docRef of payload.documentIds) {
6674
- const doc = await deps.engine.get(payload.indexName, docRef.docId);
6675
- if (doc !== void 0) {
6676
- documents.push({
6677
- docId: docRef.docId,
6678
- document: doc,
6679
- highlights: null
6680
- });
6514
+ { indexName: entry.indexName, partitionId: entry.partitionId, seqNo: entry.seqNo }
6515
+ );
6681
6516
  }
6517
+ respond(createAckMessage(entry.seqNo, entry.partitionId, entry.indexName, deps.nodeId, message.requestId));
6518
+ return;
6682
6519
  }
6683
- const resultPayload = { documents };
6684
- respond({
6685
- type: QueryMessageTypes.FETCH_RESULT,
6686
- sourceId: deps.nodeId,
6687
- requestId: message.requestId,
6688
- payload: encode(resultPayload)
6689
- });
6690
- }
6691
- async function handleStats(message, respond, deps) {
6692
- const decoded = decode(message.payload);
6693
- const payload = validateStatsPayload(decoded);
6694
- const stats = deps.engine.getStats(payload.indexName);
6695
- const resultPayload = {
6696
- totalDocuments: stats.documentCount,
6697
- docFrequencies: {},
6698
- totalFieldLengths: {}
6699
- };
6700
- respond({
6701
- type: QueryMessageTypes.STATS_RESULT,
6702
- sourceId: deps.nodeId,
6703
- requestId: message.requestId,
6704
- payload: encode(resultPayload)
6705
- });
6520
+ const expectedSeqNo = (log.newestSeqNo ?? 0) + 1;
6521
+ if (entry.seqNo !== expectedSeqNo) {
6522
+ const canSeedFromCompletedBootstrap = log.entryCount === 0 && entry.seqNo > expectedSeqNo && deps.isBootstrapSynced?.(entry.indexName, entry.partitionId) === true;
6523
+ if (!canSeedFromCompletedBootstrap) {
6524
+ throw new NarsilError(
6525
+ ErrorCodes.REPLICATION_ENTRY_INVALID,
6526
+ `Out-of-order replication entry ${entry.seqNo}; expected ${expectedSeqNo}`,
6527
+ { indexName: entry.indexName, partitionId: entry.partitionId, seqNo: entry.seqNo, expectedSeqNo }
6528
+ );
6529
+ }
6530
+ deps.writeDeps.resetReplicationLog(entry.indexName, entry.partitionId, entry.seqNo, entry.primaryTerm);
6531
+ log = deps.writeDeps.getReplicationLog(entry.indexName, entry.partitionId);
6532
+ }
6533
+ const validation = validateReplicationEntry(entry, assignment.primaryTerm, log);
6534
+ if (!validation.valid) {
6535
+ throw new NarsilError(
6536
+ validation.error ?? ErrorCodes.REPLICATION_ENTRY_INVALID,
6537
+ `Invalid replication entry ${entry.seqNo}`,
6538
+ { indexName: entry.indexName, partitionId: entry.partitionId, seqNo: entry.seqNo }
6539
+ );
6540
+ }
6541
+ await deps.engine.applyReplicationEntry(entry);
6542
+ log.appendCommitted(entry);
6543
+ respond(createAckMessage(entry.seqNo, entry.partitionId, entry.indexName, deps.nodeId, message.requestId));
6706
6544
  }
6707
6545
  function validateForwardPayload(decoded) {
6708
6546
  if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
@@ -6723,35 +6561,127 @@ function validateForwardPayload(decoded) {
6723
6561
  }
6724
6562
  return decoded;
6725
6563
  }
6726
- function convertWireSortToLocal(wireSort) {
6727
- if (wireSort === null || wireSort.length === 0) {
6728
- return void 0;
6729
- }
6730
- const result = {};
6731
- for (const entry of wireSort) {
6732
- result[entry.field] = entry.direction;
6733
- }
6734
- return result;
6564
+
6565
+ // src/distribution/cluster-node/message-handler/index.ts
6566
+ function createDataNodeHandler(deps) {
6567
+ return async (message, respond) => {
6568
+ if (message.type === ReplicationMessageTypes.SNAPSHOT_SYNC_REQUEST) {
6569
+ await handleSnapshotSyncRequest(message, respond, {
6570
+ nodeId: deps.nodeId,
6571
+ engine: deps.engine,
6572
+ coordinator: deps.coordinator,
6573
+ state: deps.snapshotSyncState,
6574
+ resolveHeaderMetadata: deps.resolveHeaderMetadata
6575
+ });
6576
+ return;
6577
+ }
6578
+ try {
6579
+ switch (message.type) {
6580
+ case ReplicationMessageTypes.SYNC_REQUEST:
6581
+ await handleSyncRequestMessage(message, respond, deps);
6582
+ return;
6583
+ case ReplicationMessageTypes.FORWARD:
6584
+ await handleForward(message, respond, deps);
6585
+ return;
6586
+ case ReplicationMessageTypes.ENTRY:
6587
+ await handleReplicationEntry(message, respond, deps);
6588
+ return;
6589
+ case QueryMessageTypes.SEARCH:
6590
+ await handleSearch(message, respond, deps);
6591
+ return;
6592
+ case QueryMessageTypes.FETCH:
6593
+ await handleFetch(message, respond, deps);
6594
+ return;
6595
+ case QueryMessageTypes.STATS:
6596
+ await handleStats(message, respond, deps);
6597
+ return;
6598
+ default:
6599
+ return;
6600
+ }
6601
+ } catch (err) {
6602
+ const errorPayload = encode({
6603
+ error: true,
6604
+ code: err instanceof NarsilError ? err.code : "INTERNAL_ERROR",
6605
+ message: err instanceof Error ? err.message : String(err)
6606
+ });
6607
+ respond({
6608
+ type: `${message.type}.error`,
6609
+ sourceId: deps.nodeId,
6610
+ requestId: message.requestId,
6611
+ payload: errorPayload
6612
+ });
6613
+ }
6614
+ };
6735
6615
  }
6736
- function convertWireFacetsToLocal(facets, limit) {
6737
- if (facets === null || facets.length === 0) {
6738
- return void 0;
6616
+ async function resolveNodeTargets2(config, targetNodeId) {
6617
+ const targets = [targetNodeId];
6618
+ const nodes = await config.coordinator.listNodes();
6619
+ const registration = nodes.find((node) => node.nodeId === targetNodeId);
6620
+ if (registration !== void 0 && registration.address.length > 0 && registration.address !== targetNodeId) {
6621
+ targets.push(registration.address);
6739
6622
  }
6740
- const result = {};
6741
- for (const field of facets) {
6742
- result[field] = limit !== null ? { limit } : {};
6623
+ return targets;
6624
+ }
6625
+ async function sendToNode2(config, targetNodeId, message) {
6626
+ const targets = await resolveNodeTargets2(config, targetNodeId);
6627
+ let lastError;
6628
+ for (const target of targets) {
6629
+ try {
6630
+ return await config.transport.send(target, message);
6631
+ } catch (error) {
6632
+ lastError = error;
6633
+ }
6743
6634
  }
6744
- return result;
6635
+ throw lastError instanceof Error ? lastError : new Error(String(lastError));
6745
6636
  }
6746
- function convertLocalFacetsToWire(facets) {
6747
- if (facets === void 0) {
6748
- return null;
6637
+ async function fetchDistributedDocuments(config, nodeId, engine, indexName, result, allocation) {
6638
+ const partitionCount = allocation.assignments.size;
6639
+ const nodeToDocumentIds = /* @__PURE__ */ new Map();
6640
+ for (const entry of result.scored) {
6641
+ const partitionId = resolvePartitionId(entry.docId, partitionCount);
6642
+ const assignment = allocation.assignments.get(partitionId);
6643
+ if (assignment === void 0) {
6644
+ continue;
6645
+ }
6646
+ const selectedNodeId = selectReplica(assignment, nodeId, void 0, partitionId);
6647
+ if (selectedNodeId === null) {
6648
+ continue;
6649
+ }
6650
+ let documentIds = nodeToDocumentIds.get(selectedNodeId);
6651
+ if (documentIds === void 0) {
6652
+ documentIds = [];
6653
+ nodeToDocumentIds.set(selectedNodeId, documentIds);
6654
+ }
6655
+ documentIds.push({ docId: entry.docId, partitionId });
6749
6656
  }
6750
- const result = {};
6751
- for (const [field, facet] of Object.entries(facets)) {
6752
- result[field] = Object.entries(facet.values).map(([value, count]) => ({ value, count }));
6657
+ const documents = /* @__PURE__ */ new Map();
6658
+ for (const [targetNodeId, documentIds] of nodeToDocumentIds) {
6659
+ if (targetNodeId === nodeId) {
6660
+ for (const { docId } of documentIds) {
6661
+ const document = await engine.get(indexName, docId);
6662
+ if (document !== void 0) {
6663
+ documents.set(docId, document);
6664
+ }
6665
+ }
6666
+ continue;
6667
+ }
6668
+ const fetchMessage = createFetchMessage(
6669
+ {
6670
+ indexName,
6671
+ documentIds,
6672
+ fields: null,
6673
+ highlight: null
6674
+ },
6675
+ nodeId
6676
+ );
6677
+ const response = await sendToNode2(config, targetNodeId, fetchMessage);
6678
+ const decoded = decode(response.payload);
6679
+ const payload = validateFetchResultPayload(decoded);
6680
+ for (const fetched of payload.documents) {
6681
+ documents.set(fetched.docId, fetched.document);
6682
+ }
6753
6683
  }
6754
- return result;
6684
+ return documents;
6755
6685
  }
6756
6686
 
6757
6687
  // src/distribution/cluster-node/query-conversion.ts
@@ -6798,50 +6728,275 @@ function convertLocalGroupToWire(group) {
6798
6728
  if (group === void 0) {
6799
6729
  return null;
6800
6730
  }
6801
- return {
6802
- field: group.fields[0],
6803
- maxPerGroup: group.maxPerGroup ?? 1
6804
- };
6805
- }
6806
- function convertLocalFacetsToWire2(facets) {
6807
- if (facets === void 0) {
6808
- return null;
6731
+ return {
6732
+ field: group.fields[0],
6733
+ maxPerGroup: group.maxPerGroup ?? 1
6734
+ };
6735
+ }
6736
+ function convertLocalFacetsToWire2(facets) {
6737
+ if (facets === void 0) {
6738
+ return null;
6739
+ }
6740
+ return Object.keys(facets);
6741
+ }
6742
+ function convertLocalVectorToWire(vector) {
6743
+ if (vector === void 0) {
6744
+ return null;
6745
+ }
6746
+ return {
6747
+ field: vector.field,
6748
+ value: vector.value ?? null,
6749
+ text: vector.text ?? null,
6750
+ similarity: vector.similarity ?? null
6751
+ };
6752
+ }
6753
+ function convertLocalHybridToWire(hybrid) {
6754
+ if (hybrid === void 0) {
6755
+ return null;
6756
+ }
6757
+ return {
6758
+ strategy: hybrid.strategy ?? "rrf",
6759
+ k: hybrid.k ?? 60,
6760
+ alpha: hybrid.alpha ?? 0.5
6761
+ };
6762
+ }
6763
+ function convertWireFacetsToLocal2(wireFacets) {
6764
+ const result = {};
6765
+ for (const [field, buckets] of Object.entries(wireFacets)) {
6766
+ const values = {};
6767
+ let totalCount = 0;
6768
+ for (const bucket of buckets) {
6769
+ values[bucket.value] = bucket.count;
6770
+ totalCount += bucket.count;
6771
+ }
6772
+ result[field] = { values, count: totalCount };
6773
+ }
6774
+ return result;
6775
+ }
6776
+
6777
+ // src/distribution/replication/types.ts
6778
+ var DEFAULT_LOG_RETENTION_BYTES = 268435456;
6779
+
6780
+ // src/distribution/replication/log.ts
6781
+ var ENTRY_FIXED_OVERHEAD_BYTES = 40;
6782
+ function estimateEntrySize(entry) {
6783
+ return ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
6784
+ }
6785
+ var computeChecksum = computeEntryChecksum;
6786
+ function createReplicationLog(partitionId, config) {
6787
+ const retentionBytes = config?.logRetentionBytes ?? DEFAULT_LOG_RETENTION_BYTES;
6788
+ let nextSeqNo = config?.startSeqNo ?? 1;
6789
+ let committedSeqNo = nextSeqNo - 1;
6790
+ let committedPrimaryTerm = config?.lastPrimaryTerm ?? 0;
6791
+ let entries = [];
6792
+ let headIndex = 0;
6793
+ let totalSizeBytes = 0;
6794
+ function liveCount() {
6795
+ return entries.length - headIndex;
6796
+ }
6797
+ function compact() {
6798
+ if (headIndex > 0) {
6799
+ entries = entries.slice(headIndex);
6800
+ headIndex = 0;
6801
+ }
6802
+ }
6803
+ function findIndexBySeqNo(seqNo) {
6804
+ let low = headIndex;
6805
+ let high = entries.length - 1;
6806
+ while (low <= high) {
6807
+ const mid = low + high >>> 1;
6808
+ const midSeqNo = entries[mid].seqNo;
6809
+ if (midSeqNo < seqNo) {
6810
+ low = mid + 1;
6811
+ } else if (midSeqNo > seqNo) {
6812
+ high = mid - 1;
6813
+ } else {
6814
+ return mid;
6815
+ }
6816
+ }
6817
+ return low;
6818
+ }
6819
+ function evictOldEntries() {
6820
+ while (liveCount() > 0 && totalSizeBytes > retentionBytes) {
6821
+ totalSizeBytes -= estimateEntrySize(entries[headIndex]);
6822
+ entries[headIndex] = null;
6823
+ headIndex += 1;
6824
+ }
6825
+ if (totalSizeBytes < 0) {
6826
+ totalSizeBytes = 0;
6827
+ }
6828
+ if (headIndex >= 256 || headIndex > liveCount()) {
6829
+ compact();
6830
+ }
6831
+ }
6832
+ function assertEntryFitsRetention(entry) {
6833
+ const entrySize = ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
6834
+ if (entrySize > retentionBytes) {
6835
+ throw new NarsilError(
6836
+ ErrorCodes.REPLICATION_LOG_FULL,
6837
+ `Replication log entry of ${entrySize} bytes exceeds the log retention limit of ${retentionBytes} bytes and cannot be buffered`,
6838
+ { partitionId, entrySize, retentionBytes, seqNo: entry.seqNo }
6839
+ );
6840
+ }
6841
+ }
6842
+ function storeEntry(entry) {
6843
+ entries.push(entry);
6844
+ if (entry.seqNo >= committedSeqNo) {
6845
+ committedSeqNo = entry.seqNo;
6846
+ committedPrimaryTerm = entry.primaryTerm;
6847
+ }
6848
+ totalSizeBytes += estimateEntrySize(entry);
6849
+ evictOldEntries();
6850
+ return { ...entry };
6851
+ }
6852
+ function readEntry(seqNo) {
6853
+ if (liveCount() === 0) return void 0;
6854
+ const idx = findIndexBySeqNo(seqNo);
6855
+ if (idx < entries.length && entries[idx].seqNo === seqNo) {
6856
+ return { ...entries[idx] };
6857
+ }
6858
+ return void 0;
6809
6859
  }
6810
- return Object.keys(facets);
6811
- }
6812
- function convertLocalVectorToWire(vector) {
6813
- if (vector === void 0) {
6814
- return null;
6860
+ function verifyEntryChecksum(entry) {
6861
+ const expected = computeChecksum({
6862
+ seqNo: entry.seqNo,
6863
+ primaryTerm: entry.primaryTerm,
6864
+ operation: entry.operation,
6865
+ partitionId: entry.partitionId,
6866
+ indexName: entry.indexName,
6867
+ documentId: entry.documentId,
6868
+ document: entry.document
6869
+ });
6870
+ return expected === entry.checksum;
6815
6871
  }
6816
6872
  return {
6817
- field: vector.field,
6818
- value: vector.value ?? null,
6819
- text: vector.text ?? null,
6820
- similarity: vector.similarity ?? null
6873
+ append(partial) {
6874
+ assertEntryFitsRetention(partial);
6875
+ const seqNo = nextSeqNo;
6876
+ nextSeqNo += 1;
6877
+ const entryWithoutChecksum = {
6878
+ seqNo,
6879
+ primaryTerm: partial.primaryTerm,
6880
+ operation: partial.operation,
6881
+ partitionId,
6882
+ indexName: partial.indexName,
6883
+ documentId: partial.documentId,
6884
+ document: partial.document
6885
+ };
6886
+ const checksum = computeChecksum(entryWithoutChecksum);
6887
+ const entry = { ...entryWithoutChecksum, checksum };
6888
+ return storeEntry(entry);
6889
+ },
6890
+ appendCommitted(entry) {
6891
+ if (!Number.isSafeInteger(entry.seqNo) || entry.seqNo < 1) {
6892
+ throw new NarsilError(
6893
+ ErrorCodes.REPLICATION_ENTRY_INVALID,
6894
+ `Invalid replication sequence number ${entry.seqNo}`,
6895
+ {
6896
+ seqNo: entry.seqNo,
6897
+ partitionId
6898
+ }
6899
+ );
6900
+ }
6901
+ if (entry.partitionId !== partitionId) {
6902
+ throw new NarsilError(
6903
+ ErrorCodes.REPLICATION_ENTRY_INVALID,
6904
+ `Replication entry partition ${entry.partitionId} does not match log partition ${partitionId}`,
6905
+ { entryPartitionId: entry.partitionId, logPartitionId: partitionId }
6906
+ );
6907
+ }
6908
+ const existing = readEntry(entry.seqNo);
6909
+ if (existing !== void 0) {
6910
+ if (existing.checksum !== entry.checksum) {
6911
+ throw new NarsilError(
6912
+ ErrorCodes.REPLICATION_ENTRY_INVALID,
6913
+ `Conflicting replication entry at sequence number ${entry.seqNo}`,
6914
+ { seqNo: entry.seqNo, partitionId }
6915
+ );
6916
+ }
6917
+ return existing;
6918
+ }
6919
+ if (entry.seqNo !== nextSeqNo) {
6920
+ throw new NarsilError(
6921
+ ErrorCodes.REPLICATION_ENTRY_INVALID,
6922
+ `Out-of-order replication entry ${entry.seqNo}; expected ${nextSeqNo}`,
6923
+ { seqNo: entry.seqNo, expectedSeqNo: nextSeqNo, partitionId }
6924
+ );
6925
+ }
6926
+ if (!verifyEntryChecksum(entry)) {
6927
+ throw new NarsilError(
6928
+ ErrorCodes.REPLICATION_ENTRY_CORRUPT,
6929
+ `Invalid checksum for replication entry ${entry.seqNo}`,
6930
+ { seqNo: entry.seqNo, partitionId }
6931
+ );
6932
+ }
6933
+ assertEntryFitsRetention(entry);
6934
+ nextSeqNo = entry.seqNo + 1;
6935
+ return storeEntry({ ...entry });
6936
+ },
6937
+ getEntriesFrom(fromSeqNo) {
6938
+ if (liveCount() === 0) return [];
6939
+ const idx = findIndexBySeqNo(fromSeqNo);
6940
+ if (idx >= entries.length) return [];
6941
+ return entries.slice(idx).map((e) => ({ ...e }));
6942
+ },
6943
+ getEntry(seqNo) {
6944
+ return readEntry(seqNo);
6945
+ },
6946
+ verifyChecksum(entry) {
6947
+ return verifyEntryChecksum(entry);
6948
+ },
6949
+ get committedSeqNo() {
6950
+ return committedSeqNo;
6951
+ },
6952
+ get committedPrimaryTerm() {
6953
+ return committedPrimaryTerm;
6954
+ },
6955
+ get oldestSeqNo() {
6956
+ if (liveCount() === 0) return void 0;
6957
+ return entries[headIndex].seqNo;
6958
+ },
6959
+ get newestSeqNo() {
6960
+ if (liveCount() === 0) return void 0;
6961
+ return entries[entries.length - 1].seqNo;
6962
+ },
6963
+ get entryCount() {
6964
+ return liveCount();
6965
+ },
6966
+ get sizeBytes() {
6967
+ return totalSizeBytes;
6968
+ },
6969
+ clear() {
6970
+ entries = [];
6971
+ headIndex = 0;
6972
+ totalSizeBytes = 0;
6973
+ committedSeqNo = nextSeqNo - 1;
6974
+ }
6821
6975
  };
6822
6976
  }
6823
- function convertLocalHybridToWire(hybrid) {
6824
- if (hybrid === void 0) {
6825
- return null;
6826
- }
6827
- return {
6828
- strategy: hybrid.strategy ?? "rrf",
6829
- k: hybrid.k ?? 60,
6830
- alpha: hybrid.alpha ?? 0.5
6831
- };
6977
+
6978
+ // src/distribution/cluster-node/replication-logs.ts
6979
+ function replicationLogKey(indexName, partitionId) {
6980
+ return `${indexName}:${partitionId}`;
6832
6981
  }
6833
- function convertWireFacetsToLocal2(wireFacets) {
6834
- const result = {};
6835
- for (const [field, buckets] of Object.entries(wireFacets)) {
6836
- const values = {};
6837
- let totalCount = 0;
6838
- for (const bucket of buckets) {
6839
- values[bucket.value] = bucket.count;
6840
- totalCount += bucket.count;
6841
- }
6842
- result[field] = { values, count: totalCount };
6843
- }
6844
- return result;
6982
+ function getReplicationLog(replicationLogs, indexName, partitionId, replicationConfig) {
6983
+ const key = replicationLogKey(indexName, partitionId);
6984
+ let log = replicationLogs.get(key);
6985
+ if (log === void 0) {
6986
+ log = createReplicationLog(partitionId, { logRetentionBytes: replicationConfig?.logRetentionBytes });
6987
+ replicationLogs.set(key, log);
6988
+ }
6989
+ return log;
6990
+ }
6991
+ function seedReplicationLog(replicationLogs, indexName, partitionId, startSeqNo, lastPrimaryTerm = 0, replicationConfig) {
6992
+ replicationLogs.set(
6993
+ replicationLogKey(indexName, partitionId),
6994
+ createReplicationLog(partitionId, {
6995
+ startSeqNo,
6996
+ lastPrimaryTerm,
6997
+ logRetentionBytes: replicationConfig?.logRetentionBytes
6998
+ })
6999
+ );
6845
7000
  }
6846
7001
 
6847
7002
  // src/distribution/cluster-node/transport-listener.ts
@@ -6897,6 +7052,53 @@ function createSingleResponseRespond(respond) {
6897
7052
  };
6898
7053
  }
6899
7054
 
7055
+ // src/distribution/cluster-node/validate-config.ts
7056
+ var MAX_WAIT_FOR_ACTIVE_REPLICAS = 255;
7057
+ function validateClusterNodeConfig(config) {
7058
+ if (config.address.length === 0) {
7059
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "ClusterNodeConfig.address must not be empty");
7060
+ }
7061
+ if (config.roles !== void 0) {
7062
+ if (config.roles.length === 0) {
7063
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "ClusterNodeConfig.roles must contain at least one role");
7064
+ }
7065
+ const validRoles = /* @__PURE__ */ new Set(["data", "coordinator", "controller"]);
7066
+ for (const role of config.roles) {
7067
+ if (!validRoles.has(role)) {
7068
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Invalid role: '${role}'`, { role });
7069
+ }
7070
+ }
7071
+ }
7072
+ if (config.capacity !== void 0) {
7073
+ if (config.capacity.memoryBytes <= 0 || !Number.isFinite(config.capacity.memoryBytes)) {
7074
+ throw new NarsilError(
7075
+ ErrorCodes.CONFIG_INVALID,
7076
+ "ClusterNodeConfig.capacity.memoryBytes must be a positive finite number"
7077
+ );
7078
+ }
7079
+ if (config.capacity.cpuCores <= 0 || !Number.isInteger(config.capacity.cpuCores)) {
7080
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "ClusterNodeConfig.capacity.cpuCores must be a positive integer");
7081
+ }
7082
+ }
7083
+ if (config.replication !== void 0) {
7084
+ const { logRetentionBytes, waitForActiveReplicas } = config.replication;
7085
+ if (logRetentionBytes !== void 0 && (!Number.isInteger(logRetentionBytes) || logRetentionBytes <= 0)) {
7086
+ throw new NarsilError(
7087
+ ErrorCodes.CONFIG_INVALID,
7088
+ "ClusterNodeConfig.replication.logRetentionBytes must be a positive integer",
7089
+ { logRetentionBytes }
7090
+ );
7091
+ }
7092
+ if (waitForActiveReplicas !== void 0 && (!Number.isInteger(waitForActiveReplicas) || waitForActiveReplicas < 1 || waitForActiveReplicas > MAX_WAIT_FOR_ACTIVE_REPLICAS)) {
7093
+ throw new NarsilError(
7094
+ ErrorCodes.CONFIG_INVALID,
7095
+ `ClusterNodeConfig.replication.waitForActiveReplicas must be an integer between 1 and ${MAX_WAIT_FOR_ACTIVE_REPLICAS}`,
7096
+ { waitForActiveReplicas }
7097
+ );
7098
+ }
7099
+ }
7100
+ }
7101
+
6900
7102
  // src/distribution/cluster-node/index.ts
6901
7103
  var SPEC_VERSION = "1.0";
6902
7104
  async function createClusterNode(config) {
@@ -6908,6 +7110,21 @@ async function createClusterNode(config) {
6908
7110
  const bootstrapSyncState = createBootstrapSyncState();
6909
7111
  const snapshotSyncHandlerState = createSnapshotSyncHandlerState();
6910
7112
  const replicationLogs = /* @__PURE__ */ new Map();
7113
+ function replicationLogKey2(indexName, partitionId) {
7114
+ return replicationLogKey(indexName, partitionId);
7115
+ }
7116
+ function getReplicationLog2(indexName, partitionId) {
7117
+ return getReplicationLog(replicationLogs, indexName, partitionId, config.replication);
7118
+ }
7119
+ function seedReplicationLog2(indexName, partitionId, startSeqNo, lastPrimaryTerm = 0) {
7120
+ seedReplicationLog(replicationLogs, indexName, partitionId, startSeqNo, lastPrimaryTerm, config.replication);
7121
+ }
7122
+ async function resolveNodeTargets3(targetNodeId) {
7123
+ return resolveNodeTargets2(config, targetNodeId);
7124
+ }
7125
+ async function fetchDistributedDocuments2(indexName, result, allocation) {
7126
+ return fetchDistributedDocuments(config, nodeId, engine, indexName, result, allocation);
7127
+ }
6911
7128
  const forwardOnError = (error) => {
6912
7129
  if (config.onError === void 0) {
6913
7130
  return;
@@ -6931,9 +7148,10 @@ async function createClusterNode(config) {
6931
7148
  coordinator: config.coordinator,
6932
7149
  engine,
6933
7150
  transport: config.transport,
6934
- getReplicationLog,
6935
- resetReplicationLog: seedReplicationLog,
6936
- resolveNodeTargets: resolveNodeTargets2
7151
+ getReplicationLog: getReplicationLog2,
7152
+ resetReplicationLog: seedReplicationLog2,
7153
+ resolveNodeTargets: resolveNodeTargets3,
7154
+ waitForActiveReplicas: config.replication?.waitForActiveReplicas
6937
7155
  };
6938
7156
  let lifecycle = null;
6939
7157
  let controller = null;
@@ -6956,16 +7174,16 @@ async function createClusterNode(config) {
6956
7174
  coordinator: config.coordinator,
6957
7175
  transport: config.transport,
6958
7176
  sourceNodeId: nodeId,
6959
- resolveNodeTargets: resolveNodeTargets2,
6960
- getReplicationLog,
6961
- resetReplicationLog: seedReplicationLog,
7177
+ resolveNodeTargets: resolveNodeTargets3,
7178
+ getReplicationLog: getReplicationLog2,
7179
+ resetReplicationLog: seedReplicationLog2,
6962
7180
  applyReplicationEntry: engine.applyReplicationEntry,
6963
7181
  restoreReplicationPartition: engine.restoreReplicationPartition,
6964
7182
  onError: forwardOnError
6965
7183
  }),
6966
7184
  onRemovePartition: (indexName, partitionId) => {
6967
7185
  clearBootstrapSyncIndex(bootstrapSyncState, indexName, partitionId);
6968
- replicationLogs.delete(replicationLogKey(indexName, partitionId));
7186
+ replicationLogs.delete(replicationLogKey2(indexName, partitionId));
6969
7187
  void cleanupRemovedPartition(indexName, partitionId, {
6970
7188
  engine,
6971
7189
  coordinator: config.coordinator,
@@ -7061,10 +7279,10 @@ async function createClusterNode(config) {
7061
7279
  transport: config.transport,
7062
7280
  sourceNodeId: nodeId,
7063
7281
  getAllocation: (idx) => config.coordinator.getAllocation(idx),
7064
- resolveNodeTargets: resolveNodeTargets2
7282
+ resolveNodeTargets: resolveNodeTargets3
7065
7283
  };
7066
7284
  const distributed = await distributedQuery(indexName, wireParams, queryDeps);
7067
- const documents = await fetchDistributedDocuments(indexName, distributed, allocation);
7285
+ const documents = await fetchDistributedDocuments2(indexName, distributed, allocation);
7068
7286
  return distributedResultToLocal(distributed, documents);
7069
7287
  });
7070
7288
  },
@@ -7115,24 +7333,6 @@ async function createClusterNode(config) {
7115
7333
  }
7116
7334
  };
7117
7335
  return node;
7118
- function replicationLogKey(indexName, partitionId) {
7119
- return `${indexName}:${partitionId}`;
7120
- }
7121
- function getReplicationLog(indexName, partitionId) {
7122
- const key = replicationLogKey(indexName, partitionId);
7123
- let log = replicationLogs.get(key);
7124
- if (log === void 0) {
7125
- log = createReplicationLog(partitionId);
7126
- replicationLogs.set(key, log);
7127
- }
7128
- return log;
7129
- }
7130
- function seedReplicationLog(indexName, partitionId, startSeqNo, lastPrimaryTerm = 0) {
7131
- replicationLogs.set(
7132
- replicationLogKey(indexName, partitionId),
7133
- createReplicationLog(partitionId, { startSeqNo, lastPrimaryTerm })
7134
- );
7135
- }
7136
7336
  async function resolveSnapshotHeaderMetadata(indexName, partitionId) {
7137
7337
  const fallback = await defaultSnapshotHeaderMetadataProvider(config.coordinator, indexName);
7138
7338
  if (partitionId === null) {
@@ -7140,110 +7340,13 @@ async function createClusterNode(config) {
7140
7340
  }
7141
7341
  const allocation = await config.coordinator.getAllocation(indexName);
7142
7342
  const assignment = allocation?.assignments.get(partitionId);
7143
- const log = getReplicationLog(indexName, partitionId);
7343
+ const log = getReplicationLog2(indexName, partitionId);
7144
7344
  return {
7145
7345
  partitionId,
7146
7346
  primaryTerm: assignment?.primaryTerm ?? fallback.primaryTerm,
7147
7347
  lastSeqNo: log.newestSeqNo ?? 0
7148
7348
  };
7149
7349
  }
7150
- async function resolveNodeTargets2(targetNodeId) {
7151
- const targets = [targetNodeId];
7152
- const nodes = await config.coordinator.listNodes();
7153
- const registration2 = nodes.find((node2) => node2.nodeId === targetNodeId);
7154
- if (registration2 !== void 0 && registration2.address.length > 0 && registration2.address !== targetNodeId) {
7155
- targets.push(registration2.address);
7156
- }
7157
- return targets;
7158
- }
7159
- async function fetchDistributedDocuments(indexName, result, allocation) {
7160
- const partitionCount = allocation.assignments.size;
7161
- const nodeToDocumentIds = /* @__PURE__ */ new Map();
7162
- for (const entry of result.scored) {
7163
- const partitionId = resolvePartitionId(entry.docId, partitionCount);
7164
- const assignment = allocation.assignments.get(partitionId);
7165
- if (assignment === void 0) {
7166
- continue;
7167
- }
7168
- const selectedNodeId = selectReplica(assignment, nodeId, void 0, partitionId);
7169
- if (selectedNodeId === null) {
7170
- continue;
7171
- }
7172
- let documentIds = nodeToDocumentIds.get(selectedNodeId);
7173
- if (documentIds === void 0) {
7174
- documentIds = [];
7175
- nodeToDocumentIds.set(selectedNodeId, documentIds);
7176
- }
7177
- documentIds.push({ docId: entry.docId, partitionId });
7178
- }
7179
- const documents = /* @__PURE__ */ new Map();
7180
- for (const [targetNodeId, documentIds] of nodeToDocumentIds) {
7181
- if (targetNodeId === nodeId) {
7182
- for (const { docId } of documentIds) {
7183
- const document = await engine.get(indexName, docId);
7184
- if (document !== void 0) {
7185
- documents.set(docId, document);
7186
- }
7187
- }
7188
- continue;
7189
- }
7190
- const fetchMessage = createFetchMessage(
7191
- {
7192
- indexName,
7193
- documentIds,
7194
- fields: null,
7195
- highlight: null
7196
- },
7197
- nodeId
7198
- );
7199
- const response = await sendToNode2(targetNodeId, fetchMessage);
7200
- const decoded = decode(response.payload);
7201
- const payload = validateFetchResultPayload(decoded);
7202
- for (const fetched of payload.documents) {
7203
- documents.set(fetched.docId, fetched.document);
7204
- }
7205
- }
7206
- return documents;
7207
- }
7208
- async function sendToNode2(targetNodeId, message) {
7209
- const targets = await resolveNodeTargets2(targetNodeId);
7210
- let lastError;
7211
- for (const target of targets) {
7212
- try {
7213
- return await config.transport.send(target, message);
7214
- } catch (error) {
7215
- lastError = error;
7216
- }
7217
- }
7218
- throw lastError instanceof Error ? lastError : new Error(String(lastError));
7219
- }
7220
- }
7221
- function validateClusterNodeConfig(config) {
7222
- if (config.address.length === 0) {
7223
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, "ClusterNodeConfig.address must not be empty");
7224
- }
7225
- if (config.roles !== void 0) {
7226
- if (config.roles.length === 0) {
7227
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, "ClusterNodeConfig.roles must contain at least one role");
7228
- }
7229
- const validRoles = /* @__PURE__ */ new Set(["data", "coordinator", "controller"]);
7230
- for (const role of config.roles) {
7231
- if (!validRoles.has(role)) {
7232
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Invalid role: '${role}'`, { role });
7233
- }
7234
- }
7235
- }
7236
- if (config.capacity !== void 0) {
7237
- if (config.capacity.memoryBytes <= 0 || !Number.isFinite(config.capacity.memoryBytes)) {
7238
- throw new NarsilError(
7239
- ErrorCodes.CONFIG_INVALID,
7240
- "ClusterNodeConfig.capacity.memoryBytes must be a positive finite number"
7241
- );
7242
- }
7243
- if (config.capacity.cpuCores <= 0 || !Number.isInteger(config.capacity.cpuCores)) {
7244
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, "ClusterNodeConfig.capacity.cpuCores must be a positive integer");
7245
- }
7246
- }
7247
7350
  }
7248
7351
 
7249
7352
  // src/distribution/transport/in-memory.ts