@delali/narsil 0.3.1-next.185 → 0.3.1-next.189

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/README.md +1 -0
  2. package/dist/adapters/filesystem.mjs +2 -2
  3. package/dist/adapters/gcs.mjs +2 -2
  4. package/dist/adapters/indexeddb.mjs +1 -1
  5. package/dist/adapters/s3.mjs +2 -2
  6. package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
  7. package/dist/browser/{chunk-4ZICWS4V.mjs → chunk-QAOVAYAG.mjs} +1 -1
  8. package/dist/browser/{chunk-CPSHS46N.mjs → chunk-WFQLSNCO.mjs} +27 -22
  9. package/dist/browser/{chunk-2Q45HI5H.mjs → chunk-YINZFPLT.mjs} +7 -0
  10. package/dist/browser/{module-XVLHJ7T7.mjs → module-VOTN3Q6X.mjs} +15 -0
  11. package/dist/browser/{optional-writes-ZTBHYQMN.mjs → optional-writes-J5BHKEQL.mjs} +21 -8
  12. package/dist/browser/{storage-leaf-4XMCITJ6.mjs → storage-leaf-62LA56U2.mjs} +14558 -13739
  13. package/dist/{capabilities-DAHXphRn.d.ts → capabilities-CLnQHIqV.d.ts} +51 -9
  14. package/dist/{chunk-KLVMK3R5.mjs → chunk-2CKDCC65.mjs} +1 -1
  15. package/dist/{chunk-7WXGUTWK.mjs → chunk-3JRN23HA.mjs} +17 -6
  16. package/dist/{chunk-5BUEBIXR.mjs → chunk-3OZ2UQJV.mjs} +18 -5
  17. package/dist/{chunk-VYSWVQCE.mjs → chunk-5HPILB7R.mjs} +2 -2
  18. package/dist/{chunk-FBQYC5OX.mjs → chunk-6G3VRWH5.mjs} +16 -13
  19. package/dist/{chunk-QJGDB3WR.mjs → chunk-6GHX35ZM.mjs} +2 -2
  20. package/dist/{chunk-H4QYQ5DE.mjs → chunk-7NCVV3ST.mjs} +2 -2
  21. package/dist/{chunk-X5NHWOXT.mjs → chunk-7P5YWDRQ.mjs} +2 -2
  22. package/dist/{chunk-F4OOBPKA.mjs → chunk-AOYQTX6W.mjs} +1 -1
  23. package/dist/{chunk-D6FYUTFN.mjs → chunk-ASZLNVWW.mjs} +1 -1
  24. package/dist/{chunk-DU23J5JG.mjs → chunk-EFS5GVBK.mjs} +6 -6
  25. package/dist/{chunk-FETZYCH7.mjs → chunk-F6F4G322.mjs} +26 -1
  26. package/dist/{chunk-IE3Q62QD.mjs → chunk-GR5GMFJJ.mjs} +1 -1
  27. package/dist/{chunk-Y7QUFIGV.mjs → chunk-GXED6R3Q.mjs} +2 -2
  28. package/dist/{chunk-LR6JJVML.mjs → chunk-GYLRKKUT.mjs} +764 -699
  29. package/dist/{react-native/chunks/chunk-ZUFOYZ5P.mjs → chunk-IDWDYS2N.mjs} +1 -1
  30. package/dist/{chunk-YM2HI5N5.mjs → chunk-IL5DRCNU.mjs} +2 -2
  31. package/dist/{chunk-BVZWWMOU.mjs → chunk-IWKYAO45.mjs} +2 -2
  32. package/dist/chunk-JEMK23OY.mjs +602 -0
  33. package/dist/{chunk-2YWWHMVC.mjs → chunk-JOWXHENJ.mjs} +2 -2
  34. package/dist/{chunk-SL53DHVE.mjs → chunk-JXDY3ZA3.mjs} +66 -40
  35. package/dist/{chunk-WLFOLAQI.mjs → chunk-KBU3ZFJN.mjs} +2 -2
  36. package/dist/{chunk-N3M4NPML.mjs → chunk-M2Z5WVWW.mjs} +1 -1
  37. package/dist/{chunk-2KCCQK4R.mjs → chunk-NFAWR2O4.mjs} +2 -2
  38. package/dist/{chunk-EWGIF6FQ.mjs → chunk-NJJN4FMM.mjs} +1 -1
  39. package/dist/{chunk-MHBG33R5.mjs → chunk-NM2ITITI.mjs} +76 -46
  40. package/dist/{chunk-WSIU6YXQ.mjs → chunk-NULDX3ID.mjs} +1732 -743
  41. package/dist/{chunk-II6XF6M4.mjs → chunk-P2IPUYT2.mjs} +6 -6
  42. package/dist/{chunk-C7YQHCSO.mjs → chunk-P4F4UWLC.mjs} +1 -1
  43. package/dist/{chunk-A3OAFUZI.mjs → chunk-PLLKT5NU.mjs} +2 -2
  44. package/dist/{chunk-FCQMZDRI.mjs → chunk-PPD4J2JI.mjs} +3 -3
  45. package/dist/{chunk-EDEE7ETQ.mjs → chunk-PZYBWMCL.mjs} +3 -3
  46. package/dist/{chunk-I2WJCRIH.mjs → chunk-R2PJKSNL.mjs} +1 -1
  47. package/dist/{chunk-M5I64AQW.mjs → chunk-REBDKXAH.mjs} +33 -33
  48. package/dist/{chunk-6SZQETSC.mjs → chunk-RORNUNXW.mjs} +5 -2
  49. package/dist/{chunk-CYCVPTTR.mjs → chunk-SCZI7COG.mjs} +2 -2
  50. package/dist/{chunk-O337MDRJ.mjs → chunk-U5IMECEX.mjs} +38 -13
  51. package/dist/{chunk-CODGHJVW.mjs → chunk-X2NE4VJI.mjs} +40 -12
  52. package/dist/{chunk-2GONI5HL.mjs → chunk-YFMM7QOT.mjs} +5 -4
  53. package/dist/{chunk-JDG6CCXM.mjs → chunk-YQ4RZIIN.mjs} +2 -2
  54. package/dist/{chunk-G3XRHAJA.mjs → chunk-ZGMCJXY5.mjs} +3 -3
  55. package/dist/{chunk-Q3ZZH5Z7.mjs → chunk-ZKOQR7EA.mjs} +7 -0
  56. package/dist/{chunk-CKEOTSF5.mjs → chunk-ZS2YUB2L.mjs} +6 -3
  57. package/dist/{client-DpqtT2XG.d.ts → client-uB0kp2yT.d.ts} +88 -5
  58. package/dist/client.d.ts +7 -7
  59. package/dist/client.mjs +3 -1265
  60. package/dist/{config-D9WR3MhO.d.ts → config-BvHQ-C7U.d.ts} +3 -3
  61. package/dist/{config-C687-reW.d.ts → config-PqgNY1H7.d.ts} +1 -1
  62. package/dist/{core-leaf-NEUM7PU3.mjs → core-leaf-TAR3GJVE.mjs} +3 -3
  63. package/dist/{direct-executor-37IH3ISS.mjs → direct-executor-QLBSVTEE.mjs} +8 -8
  64. package/dist/distribution/coordinator/etcd.d.ts +1 -1
  65. package/dist/distribution/coordinator/etcd.mjs +35 -15
  66. package/dist/distribution/coordinator/in-memory.d.ts +1 -1
  67. package/dist/distribution/coordinator/in-memory.mjs +16 -6
  68. package/dist/distribution/transport/grpc.mjs +2 -2
  69. package/dist/distribution/transport/tcp.mjs +2 -2
  70. package/dist/distribution.d.ts +70 -9
  71. package/dist/distribution.mjs +1694 -679
  72. package/dist/{durability-config-NWU2G2SJ.mjs → durability-config-VWS2K24E.mjs} +1 -1
  73. package/dist/{react-native/chunks/embed-4LBSQD7U.mjs → embed-4PHHKGBK.mjs} +1 -1
  74. package/dist/embeddings/openai.mjs +2 -2
  75. package/dist/{engine-De6JzTW8.d.ts → engine-qjAxJSKk.d.ts} +206 -7
  76. package/dist/{index-CBUl4z3E.d.ts → index-BaQIAnk0.d.ts} +137 -111
  77. package/dist/index.browser.mjs +3 -3
  78. package/dist/index.d.ts +4 -4
  79. package/dist/index.mjs +8 -8
  80. package/dist/index.react-native.mjs +9 -9
  81. package/dist/{long-query-thread-DIPJBMV7.mjs → long-query-thread-IKD4Z37M.mjs} +21 -21
  82. package/dist/{module-Z3CL4GJ4.mjs → module-NIVZ4SF2.mjs} +22 -10
  83. package/dist/{msgpack-leaf-7KBYOVTF.mjs → msgpack-leaf-H5ZYW7S7.mjs} +4 -4
  84. package/dist/{optional-reads-ENKMVQ4Q.mjs → optional-reads-5HFMF2QZ.mjs} +2 -2
  85. package/dist/{optional-writes-YIVJMQ4D.mjs → optional-writes-DVTVX2XY.mjs} +31 -18
  86. package/dist/{paged-partition-archive-IZ6U7DVN.mjs → paged-partition-archive-ES4Z5SM4.mjs} +20 -20
  87. package/dist/partial-words.browser.mjs +3 -3
  88. package/dist/partial-words.mjs +8 -8
  89. package/dist/pattern.browser.mjs +2 -2
  90. package/dist/pattern.mjs +3 -3
  91. package/dist/persistence/durability/checkpoint-worker.d.ts +7 -0
  92. package/dist/persistence/durability/checkpoint-worker.mjs +18 -18
  93. package/dist/{phrase-module-ZY3TLTQA.mjs → phrase-module-EFESWOCR.mjs} +2 -2
  94. package/dist/{query-actions-7EOKT3WD.mjs → query-actions-IC5UEWZG.mjs} +8 -8
  95. package/dist/query-suggestions/merge-worker.mjs +11 -11
  96. package/dist/query-suggestions.browser.mjs +3 -3
  97. package/dist/query-suggestions.d.ts +1 -1
  98. package/dist/query-suggestions.mjs +9 -9
  99. package/dist/react-native/chunks/{chunk-T5HF3FW3.mjs → chunk-3553RYCS.mjs} +3 -3
  100. package/dist/react-native/chunks/{chunk-TQ26BKBH.mjs → chunk-43R4GCM7.mjs} +7 -0
  101. package/dist/react-native/chunks/{chunk-BN6FUB7N.mjs → chunk-AN4HKLB4.mjs} +1 -1
  102. package/dist/react-native/chunks/{chunk-PVMEHBQO.mjs → chunk-CW2ZFXSE.mjs} +1 -1
  103. package/dist/react-native/chunks/{chunk-ZQPSXN6C.mjs → chunk-FJHZWUCB.mjs} +5 -5
  104. package/dist/react-native/chunks/{chunk-NLEFMPY3.mjs → chunk-G434H5WG.mjs} +15 -4
  105. package/dist/react-native/chunks/{chunk-IEY4DARA.mjs → chunk-GXY4R3GA.mjs} +2 -2
  106. package/dist/react-native/chunks/{chunk-SK3DPAI4.mjs → chunk-HTBY4GON.mjs} +2 -2
  107. package/dist/react-native/chunks/{chunk-7TPO4FLE.mjs → chunk-JZOHEHBD.mjs} +3 -3
  108. package/dist/react-native/chunks/{chunk-ZDDJVR7X.mjs → chunk-MRH36RN3.mjs} +2 -2
  109. package/dist/react-native/chunks/{chunk-YOTP2FPE.mjs → chunk-UO44MA5M.mjs} +1 -1
  110. package/dist/react-native/chunks/{chunk-WFQ6YQZA.mjs → chunk-UYKIV2KT.mjs} +1 -1
  111. package/dist/{chunk-4JBTALUC.mjs → react-native/chunks/chunk-VSN76LWE.mjs} +1 -1
  112. package/dist/react-native/chunks/{chunk-CBU4M3EU.mjs → chunk-XLYQ2ZMC.mjs} +222 -47
  113. package/dist/react-native/chunks/{chunk-OZ3XEM6D.mjs → chunk-ZUJCUPVS.mjs} +1 -1
  114. package/dist/react-native/chunks/{core-leaf-TPBRRWVN.mjs → core-leaf-5PVST5PR.mjs} +4 -4
  115. package/dist/react-native/chunks/{durability-config-RVTWSXPH.mjs → durability-config-FCSD4MXY.mjs} +1 -1
  116. package/dist/{embed-2QMYCD2Q.mjs → react-native/chunks/embed-HA2I7RCF.mjs} +1 -1
  117. package/dist/react-native/chunks/{gzip.react-native-DLO4WW27.mjs → gzip.react-native-CVM4NQG2.mjs} +2 -2
  118. package/dist/react-native/chunks/{module-COLNJNML.mjs → module-TN7HXWJ3.mjs} +21 -6
  119. package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
  120. package/dist/react-native/chunks/{optional-reads-74HYI4DO.mjs → optional-reads-AFPO2FEB.mjs} +3 -3
  121. package/dist/react-native/chunks/{optional-writes-QETIM6BI.mjs → optional-writes-2UPNQNAN.mjs} +31 -18
  122. package/dist/react-native/chunks/{phrase-module-UC4YGEBB.mjs → phrase-module-IGYBK7XH.mjs} +3 -3
  123. package/dist/react-native/chunks/{resolve-vector-text-FU7D4QP5.mjs → resolve-vector-text-GE2CRJKI.mjs} +1 -1
  124. package/dist/react-native/chunks/{restore-queue-3E2MJIJS.mjs → restore-queue-DEAWTVHW.mjs} +12 -12
  125. package/dist/react-native/chunks/{segment-codec-leaf-IE4DXUZV.mjs → segment-codec-leaf-3LZ6HZFD.mjs} +10 -10
  126. package/dist/react-native/chunks/{segment-merge-leaf-62LQ5WZB.mjs → segment-merge-leaf-5OIWAPCI.mjs} +11 -11
  127. package/dist/react-native/chunks/{storage-leaf-LNOLKWZB.mjs → storage-leaf-LC4RZU3T.mjs} +15614 -14805
  128. package/dist/react-native/chunks/{store-leaf-Y7CBOPFZ.mjs → store-leaf-LQ6LWM2Q.mjs} +11 -11
  129. package/dist/react-native/chunks/{suggestions-leaf-PSUGL6MI.mjs → suggestions-leaf-GPNU6I72.mjs} +11 -11
  130. package/dist/react-native/chunks/{worker-copies-MW5O5U3D.mjs → worker-copies-32TCHOYP.mjs} +10 -10
  131. package/dist/react-native/partial-words.mjs +9 -9
  132. package/dist/react-native/pattern.mjs +5 -5
  133. package/dist/react-native/query-suggestions.mjs +9 -9
  134. package/dist/react-native/sync.mjs +433 -51
  135. package/dist/react-native/synonyms.mjs +9 -9
  136. package/dist/react.d.ts +8 -8
  137. package/dist/react.mjs +7 -0
  138. package/dist/{rerank-KTV5L7MZ.mjs → rerank-ELMGHQ3T.mjs} +8 -8
  139. package/dist/rerankers/http.mjs +2 -2
  140. package/dist/{resolve-vector-text-SD6BEMKZ.mjs → resolve-vector-text-OD4J3WHF.mjs} +2 -2
  141. package/dist/{restore-queue-ES7TDJ5V.mjs → restore-queue-6VCVZT5C.mjs} +11 -11
  142. package/dist/{segment-codec-leaf-DO75VKEV.mjs → segment-codec-leaf-XNMCUIJF.mjs} +9 -9
  143. package/dist/{segment-merge-leaf-JGND6LG5.mjs → segment-merge-leaf-RA4S7CC6.mjs} +9 -9
  144. package/dist/server.d.ts +13 -9
  145. package/dist/server.mjs +875 -348
  146. package/dist/{storage-leaf-ZI3H2WJY.mjs → storage-leaf-36SAEVV3.mjs} +5243 -5846
  147. package/dist/{store-leaf-YIH36Y5J.mjs → store-leaf-QAGHHLEK.mjs} +10 -10
  148. package/dist/{suggestions-leaf-FP5HSTFP.mjs → suggestions-leaf-JG2IP2VQ.mjs} +12 -12
  149. package/dist/sync/filesystem.d.ts +2 -2
  150. package/dist/sync/filesystem.mjs +16 -16
  151. package/dist/sync/indexeddb.browser.mjs +1 -1
  152. package/dist/sync/indexeddb.d.ts +2 -2
  153. package/dist/sync/indexeddb.mjs +3 -3
  154. package/dist/sync.browser.mjs +35 -1108
  155. package/dist/sync.d.ts +3 -3
  156. package/dist/sync.mjs +430 -51
  157. package/dist/synonyms.browser.mjs +3 -3
  158. package/dist/synonyms.mjs +8 -8
  159. package/dist/{tasks-X2IJQc8J.d.ts → tasks-C65V7lIr.d.ts} +3 -3
  160. package/dist/thread-HAE2XXFR.mjs +31 -0
  161. package/dist/{types-D0YasjoX.d.ts → types-CpCg-Gq-.d.ts} +1 -1
  162. package/dist/vector/search-worker.mjs +5 -5
  163. package/dist/{worker-copies-WYPRCNB7.mjs → worker-copies-IXC43Z4Q.mjs} +11 -11
  164. package/dist/workers/entry.mjs +14 -14
  165. package/openapi.json +144 -0
  166. package/package.json +7 -7
  167. package/dist/chunk-URVH6BLO.mjs +0 -217
  168. package/dist/react-native/chunks/msgpack-leaf-GTKLF6GJ.mjs +0 -7
  169. package/dist/thread-NQRNW3IH.mjs +0 -31
  170. /package/dist/{chunk-CV3Q2EKO.mjs → chunk-S5QMWECL.mjs} +0 -0
package/dist/server.mjs CHANGED
@@ -1,105 +1,559 @@
1
- import { answerRequest } from './chunk-JDG6CCXM.mjs';
2
- import { resolveVectorText } from './chunk-EWGIF6FQ.mjs';
3
- import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, registerFeedRoute, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, DEFAULT_MAX_FEED_BUFFERED_BYTES, DEFAULT_MAX_FEED_CATCH_UPS, DEFAULT_MAX_FEED_CONNECTIONS, DEFAULT_FEED_WINDOW_BYTES, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, createSyncSnapshotHandler, acceptorOf, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, toHttpError, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, PROMETHEUS_TEXT_CONTENT_TYPE, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, clearGateSlot, clearBodyBudgetSlot, createCloseAtExitWords, createDepartureWord, departureConfirmed, TIMER_DELAY_LIMIT_MS, FEED_HEAD_RETRY_MS, parseJson, validateCreateIndex, rejectInvalid, isRemoveOnlyBatch, validateBatch, serializeBatchResult, parseWriteJson, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, requestSignal, withServerRerankWindow, respondAnswer, encodeQueryResult, parseTaskListQuery, parseJsonOptional, validateChangeAnalysis, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, parseRefusalOf, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, isThreadFeedNote, ENTRIES_MESSAGE_LIMIT_BYTES, SERVER_CAPABILITIES, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-MHBG33R5.mjs';
4
- export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, CHANGE_ANALYSIS_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-MHBG33R5.mjs';
5
- import { startSyncRetentionSweep, publishedSyncSnapshot, publishedArchiveLayout } from './chunk-O337MDRJ.mjs';
6
- import { SYNC_CLOSE_RESYNC, SYNC_CLOSE_CREDENTIAL_EXPIRED, SYNC_CLOSE_POLICY, MAX_DEVICE_MESSAGE_BYTES, SYNC_CLOSE_TOO_LARGE, deviceMessageOf, SYNC_CLOSE_SERVER_ERROR, positionList, positionsByPartition, SYNC_CLOSE_GOING_AWAY } from './chunk-FETZYCH7.mjs';
7
- import { syncServingSourceOf, writeBlockLookupOf, indexReadsOf, validateBackupName, readRestorePlacement } from './chunk-5BUEBIXR.mjs';
8
- import { isRecord, validateIndexName } from './chunk-CODGHJVW.mjs';
9
- import './chunk-2GONI5HL.mjs';
10
- import './chunk-URVH6BLO.mjs';
11
- import './chunk-N3M4NPML.mjs';
12
- import { estimatedArchiveBytes } from './chunk-7WXGUTWK.mjs';
1
+ import { answerRequest } from './chunk-YQ4RZIIN.mjs';
2
+ import { resolveVectorText } from './chunk-NJJN4FMM.mjs';
3
+ import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, registerFeedRoute, SYNC_FEED_ROUTE, CHANGES_FEED_ROUTE, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, createSyncSnapshotHandler, acceptorOf, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, PROMETHEUS_TEXT_CONTENT_TYPE, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, clearGateSlot, clearBodyBudgetSlot, createCloseAtExitWords, createDepartureWord, departureConfirmed, refusedRegex, toHttpError, parseJson, validateCreateIndex, rejectInvalid, isRemoveOnlyBatch, validateBatch, serializeBatchResult, parseWriteJson, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, requestSignal, withServerRerankWindow, respondAnswer, encodeQueryResult, parseTaskListQuery, parseJsonOptional, validateChangeAnalysis, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, respondPreparingError, parseRefusalOf, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, isThreadFeedNote, SERVER_CAPABILITIES, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-NM2ITITI.mjs';
4
+ export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, CHANGE_ANALYSIS_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-NM2ITITI.mjs';
5
+ import { startSyncRetentionSweep, publishedSyncSnapshot, publishedArchiveLayout } from './chunk-U5IMECEX.mjs';
6
+ import { SYNC_CLOSE_RESYNC, SYNC_CLOSE_CREDENTIAL_EXPIRED, SYNC_CLOSE_POLICY, MAX_DEVICE_MESSAGE_BYTES, SYNC_CLOSE_TOO_LARGE, deviceMessageOf, SYNC_CLOSE_SERVER_ERROR, positionList, positionsByPartition, CHANGES_CLOSE_RESYNC, CHANGES_CLOSE_POLICY, MAX_SUBSCRIBER_MESSAGE_BYTES, CHANGES_CLOSE_TOO_LARGE, closeReasonOf, CHANGES_CLOSE_CREDENTIAL_EXPIRED, CHANGES_CLOSE_SETTING, CHANGES_CLOSE_SERVER_ERROR, CHANGES_CLOSE_GOING_AWAY, SYNC_CLOSE_GOING_AWAY } from './chunk-F6F4G322.mjs';
7
+ import { syncServingSourceOf, changesServingSourceOf, writeBlockLookupOf, indexReadsOf, validateBackupName, readRestorePlacement } from './chunk-3OZ2UQJV.mjs';
8
+ import { isRecord, validateIndexName } from './chunk-X2NE4VJI.mjs';
9
+ import './chunk-YFMM7QOT.mjs';
10
+ import { engineChangeSource, createSemaphore, createByteBudget, changesNotEnabled, subscriberMessageOf, ChangeCursor, changePositionList, seqNosByPartition } from './chunk-JEMK23OY.mjs';
11
+ import './chunk-M2Z5WVWW.mjs';
12
+ import { DEFAULT_MAX_FEED_BUFFERED_BYTES, DEFAULT_MAX_FEED_CATCH_UPS, DEFAULT_MAX_FEED_CONNECTIONS, DEFAULT_FEED_WINDOW_BYTES, estimatedArchiveBytes, TIMER_DELAY_LIMIT_MS, FEED_HEAD_RETRY_MS, ENTRIES_MESSAGE_LIMIT_BYTES } from './chunk-3JRN23HA.mjs';
13
13
  import './chunk-JDDXV6UF.mjs';
14
- import { manifestFileRefs } from './chunk-2KCCQK4R.mjs';
14
+ import { manifestFileRefs } from './chunk-NFAWR2O4.mjs';
15
15
  import './chunk-MSIN4NM3.mjs';
16
- import './chunk-CV3Q2EKO.mjs';
16
+ import { encode as encode$1 } from './chunk-S5QMWECL.mjs';
17
17
  import './chunk-LNGE7SLY.mjs';
18
- import './chunk-I2WJCRIH.mjs';
19
- import './chunk-4JBTALUC.mjs';
20
- import './chunk-H4QYQ5DE.mjs';
21
- import './chunk-WLFOLAQI.mjs';
18
+ import './chunk-R2PJKSNL.mjs';
19
+ import './chunk-IDWDYS2N.mjs';
20
+ import './chunk-7NCVV3ST.mjs';
21
+ import './chunk-KBU3ZFJN.mjs';
22
22
  import './chunk-V6EUBAIT.mjs';
23
- import './chunk-EDEE7ETQ.mjs';
24
- import { querySuggestions } from './chunk-VYSWVQCE.mjs';
23
+ import './chunk-PZYBWMCL.mjs';
24
+ import { querySuggestions } from './chunk-5HPILB7R.mjs';
25
25
  import './chunk-S2POFTLN.mjs';
26
- import { rerankWindowCapOf, engineCoreOf, readMemoryFigures, listenForSyncChanges, MAX_RERANK_WINDOW, rulesOfStoredSynonyms, bindQuerySignal, clearQuerySignal, providedDocId, loadRerankRuntime } from './chunk-SL53DHVE.mjs';
27
- import './chunk-KLVMK3R5.mjs';
28
- import './chunk-F4OOBPKA.mjs';
26
+ import { rerankWindowCapOf, engineCoreOf, readMemoryFigures, listenForSyncChanges, MAX_RERANK_WINDOW, bindQuerySignal, clearQuerySignal, providedDocId, loadRerankRuntime, rulesOfStoredSynonyms } from './chunk-JXDY3ZA3.mjs';
27
+ import './chunk-2CKDCC65.mjs';
28
+ import './chunk-AOYQTX6W.mjs';
29
29
  import { decode } from './chunk-GXCDWLQI.mjs';
30
30
  import './chunk-BUMREPLU.mjs';
31
31
  import './chunk-7T4LAYT4.mjs';
32
- import './chunk-YM2HI5N5.mjs';
33
- import './chunk-X5NHWOXT.mjs';
32
+ import './chunk-IL5DRCNU.mjs';
33
+ import './chunk-7P5YWDRQ.mjs';
34
34
  import './chunk-U2BKYEC7.mjs';
35
- import { detectRuntime } from './chunk-6SZQETSC.mjs';
35
+ import { detectRuntime } from './chunk-RORNUNXW.mjs';
36
36
  import { startIdleHeapCleanup } from './chunk-UPDNNOTG.mjs';
37
37
  import './chunk-3ECIRFS5.mjs';
38
- import './chunk-D6FYUTFN.mjs';
38
+ import './chunk-ASZLNVWW.mjs';
39
39
  import './chunk-ZXX5GZEX.mjs';
40
- import './chunk-IE3Q62QD.mjs';
40
+ import './chunk-GR5GMFJJ.mjs';
41
41
  import './chunk-4PY4JYMY.mjs';
42
- import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-Q3ZZH5Z7.mjs';
43
- export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-Q3ZZH5Z7.mjs';
42
+ import { ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-ZKOQR7EA.mjs';
43
+ export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-ZKOQR7EA.mjs';
44
44
  import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
45
45
  import { randomUUID } from 'crypto';
46
46
  import { encode } from '@msgpack/msgpack';
47
47
  import { createReadStream } from 'fs';
48
48
  import { MessageChannel } from 'worker_threads';
49
49
 
50
- // src/sync/feed/limits.ts
51
- function createSemaphore(limit) {
52
- let active = 0;
53
- const waiting = [];
54
- const release = () => {
55
- active -= 1;
56
- waiting.shift()?.();
57
- };
58
- return {
59
- async run(task) {
60
- if (active >= limit) await new Promise((resolve) => waiting.push(resolve));
61
- active += 1;
62
- try {
63
- return await task();
64
- } finally {
65
- release();
50
+ // src/server/changes/setting-handlers.ts
51
+ function createChangesSettingHandlers(engine) {
52
+ async function getChanges(ctx) {
53
+ try {
54
+ respondJson(ctx, await engine.getChanges(ctx.params[0]));
55
+ } catch (err) {
56
+ respondError(ctx, err);
57
+ }
58
+ }
59
+ async function updateChanges(ctx) {
60
+ const body = parseJson(ctx);
61
+ if (body === void 0) return;
62
+ try {
63
+ await engine.updateChanges(ctx.params[0], body);
64
+ respondJson(ctx, { ok: true });
65
+ } catch (err) {
66
+ respondError(ctx, err);
67
+ }
68
+ }
69
+ return { getChanges, updateChanges };
70
+ }
71
+
72
+ // src/changes/encode.ts
73
+ var FIXMAP_TAG = 128;
74
+ var FIXARRAY_LIMIT = 16;
75
+ var ARRAY16_LIMIT = 65536;
76
+ var FIXARRAY_TAG = 144;
77
+ var ARRAY16_TAG = 220;
78
+ var ARRAY32_TAG = 221;
79
+ var CHANGES_MESSAGE_KEY_COUNT = 4;
80
+ var CHANGE_FIXED_KEY_COUNT = 3;
81
+ var keys = null;
82
+ function keyBytes(name) {
83
+ keys ??= {};
84
+ let bytes = keys[name];
85
+ if (bytes === void 0) {
86
+ bytes = encode$1(name);
87
+ keys[name] = bytes;
88
+ }
89
+ return bytes;
90
+ }
91
+ function arrayHeader(count) {
92
+ if (count < FIXARRAY_LIMIT) return Uint8Array.of(FIXARRAY_TAG | count);
93
+ if (count < ARRAY16_LIMIT) return Uint8Array.of(ARRAY16_TAG, count >>> 8, count & 255);
94
+ return Uint8Array.of(ARRAY32_TAG, count >>> 24, count >>> 16 & 255, count >>> 8 & 255, count & 255);
95
+ }
96
+ function concat(parts) {
97
+ let length = 0;
98
+ for (const part of parts) length += part.byteLength;
99
+ const out = new Uint8Array(length);
100
+ let at = 0;
101
+ for (const part of parts) {
102
+ out.set(part, at);
103
+ at += part.byteLength;
104
+ }
105
+ return out;
106
+ }
107
+ function encodedChangeParts(change) {
108
+ const keyCount = CHANGE_FIXED_KEY_COUNT + (change.document === null ? 0 : 1) + (change.previousDocument === null ? 0 : 1);
109
+ const parts = [
110
+ Uint8Array.of(FIXMAP_TAG | keyCount),
111
+ keyBytes("seqNo"),
112
+ encode$1(change.seqNo),
113
+ keyBytes("type"),
114
+ keyBytes(change.type),
115
+ keyBytes("documentId"),
116
+ encode$1(change.documentId)
117
+ ];
118
+ if (change.document !== null) parts.push(keyBytes("document"), change.document);
119
+ if (change.previousDocument !== null) parts.push(keyBytes("previousDocument"), change.previousDocument);
120
+ return parts;
121
+ }
122
+ function encodeChangesMessage(partitionId, changes, scannedSeqNo) {
123
+ const parts = [
124
+ Uint8Array.of(FIXMAP_TAG | CHANGES_MESSAGE_KEY_COUNT),
125
+ keyBytes("type"),
126
+ keyBytes("changes"),
127
+ keyBytes("partitionId"),
128
+ encode$1(partitionId),
129
+ keyBytes("scannedSeqNo"),
130
+ encode$1(scannedSeqNo),
131
+ keyBytes("changes"),
132
+ arrayHeader(changes.length)
133
+ ];
134
+ for (const change of changes) parts.push(...encodedChangeParts(change));
135
+ return concat(parts);
136
+ }
137
+ function encodeFeedMessage(message) {
138
+ return encode$1(message, { ignoreUndefined: true });
139
+ }
140
+
141
+ // src/changes/session.ts
142
+ var POLICY_CODES = /* @__PURE__ */ new Set([
143
+ ErrorCodes.CHANGES_INVALID_POSITIONS,
144
+ ErrorCodes.SEARCH_INVALID_FILTER,
145
+ ErrorCodes.SEARCH_INVALID_FIELD
146
+ ]);
147
+ var ChangesSession = class {
148
+ constructor(deps) {
149
+ this.deps = deps;
150
+ this.stopBudgetWatch = deps.buffered.onFreed(() => {
151
+ if (this.state === "subscribed" && this.charged === 0) this.advance();
152
+ });
153
+ this.armExpiry();
154
+ }
155
+ state = "awaiting";
156
+ cursor = null;
157
+ sentSeqNos = [];
158
+ silentBytes = [];
159
+ unacked = [];
160
+ unackedBytes = 0;
161
+ charged = 0;
162
+ pumping = false;
163
+ pumpAgain = false;
164
+ retry = null;
165
+ expiry = null;
166
+ incoming = Promise.resolve();
167
+ stopBudgetWatch;
168
+ get indexName() {
169
+ return this.deps.indexName;
170
+ }
171
+ receive(bytes, binary = true) {
172
+ if (this.state === "closed") return;
173
+ if (!binary) {
174
+ this.close(CHANGES_CLOSE_POLICY, "feed messages travel in binary frames");
175
+ return;
176
+ }
177
+ if (bytes.byteLength > MAX_SUBSCRIBER_MESSAGE_BYTES) {
178
+ this.close(CHANGES_CLOSE_TOO_LARGE, "message too large");
179
+ return;
180
+ }
181
+ let message;
182
+ try {
183
+ message = subscriberMessageOf(decode(bytes));
184
+ } catch {
185
+ message = null;
186
+ }
187
+ if (message === null) {
188
+ this.close(CHANGES_CLOSE_POLICY, "invalid message");
189
+ return;
190
+ }
191
+ const received = message;
192
+ this.incoming = this.incoming.then(() => this.handle(received)).catch((error) => this.fail(error));
193
+ }
194
+ advance() {
195
+ if (this.state !== "subscribed") return;
196
+ void this.pump();
197
+ }
198
+ drained() {
199
+ this.settleBuffered();
200
+ this.advance();
201
+ }
202
+ resync(reason) {
203
+ if (this.state === "closed") return;
204
+ this.send(encodeFeedMessage({ type: "resync", reason }));
205
+ this.close(CHANGES_CLOSE_RESYNC, `resync ${reason}`);
206
+ }
207
+ previousDocumentOff() {
208
+ if (this.cursor?.filtered === true) this.refuseFilterSetting();
209
+ }
210
+ close(code, reason) {
211
+ if (this.state === "closed") return;
212
+ this.state = "closed";
213
+ this.cleanUp();
214
+ this.deps.socket.close(code, closeReasonOf(reason));
215
+ this.deps.onClosed(this);
216
+ }
217
+ closedByPeer() {
218
+ if (this.state === "closed") return;
219
+ this.state = "closed";
220
+ this.cleanUp();
221
+ this.deps.onClosed(this);
222
+ }
223
+ armExpiry() {
224
+ const { expiresAt } = this.deps;
225
+ if (expiresAt === null || this.state === "closed") return;
226
+ const remaining = expiresAt - Date.now();
227
+ if (remaining <= 0) {
228
+ queueMicrotask(() => this.close(CHANGES_CLOSE_CREDENTIAL_EXPIRED, "credential expired"));
229
+ return;
230
+ }
231
+ this.expiry = setTimeout(
232
+ () => {
233
+ this.expiry = null;
234
+ this.armExpiry();
235
+ },
236
+ Math.min(remaining, TIMER_DELAY_LIMIT_MS)
237
+ );
238
+ }
239
+ refuseFilterSetting() {
240
+ this.close(CHANGES_CLOSE_SETTING, ErrorCodes.CHANGES_PREVIOUS_DOCUMENT_REQUIRED);
241
+ }
242
+ cleanUp() {
243
+ if (this.retry !== null) clearTimeout(this.retry);
244
+ this.retry = null;
245
+ if (this.expiry !== null) clearTimeout(this.expiry);
246
+ this.expiry = null;
247
+ this.stopBudgetWatch();
248
+ this.deps.buffered.give(this.charged);
249
+ this.charged = 0;
250
+ }
251
+ fail(error) {
252
+ this.deps.onError(error);
253
+ this.close(CHANGES_CLOSE_SERVER_ERROR, "feed read failed");
254
+ }
255
+ send(bytes) {
256
+ if (this.state === "closed") return;
257
+ this.deps.socket.send(bytes);
258
+ this.settleBuffered();
259
+ }
260
+ settleBuffered() {
261
+ const now = this.state === "closed" ? 0 : this.deps.socket.bufferedAmount();
262
+ if (now < this.charged) this.deps.buffered.give(this.charged - now);
263
+ this.charged = Math.min(this.charged, now);
264
+ }
265
+ async handle(message) {
266
+ if (this.state === "closed") return;
267
+ if (message.type === "subscribe") {
268
+ if (this.state !== "awaiting" || this.cursor !== null) {
269
+ return this.close(CHANGES_CLOSE_POLICY, "subscribe after the first message");
270
+ }
271
+ return this.subscribe(message);
272
+ }
273
+ if (this.state !== "subscribed") return this.close(CHANGES_CLOSE_POLICY, "subscribe must come first");
274
+ this.acknowledge(message.positions);
275
+ }
276
+ closeOver(error) {
277
+ if (!(error instanceof NarsilError)) this.fail(error);
278
+ else if (error.code === ErrorCodes.CHANGES_PREVIOUS_DOCUMENT_REQUIRED) this.refuseFilterSetting();
279
+ else if (error.code === ErrorCodes.CHANGES_NOT_ENABLED) this.close(CHANGES_CLOSE_RESYNC, "change capture is off");
280
+ else if (POLICY_CODES.has(error.code)) this.close(CHANGES_CLOSE_POLICY, error.message);
281
+ else this.fail(error);
282
+ }
283
+ async subscribe(message) {
284
+ const { source, indexName, reads } = this.deps;
285
+ if (message.filter !== void 0) {
286
+ const refused = this.deps.refuseFilter(message.filter);
287
+ if (refused !== null) return this.close(CHANGES_CLOSE_POLICY, refused.message);
288
+ }
289
+ let opened;
290
+ let vectorFields = [];
291
+ try {
292
+ opened = await ChangeCursor.open({ source, indexName, reads }, message);
293
+ if (typeof opened !== "string") vectorFields = [...await source.vectorFieldPaths(indexName)].sort();
294
+ } catch (error) {
295
+ return this.closeOver(error);
296
+ }
297
+ if (this.state === "closed") return;
298
+ if (typeof opened === "string") return this.resync(opened);
299
+ const cursor = opened;
300
+ this.cursor = cursor;
301
+ this.sentSeqNos = [...cursor.subscribedSeqNos];
302
+ this.silentBytes = new Array(cursor.state.partitionCount).fill(0);
303
+ for (let i = 0; i < cursor.state.partitionCount; i++) this.unacked.push({ items: [], head: 0 });
304
+ this.state = "subscribed";
305
+ this.send(
306
+ encodeFeedMessage({
307
+ type: "subscribed",
308
+ epoch: cursor.state.epoch,
309
+ partitionCount: cursor.state.partitionCount,
310
+ positions: changePositionList(cursor.subscribedSeqNos),
311
+ heads: changePositionList(cursor.heads),
312
+ bytesBehind: cursor.bytesBehind,
313
+ windowBytes: this.deps.windowBytes,
314
+ previousDocument: cursor.state.previousDocument,
315
+ vectorFields
316
+ })
317
+ );
318
+ void this.pump();
319
+ }
320
+ acknowledge(positions) {
321
+ const seqNos = seqNosByPartition(positions, this.sentSeqNos.length);
322
+ if (seqNos === null) {
323
+ this.close(CHANGES_CLOSE_POLICY, "positions must name each partition once");
324
+ return;
325
+ }
326
+ for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
327
+ if (seqNos[partitionId] > this.sentSeqNos[partitionId]) {
328
+ this.close(CHANGES_CLOSE_POLICY, "acknowledgement above a sent position");
329
+ return;
330
+ }
331
+ }
332
+ for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
333
+ const queue = this.unacked[partitionId];
334
+ while (queue.head < queue.items.length && queue.items[queue.head].seqNo <= seqNos[partitionId]) {
335
+ this.unackedBytes -= queue.items[queue.head].bytes;
336
+ queue.head += 1;
337
+ }
338
+ if (queue.head > 0 && queue.head * 2 >= queue.items.length) {
339
+ queue.items = queue.items.slice(queue.head);
340
+ queue.head = 0;
341
+ }
342
+ }
343
+ void this.pump();
344
+ }
345
+ windowAllows(nextBytes) {
346
+ return this.unackedBytes === 0 || this.unackedBytes + nextBytes <= this.deps.windowBytes;
347
+ }
348
+ settingAllows(cursor) {
349
+ const current = this.deps.source.changesState(this.deps.indexName);
350
+ if (current === null) {
351
+ this.close(CHANGES_CLOSE_RESYNC, "change capture is off");
352
+ return false;
353
+ }
354
+ if (current.epoch !== cursor.state.epoch) {
355
+ this.resync("epoch");
356
+ return false;
357
+ }
358
+ if (cursor.filtered && !current.previousDocument) {
359
+ this.refuseFilterSetting();
360
+ return false;
361
+ }
362
+ return true;
363
+ }
364
+ async pump() {
365
+ const { cursor } = this;
366
+ if (cursor === null) return;
367
+ if (this.pumping) {
368
+ this.pumpAgain = true;
369
+ return;
370
+ }
371
+ this.pumping = true;
372
+ try {
373
+ do {
374
+ this.pumpAgain = false;
375
+ if (!this.settingAllows(cursor)) return;
376
+ await cursor.refreshHeads();
377
+ await this.sendBehind(cursor);
378
+ } while (this.pumpAgain && this.state === "subscribed");
379
+ this.scheduleRetryWhileUndurable(cursor);
380
+ } catch (error) {
381
+ this.fail(error);
382
+ } finally {
383
+ this.pumping = false;
384
+ }
385
+ }
386
+ async sendBehind(cursor) {
387
+ const { windowBytes } = this.deps;
388
+ let progressed = true;
389
+ while (progressed && this.state === "subscribed") {
390
+ progressed = false;
391
+ for (let partitionId = 0; partitionId < this.sentSeqNos.length; partitionId++) {
392
+ if (this.state !== "subscribed" || !cursor.behind(partitionId)) continue;
393
+ if (!this.windowAllows(1) || this.charged > 0) return;
394
+ const room = this.unackedBytes === 0 ? windowBytes : windowBytes - this.unackedBytes;
395
+ const batch = await cursor.readBatch(partitionId, Math.max(1, Math.min(room, ENTRIES_MESSAGE_LIMIT_BYTES)));
396
+ if (this.state !== "subscribed") return;
397
+ if (batch === "history") return this.resync("history");
398
+ if (batch === null) continue;
399
+ if (!this.settingAllows(cursor)) return;
400
+ progressed = true;
401
+ if (batch.changes.length === 0 && this.silentBytes[partitionId] + batch.scannedBytes < windowBytes) {
402
+ this.silentBytes[partitionId] += batch.scannedBytes;
403
+ cursor.commit(batch);
404
+ continue;
405
+ }
406
+ const message = encodeChangesMessage(partitionId, batch.changes, batch.scannedSeqNo);
407
+ if (!this.windowAllows(message.byteLength)) return;
408
+ if (!this.deps.buffered.take(message.byteLength)) return;
409
+ this.charged += message.byteLength;
410
+ this.unacked[partitionId].items.push({ seqNo: batch.scannedSeqNo, bytes: message.byteLength });
411
+ this.unackedBytes += message.byteLength;
412
+ this.silentBytes[partitionId] = 0;
413
+ this.sentSeqNos[partitionId] = batch.scannedSeqNo;
414
+ cursor.commit(batch);
415
+ this.send(message);
66
416
  }
67
417
  }
68
- };
69
- }
70
- function createByteBudget(limit) {
71
- let used = 0;
72
- const listeners = /* @__PURE__ */ new Set();
418
+ }
419
+ scheduleRetryWhileUndurable(cursor) {
420
+ if (this.state !== "subscribed" || this.retry !== null || !cursor.awaitsDurability()) return;
421
+ this.retry = setTimeout(() => {
422
+ this.retry = null;
423
+ this.advance();
424
+ }, FEED_HEAD_RETRY_MS);
425
+ }
426
+ };
427
+
428
+ // src/changes/hub.ts
429
+ function createChangesHub(source, options = {}) {
430
+ const windowBytes = options.windowBytes ?? DEFAULT_FEED_WINDOW_BYTES;
431
+ const maxConnections = options.maxConnections ?? DEFAULT_MAX_FEED_CONNECTIONS;
432
+ const reads = createSemaphore(options.maxCatchUps ?? DEFAULT_MAX_FEED_CATCH_UPS);
433
+ const buffered = createByteBudget(options.maxBufferedBytes ?? DEFAULT_MAX_FEED_BUFFERED_BYTES);
434
+ const refuseFilter = options.refuseFilter ?? (() => null);
435
+ const sessionsByIndex = /* @__PURE__ */ new Map();
436
+ const unwatchByIndex = /* @__PURE__ */ new Map();
437
+ let connections = 0;
438
+ let stopListening = null;
439
+ let closed = false;
440
+ function each(indexName, action) {
441
+ const sessions = sessionsByIndex.get(indexName);
442
+ if (sessions === void 0) return;
443
+ for (const session of [...sessions]) action(session);
444
+ }
445
+ function onChange(change) {
446
+ switch (change.kind) {
447
+ case "advanced":
448
+ each(change.indexName, (session) => session.advance());
449
+ return;
450
+ case "epoch":
451
+ each(change.indexName, (session) => session.resync("epoch"));
452
+ return;
453
+ case "previous-document-off":
454
+ each(change.indexName, (session) => session.previousDocumentOff());
455
+ return;
456
+ case "changes-off":
457
+ case "dropped":
458
+ each(change.indexName, (session) => session.close(CHANGES_CLOSE_RESYNC, "change capture is off"));
459
+ return;
460
+ case "off":
461
+ case "settings":
462
+ return;
463
+ }
464
+ }
465
+ function release() {
466
+ connections -= 1;
467
+ if (connections === 0 && stopListening !== null) {
468
+ stopListening();
469
+ stopListening = null;
470
+ }
471
+ }
472
+ function forget(session) {
473
+ const sessions = sessionsByIndex.get(session.indexName);
474
+ if (sessions === void 0 || !sessions.delete(session)) return;
475
+ if (sessions.size === 0) {
476
+ sessionsByIndex.delete(session.indexName);
477
+ unwatchByIndex.get(session.indexName)?.();
478
+ unwatchByIndex.delete(session.indexName);
479
+ }
480
+ release();
481
+ }
482
+ function open(indexName, expiresAt, socket) {
483
+ if (stopListening === null) stopListening = source.listen(onChange);
484
+ const session = new ChangesSession({
485
+ source,
486
+ indexName,
487
+ socket,
488
+ windowBytes,
489
+ expiresAt,
490
+ reads,
491
+ buffered,
492
+ refuseFilter,
493
+ onClosed: forget,
494
+ onError: (error) => options.onError?.(error)
495
+ });
496
+ let sessions = sessionsByIndex.get(indexName);
497
+ if (sessions === void 0) {
498
+ sessions = /* @__PURE__ */ new Set();
499
+ sessionsByIndex.set(indexName, sessions);
500
+ const unwatch = source.watchIndex?.(indexName);
501
+ if (unwatch !== void 0) unwatchByIndex.set(indexName, unwatch);
502
+ }
503
+ sessions.add(session);
504
+ return session;
505
+ }
73
506
  return {
74
- take(bytes) {
75
- if (used > 0 && used + bytes > limit) return false;
76
- used += bytes;
77
- return true;
78
- },
79
- give(bytes) {
80
- used = Math.max(0, used - bytes);
81
- for (const listener of [...listeners]) listener();
82
- },
83
- onFreed(listener) {
84
- listeners.add(listener);
85
- return () => {
86
- listeners.delete(listener);
507
+ admit(indexName, expiresAt) {
508
+ if (closed) {
509
+ throw new NarsilError(ServerErrorCodes.TOO_MANY_REQUESTS, "The server is closing its feed connections");
510
+ }
511
+ if (source.changesState(indexName) === null) throw changesNotEnabled(indexName);
512
+ if (connections >= maxConnections) {
513
+ throw new NarsilError(
514
+ ServerErrorCodes.TOO_MANY_REQUESTS,
515
+ `The server holds ${connections} change feed connections, the most that it accepts`,
516
+ { maximum: maxConnections }
517
+ );
518
+ }
519
+ connections += 1;
520
+ let settled = false;
521
+ return {
522
+ open(socket) {
523
+ if (settled) throw new Error("A change feed admission opens one connection");
524
+ settled = true;
525
+ return open(indexName, expiresAt, socket);
526
+ },
527
+ cancel() {
528
+ if (settled) return;
529
+ settled = true;
530
+ release();
531
+ }
87
532
  };
533
+ },
534
+ connectionCount: () => connections,
535
+ close() {
536
+ closed = true;
537
+ for (const indexName of [...sessionsByIndex.keys()]) {
538
+ each(indexName, (session) => session.close(CHANGES_CLOSE_GOING_AWAY, "server closing"));
539
+ }
540
+ stopListening?.();
541
+ stopListening = null;
88
542
  }
89
543
  };
90
544
  }
91
- var FIXARRAY_LIMIT = 16;
92
- var ARRAY16_LIMIT = 65536;
93
- var FIXARRAY_TAG = 144;
94
- var ARRAY16_TAG = 220;
95
- var ARRAY32_TAG = 221;
545
+ var FIXARRAY_LIMIT2 = 16;
546
+ var ARRAY16_LIMIT2 = 65536;
547
+ var FIXARRAY_TAG2 = 144;
548
+ var ARRAY16_TAG2 = 220;
549
+ var ARRAY32_TAG2 = 221;
96
550
  var FIXMAP_OF_THREE = 131;
97
551
  var entriesKeyPrefix = (() => {
98
552
  const parts = [Uint8Array.of(FIXMAP_OF_THREE), encode("type"), encode("entries"), encode("partitionId")];
99
- return concat(parts);
553
+ return concat2(parts);
100
554
  })();
101
555
  var entriesKey = encode("entries");
102
- function concat(parts) {
556
+ function concat2(parts) {
103
557
  let length = 0;
104
558
  for (const part of parts) length += part.byteLength;
105
559
  const out = new Uint8Array(length);
@@ -110,13 +564,13 @@ function concat(parts) {
110
564
  }
111
565
  return out;
112
566
  }
113
- function arrayHeader(count) {
114
- if (count < FIXARRAY_LIMIT) return Uint8Array.of(FIXARRAY_TAG | count);
115
- if (count < ARRAY16_LIMIT) return Uint8Array.of(ARRAY16_TAG, count >>> 8, count & 255);
116
- return Uint8Array.of(ARRAY32_TAG, count >>> 24, count >>> 16 & 255, count >>> 8 & 255, count & 255);
567
+ function arrayHeader2(count) {
568
+ if (count < FIXARRAY_LIMIT2) return Uint8Array.of(FIXARRAY_TAG2 | count);
569
+ if (count < ARRAY16_LIMIT2) return Uint8Array.of(ARRAY16_TAG2, count >>> 8, count & 255);
570
+ return Uint8Array.of(ARRAY32_TAG2, count >>> 24, count >>> 16 & 255, count >>> 8 & 255, count & 255);
117
571
  }
118
572
  function encodeEntriesMessage(partitionId, payloads) {
119
- return concat([entriesKeyPrefix, encode(partitionId), entriesKey, arrayHeader(payloads.length), ...payloads]);
573
+ return concat2([entriesKeyPrefix, encode(partitionId), entriesKey, arrayHeader2(payloads.length), ...payloads]);
120
574
  }
121
575
  function encodeServerMessage(message) {
122
576
  return encode(message, { ignoreUndefined: true });
@@ -490,46 +944,315 @@ function createFeedHub(source, options = {}) {
490
944
  return session;
491
945
  }
492
946
  return {
493
- admit(indexName, expiresAt) {
494
- if (closed) {
495
- throw new NarsilError(ServerErrorCodes.TOO_MANY_REQUESTS, "The server is closing its feed connections");
496
- }
497
- if (source.indexState(indexName) === null) {
498
- throw new NarsilError(ErrorCodes.SYNC_NOT_ENABLED, `Index "${indexName}" has device sync off`, { indexName });
499
- }
500
- if (connections >= maxConnections) {
501
- throw new NarsilError(
502
- ServerErrorCodes.TOO_MANY_REQUESTS,
503
- `The server holds ${connections} feed connections, the most that it accepts`,
504
- { maximum: maxConnections }
505
- );
506
- }
507
- connections += 1;
508
- let settled = false;
509
- return {
510
- open(socket) {
511
- if (settled) throw new Error("A feed admission opens one connection");
512
- settled = true;
513
- return open(indexName, expiresAt, socket);
514
- },
515
- cancel() {
516
- if (settled) return;
517
- settled = true;
518
- release();
519
- }
520
- };
521
- },
522
- connectionCount: () => connections,
947
+ admit(indexName, expiresAt) {
948
+ if (closed) {
949
+ throw new NarsilError(ServerErrorCodes.TOO_MANY_REQUESTS, "The server is closing its feed connections");
950
+ }
951
+ if (source.indexState(indexName) === null) {
952
+ throw new NarsilError(ErrorCodes.SYNC_NOT_ENABLED, `Index "${indexName}" has device sync off`, { indexName });
953
+ }
954
+ if (connections >= maxConnections) {
955
+ throw new NarsilError(
956
+ ServerErrorCodes.TOO_MANY_REQUESTS,
957
+ `The server holds ${connections} feed connections, the most that it accepts`,
958
+ { maximum: maxConnections }
959
+ );
960
+ }
961
+ connections += 1;
962
+ let settled = false;
963
+ return {
964
+ open(socket) {
965
+ if (settled) throw new Error("A feed admission opens one connection");
966
+ settled = true;
967
+ return open(indexName, expiresAt, socket);
968
+ },
969
+ cancel() {
970
+ if (settled) return;
971
+ settled = true;
972
+ release();
973
+ }
974
+ };
975
+ },
976
+ connectionCount: () => connections,
977
+ close() {
978
+ closed = true;
979
+ for (const indexName of [...sessionsByIndex.keys()]) {
980
+ each(indexName, (session) => session.close(SYNC_CLOSE_GOING_AWAY, "server closing"));
981
+ }
982
+ stopListening?.();
983
+ stopListening = null;
984
+ }
985
+ };
986
+ }
987
+
988
+ // src/sync/feed/engine-source.ts
989
+ function syncSettingsOf(entry) {
990
+ const { config } = entry;
991
+ const synonyms = entry.synonyms === null ? [] : rulesOfStoredSynonyms(entry.synonyms);
992
+ return {
993
+ language: entry.language.name,
994
+ ...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
995
+ ...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
996
+ ...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
997
+ ...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
998
+ ...config.bm25 !== void 0 ? { bm25: config.bm25 } : {},
999
+ ...synonyms.length > 0 ? { synonyms } : {}
1000
+ };
1001
+ }
1002
+ function storageOf(core) {
1003
+ return core.durability?.manager.syncStorage?.() ?? null;
1004
+ }
1005
+ function engineFeedSource(engine, core) {
1006
+ const requireStorage = () => {
1007
+ const storage = storageOf(core);
1008
+ if (storage === null) {
1009
+ throw new NarsilError(ErrorCodes.SYNC_DURABILITY_REQUIRED, "Device sync needs write-ahead log durability");
1010
+ }
1011
+ return storage;
1012
+ };
1013
+ return {
1014
+ indexState(indexName) {
1015
+ const entry = core.indexRegistry.get(indexName);
1016
+ if (entry === void 0 || entry.sync === null || storageOf(core) === null) return null;
1017
+ return { epoch: entry.sync.epoch, partitionCount: entry.partitionCount, settings: syncSettingsOf(entry) };
1018
+ },
1019
+ async bounds(indexName, partitionId) {
1020
+ const log = await requireStorage().openPartitionLog(indexName, partitionId);
1021
+ const applied = core.durability?.manager.highestPersistedSeqNo(indexName, partitionId) ?? 0;
1022
+ return { oldestSeqNo: log.oldestSeqNo, head: log.head, highestSeqNo: Math.max(log.head, applied) };
1023
+ },
1024
+ async read(indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) {
1025
+ const read = await requireStorage().readPartitionLog(indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes);
1026
+ return { frames: read.frames, gone: read.gone };
1027
+ },
1028
+ async bytesBehind(indexName, seqNos) {
1029
+ const storage = requireStorage();
1030
+ let bytes = 0;
1031
+ for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
1032
+ const log = await storage.openPartitionLog(indexName, partitionId);
1033
+ bytes += await storage.logBytesAfter(log, seqNos[partitionId]);
1034
+ }
1035
+ return bytes;
1036
+ },
1037
+ async snapshotBytes(indexName) {
1038
+ const storage = requireStorage();
1039
+ const published = await storage.published(indexName);
1040
+ if (published !== null) return publishedArchiveLayout(indexName, published, storage).byteLength;
1041
+ const manifest = await storage.currentManifest(indexName);
1042
+ if (manifest === null) return 0;
1043
+ return estimatedArchiveBytes(manifestFileRefs(manifest));
1044
+ },
1045
+ pendingBeyondHead(indexName, partitionId, head) {
1046
+ return (core.durability?.manager.highestPersistedSeqNo(indexName, partitionId) ?? 0) > head;
1047
+ },
1048
+ listen(onChange) {
1049
+ const onWrite = (payload) => onChange({ kind: "advanced", indexName: payload.indexName });
1050
+ engine.on("write", onWrite);
1051
+ const stopSyncChanges = listenForSyncChanges(core.indexRegistry, onChange);
1052
+ return () => {
1053
+ engine.off("write", onWrite);
1054
+ stopSyncChanges();
1055
+ };
1056
+ }
1057
+ };
1058
+ }
1059
+
1060
+ // src/server/sync/engine-serving.ts
1061
+ function engineSyncServing(engine, core) {
1062
+ startSyncRetentionSweep(core);
1063
+ return {
1064
+ async snapshot(indexName) {
1065
+ const published = await publishedSyncSnapshot(core, indexName);
1066
+ try {
1067
+ return {
1068
+ entityTag: published.snapshot.entityTag,
1069
+ layout: publishedArchiveLayout(indexName, published.snapshot, published.storage),
1070
+ release: published.unpin
1071
+ };
1072
+ } catch (error) {
1073
+ published.unpin();
1074
+ throw error;
1075
+ }
1076
+ },
1077
+ feed: engineFeedSource(engine, core),
1078
+ admitFeed: async () => void 0
1079
+ };
1080
+ }
1081
+
1082
+ // src/server/sync/feed-host.ts
1083
+ var RelayedFeedSocket = class {
1084
+ constructor(port, feedId) {
1085
+ this.port = port;
1086
+ this.feedId = feedId;
1087
+ }
1088
+ inTransit = 0;
1089
+ threadBuffered = 0;
1090
+ closed = false;
1091
+ send(bytes) {
1092
+ if (this.closed) return;
1093
+ const copy = bytes.slice();
1094
+ this.inTransit += copy.byteLength;
1095
+ this.post({ type: "feed-send", feedId: this.feedId, bytes: copy }, [copy.buffer]);
1096
+ }
1097
+ bufferedAmount() {
1098
+ return this.inTransit + this.threadBuffered;
1099
+ }
1100
+ close(code, reason) {
1101
+ if (this.closed) return;
1102
+ this.closed = true;
1103
+ this.post({ type: "feed-close", feedId: this.feedId, code, reason }, []);
1104
+ }
1105
+ sent(bytes, buffered) {
1106
+ this.inTransit = Math.max(0, this.inTransit - bytes);
1107
+ this.threadBuffered = buffered;
1108
+ }
1109
+ markClosed() {
1110
+ this.closed = true;
1111
+ }
1112
+ post(note, transfer) {
1113
+ try {
1114
+ this.port.postMessage(note, transfer);
1115
+ } catch {
1116
+ this.closed = true;
1117
+ }
1118
+ }
1119
+ };
1120
+ function routeMissing() {
1121
+ return new NarsilError(ServerErrorCodes.NOT_FOUND, "Route not found");
1122
+ }
1123
+ function createFeedHost(feeds, authorize) {
1124
+ const authorizeContext = (context) => authorize === void 0 ? Promise.resolve({ allowed: true, expiresAt: null }) : authorize(context);
1125
+ function admit(feed, indexName, expiresAt) {
1126
+ const hosted = feeds[feed];
1127
+ if (hosted === void 0) return Promise.reject(routeMissing());
1128
+ return hosted.admit(indexName, expiresAt);
1129
+ }
1130
+ const gateways = {};
1131
+ for (const feed of Object.keys(feeds)) {
1132
+ gateways[feed] = {
1133
+ authorize: authorizeContext,
1134
+ async admit(indexName, expiresAt) {
1135
+ const admission = await admit(feed, indexName, expiresAt);
1136
+ return { open: (socket) => admission.open(socket), cancel: () => admission.cancel() };
1137
+ }
1138
+ };
1139
+ }
1140
+ function forThread(port) {
1141
+ const relayed = /* @__PURE__ */ new Map();
1142
+ let nextFeedId = 0;
1143
+ let closed = false;
1144
+ function closeFeed(feedId) {
1145
+ const feed = relayed.get(feedId);
1146
+ if (feed === void 0) return;
1147
+ relayed.delete(feedId);
1148
+ feed.socket.markClosed();
1149
+ if (feed.endpoint === null) feed.admission.cancel();
1150
+ else feed.endpoint.closedByPeer();
1151
+ }
1152
+ return {
1153
+ async admit(request) {
1154
+ try {
1155
+ const admission = await admit(request.feed, request.indexName, request.expiresAt);
1156
+ if (closed) {
1157
+ admission.cancel();
1158
+ return { type: "feed-admitted", requestId: request.requestId, feedId: null, error: null };
1159
+ }
1160
+ nextFeedId += 1;
1161
+ const feedId = nextFeedId;
1162
+ relayed.set(feedId, { admission, socket: new RelayedFeedSocket(port, feedId), endpoint: null });
1163
+ return { type: "feed-admitted", requestId: request.requestId, feedId, error: null };
1164
+ } catch (err) {
1165
+ return { type: "feed-admitted", requestId: request.requestId, feedId: null, error: toHttpError(err).body };
1166
+ }
1167
+ },
1168
+ handle(note) {
1169
+ const feed = relayed.get(note.feedId);
1170
+ if (feed === void 0) return;
1171
+ switch (note.type) {
1172
+ case "feed-open":
1173
+ if (feed.endpoint === null) feed.endpoint = feed.admission.open(feed.socket);
1174
+ return;
1175
+ case "feed-message":
1176
+ feed.endpoint?.receive(note.bytes, note.binary);
1177
+ return;
1178
+ case "feed-sent":
1179
+ feed.socket.sent(note.bytes, note.buffered);
1180
+ feed.endpoint?.drained();
1181
+ return;
1182
+ case "feed-cancel":
1183
+ case "feed-closed":
1184
+ closeFeed(note.feedId);
1185
+ return;
1186
+ }
1187
+ },
1188
+ closeAll() {
1189
+ closed = true;
1190
+ for (const feedId of [...relayed.keys()]) closeFeed(feedId);
1191
+ }
1192
+ };
1193
+ }
1194
+ return {
1195
+ gateways,
1196
+ forThread,
523
1197
  close() {
524
- closed = true;
525
- for (const indexName of [...sessionsByIndex.keys()]) {
526
- each(indexName, (session) => session.close(SYNC_CLOSE_GOING_AWAY, "server closing"));
527
- }
528
- stopListening?.();
529
- stopListening = null;
1198
+ for (const hosted of Object.values(feeds)) hosted?.close();
530
1199
  }
531
1200
  };
532
1201
  }
1202
+
1203
+ // src/server/feeds.ts
1204
+ function syncServingOf(engine, core) {
1205
+ const ofNode = syncServingSourceOf(engine);
1206
+ if (ofNode !== void 0) return ofNode();
1207
+ return core === void 0 ? null : engineSyncServing(engine, core);
1208
+ }
1209
+ function changesServingOf(engine, core) {
1210
+ const ofNode = changesServingSourceOf(engine);
1211
+ if (ofNode !== void 0) return ofNode();
1212
+ if (core === void 0) return null;
1213
+ return { source: engineChangeSource(core, listenForSyncChanges), admitFeed: async () => void 0 };
1214
+ }
1215
+ function hostedSyncFeed(serving, options) {
1216
+ const hub = createFeedHub(serving.feed, {
1217
+ ...options,
1218
+ onError: (error) => console.warn("A device sync feed failed to read its index:", error)
1219
+ });
1220
+ return {
1221
+ async admit(indexName, expiresAt) {
1222
+ await serving.admitFeed(indexName);
1223
+ return hub.admit(indexName, expiresAt);
1224
+ },
1225
+ close: () => hub.close()
1226
+ };
1227
+ }
1228
+ function hostedChangesFeed(serving, options, allowRegex) {
1229
+ const hub = createChangesHub(serving.source, {
1230
+ ...options,
1231
+ refuseFilter: (filter) => refusedRegex(filter, allowRegex),
1232
+ onError: (error) => console.warn("A change feed failed to read its index:", error)
1233
+ });
1234
+ return {
1235
+ async admit(indexName, expiresAt) {
1236
+ await serving.admitFeed(indexName);
1237
+ return hub.admit(indexName, expiresAt);
1238
+ },
1239
+ close: () => hub.close()
1240
+ };
1241
+ }
1242
+ function startFeeds(engine, core, settings) {
1243
+ const hosted = {};
1244
+ let syncServing = null;
1245
+ if (settings.sync !== null) {
1246
+ syncServing = syncServingOf(engine, core);
1247
+ if (syncServing !== null) hosted.sync = hostedSyncFeed(syncServing, settings.sync);
1248
+ }
1249
+ if (settings.changes !== null) {
1250
+ const serving = changesServingOf(engine, core);
1251
+ if (serving !== null) hosted.changes = hostedChangesFeed(serving, settings.changes, settings.allowRegex);
1252
+ }
1253
+ if (hosted.sync === void 0 && hosted.changes === void 0) return null;
1254
+ return { host: createFeedHost(hosted, settings.authorize), syncServing };
1255
+ }
533
1256
  function snapshotHistoryOf(raw) {
534
1257
  if (raw === null || raw === "false") return false;
535
1258
  return raw === "true" ? true : null;
@@ -1030,7 +1753,7 @@ function createDocumentExportHandlers(deps) {
1030
1753
  }
1031
1754
  await sendStream(ctx.res, ctx.abort, lines, [["Content-Type", DOCUMENT_EXPORT_CONTENT_TYPE]]);
1032
1755
  } catch (err) {
1033
- respondError(ctx, err);
1756
+ respondPreparingError(ctx, err, ErrorCodes.CHANGES_EXPORT_PREPARING);
1034
1757
  }
1035
1758
  }
1036
1759
  async function importExport(ctx) {
@@ -2235,7 +2958,7 @@ async function startRequestThreads(options) {
2235
2958
  patternWorkCap: options.core.patternWorkCap,
2236
2959
  maxRerankWindow: options.core.rerankSettings.maxWindow,
2237
2960
  allowRegex: options.allowRegex,
2238
- sync: options.feeds !== null,
2961
+ feeds: Object.keys(options.feeds?.gateways ?? {}),
2239
2962
  longQueriesInFlight,
2240
2963
  longQueryPort: longQueryChannel?.port1 ?? null
2241
2964
  }
@@ -2309,229 +3032,32 @@ async function startRequestThreads(options) {
2309
3032
  };
2310
3033
  }
2311
3034
 
2312
- // src/sync/feed/engine-source.ts
2313
- function syncSettingsOf(entry) {
2314
- const { config } = entry;
2315
- const synonyms = entry.synonyms === null ? [] : rulesOfStoredSynonyms(entry.synonyms);
2316
- return {
2317
- language: entry.language.name,
2318
- ...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
2319
- ...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
2320
- ...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
2321
- ...entry.analysisRevision !== null ? { analysisRevision: entry.analysisRevision } : {},
2322
- ...config.bm25 !== void 0 ? { bm25: config.bm25 } : {},
2323
- ...synonyms.length > 0 ? { synonyms } : {}
2324
- };
2325
- }
2326
- function storageOf(core) {
2327
- return core.durability?.manager.syncStorage?.() ?? null;
2328
- }
2329
- function engineFeedSource(engine, core) {
2330
- const requireStorage = () => {
2331
- const storage = storageOf(core);
2332
- if (storage === null) {
2333
- throw new NarsilError(ErrorCodes.SYNC_DURABILITY_REQUIRED, "Device sync needs write-ahead log durability");
2334
- }
2335
- return storage;
2336
- };
2337
- return {
2338
- indexState(indexName) {
2339
- const entry = core.indexRegistry.get(indexName);
2340
- if (entry === void 0 || entry.sync === null || storageOf(core) === null) return null;
2341
- return { epoch: entry.sync.epoch, partitionCount: entry.partitionCount, settings: syncSettingsOf(entry) };
2342
- },
2343
- async bounds(indexName, partitionId) {
2344
- const log = await requireStorage().openPartitionLog(indexName, partitionId);
2345
- const applied = core.durability?.manager.highestPersistedSeqNo(indexName, partitionId) ?? 0;
2346
- return { oldestSeqNo: log.oldestSeqNo, head: log.head, highestSeqNo: Math.max(log.head, applied) };
2347
- },
2348
- async read(indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) {
2349
- const read = await requireStorage().readPartitionLog(indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes);
2350
- return { frames: read.frames, gone: read.gone };
2351
- },
2352
- async bytesBehind(indexName, seqNos) {
2353
- const storage = requireStorage();
2354
- let bytes = 0;
2355
- for (let partitionId = 0; partitionId < seqNos.length; partitionId++) {
2356
- const log = await storage.openPartitionLog(indexName, partitionId);
2357
- bytes += await storage.logBytesAfter(log, seqNos[partitionId]);
2358
- }
2359
- return bytes;
2360
- },
2361
- async snapshotBytes(indexName) {
2362
- const storage = requireStorage();
2363
- const published = await storage.published(indexName);
2364
- if (published !== null) return publishedArchiveLayout(indexName, published, storage).byteLength;
2365
- const manifest = await storage.currentManifest(indexName);
2366
- if (manifest === null) return 0;
2367
- return estimatedArchiveBytes(manifestFileRefs(manifest));
2368
- },
2369
- pendingBeyondHead(indexName, partitionId, head) {
2370
- return (core.durability?.manager.highestPersistedSeqNo(indexName, partitionId) ?? 0) > head;
2371
- },
2372
- listen(onChange) {
2373
- const onWrite = (payload) => onChange({ kind: "advanced", indexName: payload.indexName });
2374
- engine.on("write", onWrite);
2375
- const stopSyncChanges = listenForSyncChanges(core.indexRegistry, onChange);
2376
- return () => {
2377
- engine.off("write", onWrite);
2378
- stopSyncChanges();
2379
- };
2380
- }
2381
- };
2382
- }
2383
-
2384
- // src/server/sync/engine-serving.ts
2385
- function engineSyncServing(engine, core) {
2386
- startSyncRetentionSweep(core);
2387
- return {
2388
- async snapshot(indexName) {
2389
- const published = await publishedSyncSnapshot(core, indexName);
2390
- try {
2391
- return {
2392
- entityTag: published.snapshot.entityTag,
2393
- layout: publishedArchiveLayout(indexName, published.snapshot, published.storage),
2394
- release: published.unpin
2395
- };
2396
- } catch (error) {
2397
- published.unpin();
2398
- throw error;
2399
- }
2400
- },
2401
- feed: engineFeedSource(engine, core),
2402
- admitFeed: async () => void 0
2403
- };
2404
- }
2405
-
2406
- // src/server/sync/feed-host.ts
2407
- var RelayedFeedSocket = class {
2408
- constructor(port, feedId) {
2409
- this.port = port;
2410
- this.feedId = feedId;
2411
- }
2412
- inTransit = 0;
2413
- threadBuffered = 0;
2414
- closed = false;
2415
- send(bytes) {
2416
- if (this.closed) return;
2417
- const copy = bytes.slice();
2418
- this.inTransit += copy.byteLength;
2419
- this.post({ type: "feed-send", feedId: this.feedId, bytes: copy }, [copy.buffer]);
2420
- }
2421
- bufferedAmount() {
2422
- return this.inTransit + this.threadBuffered;
2423
- }
2424
- close(code, reason) {
2425
- if (this.closed) return;
2426
- this.closed = true;
2427
- this.post({ type: "feed-close", feedId: this.feedId, code, reason }, []);
2428
- }
2429
- sent(bytes, buffered) {
2430
- this.inTransit = Math.max(0, this.inTransit - bytes);
2431
- this.threadBuffered = buffered;
2432
- }
2433
- markClosed() {
2434
- this.closed = true;
2435
- }
2436
- post(note, transfer) {
2437
- try {
2438
- this.port.postMessage(note, transfer);
2439
- } catch {
2440
- this.closed = true;
2441
- }
2442
- }
2443
- };
2444
- function createFeedHost(hub, serving, authorize) {
2445
- async function admit(indexName, expiresAt) {
2446
- await serving.admitFeed(indexName);
2447
- return hub.admit(indexName, expiresAt);
2448
- }
2449
- const gateway = {
2450
- authorize: (context) => authorize === void 0 ? Promise.resolve({ allowed: true, expiresAt: null }) : authorize(context),
2451
- async admit(indexName, expiresAt) {
2452
- const admission = await admit(indexName, expiresAt);
2453
- return { open: (socket) => admission.open(socket), cancel: () => admission.cancel() };
2454
- }
2455
- };
2456
- function forThread(port) {
2457
- const feeds = /* @__PURE__ */ new Map();
2458
- let nextFeedId = 0;
2459
- let closed = false;
2460
- function closeFeed(feedId) {
2461
- const feed = feeds.get(feedId);
2462
- if (feed === void 0) return;
2463
- feeds.delete(feedId);
2464
- feed.socket.markClosed();
2465
- if (feed.session === null) feed.admission.cancel();
2466
- else feed.session.closedByPeer();
2467
- }
2468
- return {
2469
- async admit(request) {
2470
- try {
2471
- const admission = await admit(request.indexName, request.expiresAt);
2472
- if (closed) {
2473
- admission.cancel();
2474
- return { type: "feed-admitted", requestId: request.requestId, feedId: null, error: null };
2475
- }
2476
- nextFeedId += 1;
2477
- const feedId = nextFeedId;
2478
- feeds.set(feedId, { admission, socket: new RelayedFeedSocket(port, feedId), session: null });
2479
- return { type: "feed-admitted", requestId: request.requestId, feedId, error: null };
2480
- } catch (err) {
2481
- return { type: "feed-admitted", requestId: request.requestId, feedId: null, error: toHttpError(err).body };
2482
- }
2483
- },
2484
- handle(note) {
2485
- const feed = feeds.get(note.feedId);
2486
- if (feed === void 0) return;
2487
- switch (note.type) {
2488
- case "feed-open":
2489
- if (feed.session === null) feed.session = feed.admission.open(feed.socket);
2490
- return;
2491
- case "feed-message":
2492
- feed.session?.receive(note.bytes, note.binary);
2493
- return;
2494
- case "feed-sent":
2495
- feed.socket.sent(note.bytes, note.buffered);
2496
- feed.session?.drained();
2497
- return;
2498
- case "feed-cancel":
2499
- case "feed-closed":
2500
- closeFeed(note.feedId);
2501
- return;
2502
- }
2503
- },
2504
- closeAll() {
2505
- closed = true;
2506
- for (const feedId of [...feeds.keys()]) closeFeed(feedId);
2507
- }
2508
- };
2509
- }
2510
- return { gateway, forThread, close: () => hub.close() };
2511
- }
2512
-
2513
3035
  // src/server/sync/options.ts
2514
- var SYNC_OPTION_FIELDS = ["windowBytes", "maxConnections", "maxCatchUps", "maxBufferedBytes"];
2515
- function assertCount(field, value) {
3036
+ var FEED_OPTION_FIELDS = ["windowBytes", "maxConnections", "maxCatchUps", "maxBufferedBytes"];
3037
+ var FEED_REFUSALS = {
3038
+ sync: "Refusing to start: the device sync routes are on with no onRequest hook. A device receives the whole index, so set options.onRequest to admit only the user whose documents the index holds. allowInsecure does not lift this refusal.",
3039
+ changes: "Refusing to start: the change feed is on with no onRequest hook. A subscriber receives every document of the index, so set options.onRequest to admit only the services that may read the whole index. allowInsecure does not lift this refusal."
3040
+ };
3041
+ function assertCount(feed, field, value) {
2516
3042
  if (value === void 0) return;
2517
3043
  if (typeof value === "number" && Number.isSafeInteger(value) && value >= 1) return;
2518
3044
  throw new NarsilError(
2519
3045
  ErrorCodes.CONFIG_INVALID,
2520
- `sync.${field} must be a whole number of at least 1; however, this configuration sets it to ${String(value)}`,
3046
+ `${feed}.${field} must be a whole number of at least 1; however, this configuration sets it to ${String(value)}`,
2521
3047
  { field, value: String(value) }
2522
3048
  );
2523
3049
  }
2524
- function resolveSyncOptions(sync) {
2525
- if (sync === void 0 || sync === false) return null;
2526
- if (sync !== true && (typeof sync !== "object" || sync === null)) {
3050
+ function resolveFeedOptions(feed, value) {
3051
+ if (value === void 0 || value === false) return null;
3052
+ if (value !== true && (typeof value !== "object" || value === null)) {
2527
3053
  throw new NarsilError(
2528
3054
  ErrorCodes.CONFIG_INVALID,
2529
- `sync must be true, false, or an object of settings; however, this configuration sets it to ${String(sync)}`,
2530
- { field: "sync", value: String(sync) }
3055
+ `${feed} must be true, false, or an object of settings; however, this configuration sets it to ${String(value)}`,
3056
+ { field: feed, value: String(value) }
2531
3057
  );
2532
3058
  }
2533
- const settings = sync === true ? {} : sync;
2534
- for (const field of SYNC_OPTION_FIELDS) assertCount(field, settings[field]);
3059
+ const settings = value === true ? {} : value;
3060
+ for (const field of FEED_OPTION_FIELDS) assertCount(feed, field, settings[field]);
2535
3061
  return {
2536
3062
  windowBytes: settings.windowBytes ?? DEFAULT_FEED_WINDOW_BYTES,
2537
3063
  maxConnections: settings.maxConnections ?? DEFAULT_MAX_FEED_CONNECTIONS,
@@ -2539,13 +3065,9 @@ function resolveSyncOptions(sync) {
2539
3065
  maxBufferedBytes: settings.maxBufferedBytes ?? DEFAULT_MAX_FEED_BUFFERED_BYTES
2540
3066
  };
2541
3067
  }
2542
- function assertSyncHasAuthentication(sync, options) {
2543
- if (sync === null || options.onRequest !== void 0) return;
2544
- throw new NarsilError(
2545
- ErrorCodes.CONFIG_INVALID,
2546
- "Refusing to start: the device sync routes are on with no onRequest hook. A device receives the whole index, so set options.onRequest to admit only the user whose documents the index holds. allowInsecure does not lift this refusal.",
2547
- { field: "sync" }
2548
- );
3068
+ function assertFeedHasAuthentication(feed, resolved, options) {
3069
+ if (resolved === null || options.onRequest !== void 0) return;
3070
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, FEED_REFUSALS[feed], { field: feed });
2549
3071
  }
2550
3072
 
2551
3073
  // src/server/sync/setting-handlers.ts
@@ -2849,6 +3371,7 @@ function mainThreadHandlers(deps, syncServing) {
2849
3371
  const version = createVersionHandler(deps);
2850
3372
  const capabilities = createCapabilitiesHandler();
2851
3373
  const syncSetting = createSyncSettingHandlers(deps.engine);
3374
+ const changesSetting = createChangesSettingHandlers(deps.engine);
2852
3375
  const suggestions = createQuerySuggestionHandlers(deps, deps.querySuggestions);
2853
3376
  return {
2854
3377
  livez: health.livez,
@@ -2909,6 +3432,8 @@ function mainThreadHandlers(deps, syncServing) {
2909
3432
  getSync: syncSetting.getSync,
2910
3433
  updateSync: syncSetting.updateSync,
2911
3434
  syncSnapshot: createSyncSnapshotHandler(syncServing),
3435
+ getChanges: changesSetting.getChanges,
3436
+ updateChanges: changesSetting.updateChanges,
2912
3437
  updateQuerySuggestions: suggestions.update,
2913
3438
  querySuggestionsStatus: suggestions.status,
2914
3439
  recordQuery: suggestions.record,
@@ -2931,6 +3456,7 @@ var NarsilHttpServer = class {
2931
3456
  ready = false;
2932
3457
  threads = null;
2933
3458
  sync;
3459
+ changes;
2934
3460
  syncServing = null;
2935
3461
  feeds = null;
2936
3462
  stopIdleHeapCleanup = () => void 0;
@@ -2940,7 +3466,8 @@ var NarsilHttpServer = class {
2940
3466
  assertServerLimits(options.limits, engineRerankWindow);
2941
3467
  assertAllowRegex(options.allowRegex);
2942
3468
  assertQuerySuggestionsOptions(options.querySuggestions);
2943
- this.sync = resolveSyncOptions(options.sync);
3469
+ this.sync = resolveFeedOptions("sync", options.sync);
3470
+ this.changes = resolveFeedOptions("changes", options.changes);
2944
3471
  this.host = options.host ?? "127.0.0.1";
2945
3472
  this.port = options.port ?? 9876;
2946
3473
  this.options = options;
@@ -2968,7 +3495,8 @@ var NarsilHttpServer = class {
2968
3495
  };
2969
3496
  }
2970
3497
  async listen() {
2971
- assertSyncHasAuthentication(this.sync, this.options);
3498
+ assertFeedHasAuthentication("sync", this.sync, this.options);
3499
+ assertFeedHasAuthentication("changes", this.changes, this.options);
2972
3500
  assertSecureBinding(this.host, this.options);
2973
3501
  const uws = await loadUWebSockets();
2974
3502
  this.uws = uws;
@@ -3016,20 +3544,19 @@ var NarsilHttpServer = class {
3016
3544
  get requestThreadCount() {
3017
3545
  return this.threads?.threadCount ?? 0;
3018
3546
  }
3019
- startSync(app, authorize, core) {
3020
- if (this.sync === null) return;
3021
- const servingOfNode = syncServingSourceOf(this.engine);
3022
- let serving;
3023
- if (servingOfNode !== void 0) serving = servingOfNode();
3024
- else if (core !== void 0) serving = engineSyncServing(this.engine, core);
3025
- else return;
3026
- const hub = createFeedHub(serving.feed, {
3027
- ...this.sync,
3028
- onError: (error) => console.warn("A device sync feed failed to read its index:", error)
3547
+ startFeeds(app, authorize, core) {
3548
+ const started = startFeeds(this.engine, core, {
3549
+ sync: this.sync,
3550
+ changes: this.changes,
3551
+ allowRegex: this.deps.allowRegex,
3552
+ authorize
3029
3553
  });
3030
- this.syncServing = serving;
3031
- this.feeds = createFeedHost(hub, serving, authorize);
3032
- registerFeedRoute(app, this.feeds.gateway);
3554
+ if (started === null) return;
3555
+ this.syncServing = started.syncServing;
3556
+ this.feeds = started.host;
3557
+ const { sync, changes } = started.host.gateways;
3558
+ if (sync !== void 0) registerFeedRoute(app, sync, SYNC_FEED_ROUTE);
3559
+ if (changes !== void 0) registerFeedRoute(app, changes, CHANGES_FEED_ROUTE);
3033
3560
  }
3034
3561
  async registerRoutes(app) {
3035
3562
  const cors = resolveCors(this.options.cors);
@@ -3040,7 +3567,7 @@ var NarsilHttpServer = class {
3040
3567
  const bodyBudget = createBodyBudgetBuffer(gateSlotOfWorker(threadCount));
3041
3568
  const { limits, build, allowRegex } = this.deps;
3042
3569
  const handlers = mainThreadHandlers(this.deps, () => this.syncServing);
3043
- this.startSync(app, authorize, core);
3570
+ this.startFeeds(app, authorize, core);
3044
3571
  const run = createRouteRunner({
3045
3572
  authorize,
3046
3573
  gate: createSharedRequestGate(gate, limits.maxConcurrentRequests, MAIN_THREAD_GATE_SLOT),