@delali/narsil 0.4.1 → 0.4.2-next.273

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/README.md +1 -1
  2. package/dist/adapters/filesystem.d.ts +1 -1
  3. package/dist/adapters/filesystem.mjs +9 -3
  4. package/dist/adapters/gcs.d.ts +1 -1
  5. package/dist/adapters/indexeddb.d.ts +6 -1
  6. package/dist/adapters/indexeddb.mjs +20 -26
  7. package/dist/adapters/memory.d.ts +1 -1
  8. package/dist/adapters/s3.d.ts +1 -1
  9. package/dist/{adapters-pljAqmrC.d.ts → adapters-8LcdNB77.d.ts} +18 -1
  10. package/dist/browser/{chunk-QAOVAYAG.mjs → chunk-56UNHLAF.mjs} +1 -1
  11. package/dist/browser/{chunk-GBJSEHVJ.mjs → chunk-EKISQBV2.mjs} +283 -244
  12. package/dist/browser/{chunk-YINZFPLT.mjs → chunk-VAPCFSMX.mjs} +1 -1
  13. package/dist/browser/{chunk-4E5APHTM.mjs → chunk-VIVG3OXM.mjs} +1 -1
  14. package/dist/browser/{durability-config-75NSFCQN.mjs → durability-config-UP3OTUMB.mjs} +34 -2
  15. package/dist/browser/{msgpack-leaf-44XNWSSY.mjs → msgpack-leaf-76Q5SZ5S.mjs} +23 -10
  16. package/dist/browser/{optional-reads-6LN3HMUH.mjs → optional-reads-NQO5DKM4.mjs} +18 -16
  17. package/dist/browser/{optional-writes-76DWMRQH.mjs → optional-writes-3HKIVZFU.mjs} +295 -106
  18. package/dist/browser/{segment-codec-leaf-A427QJUX.mjs → segment-codec-leaf-GXSRX5VT.mjs} +47 -30
  19. package/dist/browser/{storage-leaf-5UTR5K7J.mjs → storage-leaf-42IZEOJJ.mjs} +1473 -305
  20. package/dist/{capabilities-BOFFMx84.d.ts → capabilities-BoWiLNv7.d.ts} +76 -61
  21. package/dist/{chunk-Z7XSHKOF.mjs → chunk-35RWLUVY.mjs} +63 -144
  22. package/dist/{chunk-6GHX35ZM.mjs → chunk-43T3P3BE.mjs} +1 -1
  23. package/dist/{chunk-IL5DRCNU.mjs → chunk-4JLHHEBN.mjs} +1 -1
  24. package/dist/{chunk-DKIMHIBC.mjs → chunk-4YIVY3PB.mjs} +2 -2
  25. package/dist/{react-native/chunks/chunk-ZUJCUPVS.mjs → chunk-5SHWA6PE.mjs} +1 -1
  26. package/dist/{chunk-2K24MEYV.mjs → chunk-6LVQ3GLT.mjs} +3 -3
  27. package/dist/{chunk-DDUXEJXW.mjs → chunk-6SIEVWMG.mjs} +45 -22
  28. package/dist/{chunk-WM5KJSQV.mjs → chunk-ACNHJOGC.mjs} +1 -1
  29. package/dist/{chunk-755NM4PL.mjs → chunk-BWPVDIRN.mjs} +183 -21
  30. package/dist/{chunk-SMQPZHPS.mjs → chunk-C57GWAXN.mjs} +1 -1
  31. package/dist/{chunk-357KIU5E.mjs → chunk-DRZP5MSG.mjs} +5 -5
  32. package/dist/{chunk-MYN4K5IS.mjs → chunk-DZMD4OMD.mjs} +1 -1
  33. package/dist/{chunk-7P5YWDRQ.mjs → chunk-EY5U4P5V.mjs} +1 -1
  34. package/dist/{chunk-X3EQCQLP.mjs → chunk-EZJ6HBG7.mjs} +2 -2
  35. package/dist/{chunk-WC5KAWXQ.mjs → chunk-FE2XILWN.mjs} +4 -4
  36. package/dist/{chunk-CERPUGRK.mjs → chunk-G3KVEHIW.mjs} +2 -2
  37. package/dist/{chunk-B3MO37YF.mjs → chunk-GTGQLCEI.mjs} +1 -1
  38. package/dist/{chunk-MSJD74GN.mjs → chunk-H23FB267.mjs} +5 -5
  39. package/dist/{chunk-UNEEFTEU.mjs → chunk-HZ3OAUTL.mjs} +2 -1
  40. package/dist/{chunk-M3B7SGJB.mjs → chunk-KL4ORYJW.mjs} +6 -6
  41. package/dist/{chunk-WTMCGZ74.mjs → chunk-KXA77DPZ.mjs} +2 -2
  42. package/dist/{chunk-6I6OO6B2.mjs → chunk-LRIJ7U3T.mjs} +4 -4
  43. package/dist/{chunk-3IE3A74H.mjs → chunk-PNFP5THV.mjs} +3 -3
  44. package/dist/{react-native/chunks/chunk-4FUCRLSA.mjs → chunk-PQWK7KXS.mjs} +1 -1
  45. package/dist/{chunk-P4F4UWLC.mjs → chunk-PSO7V7VW.mjs} +17 -3
  46. package/dist/{chunk-OGYSPNL2.mjs → chunk-QNZJYMCF.mjs} +7 -7
  47. package/dist/{chunk-DBC4EJWB.mjs → chunk-R3PFPFME.mjs} +1 -1
  48. package/dist/{chunk-MSIN4NM3.mjs → chunk-S3JHYPH7.mjs} +6 -1
  49. package/dist/{chunk-MI6UZGUR.mjs → chunk-SPA7GXST.mjs} +3 -3
  50. package/dist/{chunk-PP3W5PQQ.mjs → chunk-WYH3QHBK.mjs} +698 -451
  51. package/dist/{chunk-D456ZFPQ.mjs → chunk-X5V25BPN.mjs} +2 -2
  52. package/dist/{chunk-GXCDWLQI.mjs → chunk-Y7GXGJFZ.mjs} +23 -10
  53. package/dist/{chunk-NCIGNQJE.mjs → chunk-YGRJLI2Z.mjs} +930 -387
  54. package/dist/{react-native/chunks/chunk-MMPBO4WR.mjs → chunk-ZO7VLAIT.mjs} +2 -2
  55. package/dist/{chunk-AXDB6KUQ.mjs → chunk-ZTB7AWTW.mjs} +4 -4
  56. package/dist/chunk-ZYEEOZTT.mjs +52 -0
  57. package/dist/{client-BhNxHtcZ.d.ts → client-B0NPZSKY.d.ts} +4 -4
  58. package/dist/client.d.ts +9 -9
  59. package/dist/client.mjs +2 -2
  60. package/dist/{config-P9zW3Zz7.d.ts → config-B3-sw887.d.ts} +3 -3
  61. package/dist/{config-BJSL7piq.d.ts → config-DL_nkguq.d.ts} +4 -4
  62. package/dist/{direct-executor-GWJXDFLD.mjs → direct-executor-7G57AYXH.mjs} +4 -4
  63. package/dist/distribution/coordinator/etcd.d.ts +3 -3
  64. package/dist/distribution/coordinator/etcd.mjs +33 -3
  65. package/dist/distribution/coordinator/in-memory.d.ts +3 -3
  66. package/dist/distribution/transport/grpc.mjs +4 -4
  67. package/dist/distribution/transport/in-memory.mjs +1 -1
  68. package/dist/distribution/transport/tcp.mjs +12 -7
  69. package/dist/distribution.d.ts +8 -8
  70. package/dist/distribution.mjs +167 -91
  71. package/dist/{durability-config-VWS2K24E.mjs → durability-config-HWKWP35M.mjs} +34 -2
  72. package/dist/embeddings/openai.d.ts +7 -3
  73. package/dist/embeddings/openai.mjs +37 -31
  74. package/dist/{engine-DoFnkRoF.d.ts → engine-DGG_yX0p.d.ts} +3 -3
  75. package/dist/{index-B_vmxpgW.d.ts → index-fX_FbuRT.d.ts} +1 -1
  76. package/dist/index.browser.mjs +3 -3
  77. package/dist/index.d.ts +7 -7
  78. package/dist/index.mjs +4 -4
  79. package/dist/index.react-native.mjs +4 -4
  80. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  81. package/dist/invalidation/filesystem.d.ts +1 -1
  82. package/dist/invalidation/noop.d.ts +1 -1
  83. package/dist/{long-query-thread-GNTDYDOZ.mjs → long-query-thread-KPN7NSOC.mjs} +14 -14
  84. package/dist/msgpack-leaf-LIT7TKZW.mjs +9 -0
  85. package/dist/{optional-reads-UCS23AWP.mjs → optional-reads-ESDVS225.mjs} +18 -16
  86. package/dist/{optional-writes-2AHJCGTY.mjs → optional-writes-CSULLISE.mjs} +9 -8
  87. package/dist/{paged-partition-archive-GBAHDT55.mjs → paged-partition-archive-BKQADXOT.mjs} +13 -13
  88. package/dist/partial-words.browser.mjs +3 -3
  89. package/dist/partial-words.mjs +4 -4
  90. package/dist/pattern.browser.mjs +2 -2
  91. package/dist/pattern.mjs +2 -2
  92. package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
  93. package/dist/persistence/durability/checkpoint-worker.mjs +14 -14
  94. package/dist/{query-actions-IDCUA7UV.mjs → query-actions-HTSJKZRU.mjs} +4 -4
  95. package/dist/query-suggestions/merge-worker.mjs +7 -7
  96. package/dist/query-suggestions.browser.mjs +3 -3
  97. package/dist/query-suggestions.d.ts +3 -3
  98. package/dist/query-suggestions.mjs +5 -5
  99. package/dist/react-native/chunks/{chunk-MRH36RN3.mjs → chunk-AOLLV37A.mjs} +1 -1
  100. package/dist/react-native/chunks/{chunk-UO44MA5M.mjs → chunk-DN4FT62B.mjs} +23 -10
  101. package/dist/react-native/chunks/{chunk-TG56N5JC.mjs → chunk-EF4Y5C3Y.mjs} +2 -2
  102. package/dist/react-native/chunks/{chunk-HTBY4GON.mjs → chunk-ESVBC3AT.mjs} +1 -1
  103. package/dist/{chunk-F26DOOUH.mjs → react-native/chunks/chunk-H6LK5WCF.mjs} +1 -1
  104. package/dist/react-native/chunks/{chunk-LPX26NNL.mjs → chunk-JEYZQA4Z.mjs} +699 -451
  105. package/dist/react-native/chunks/{chunk-AJAH6X7W.mjs → chunk-KJQ3BUVM.mjs} +4 -4
  106. package/dist/{chunk-UHLFCGZ3.mjs → react-native/chunks/chunk-QYVRD6C5.mjs} +2 -2
  107. package/dist/react-native/chunks/{chunk-SCGBU47K.mjs → chunk-X3VY2YT4.mjs} +2 -2
  108. package/dist/{chunk-R2PJKSNL.mjs → react-native/chunks/chunk-ZF3VSGEI.mjs} +1 -1
  109. package/dist/react-native/chunks/{durability-config-FCSD4MXY.mjs → durability-config-HJETGZVB.mjs} +34 -2
  110. package/dist/react-native/chunks/msgpack-leaf-FOUPTT7J.mjs +7 -0
  111. package/dist/react-native/chunks/{optional-reads-4XCJ34QG.mjs → optional-reads-TN3PT6M4.mjs} +18 -16
  112. package/dist/react-native/chunks/{optional-writes-XYZVTBFB.mjs → optional-writes-MGFFGKYQ.mjs} +8 -7
  113. package/dist/react-native/chunks/{restore-queue-NOKQZJ2Z.mjs → restore-queue-MVJTP3E4.mjs} +7 -7
  114. package/dist/react-native/chunks/{segment-codec-leaf-3ZXBJVKH.mjs → segment-codec-leaf-PSWHL55A.mjs} +5 -5
  115. package/dist/react-native/chunks/{segment-merge-leaf-3Z7MMPWE.mjs → segment-merge-leaf-WOJWSDLW.mjs} +6 -6
  116. package/dist/react-native/chunks/{storage-leaf-KJOX3MMW.mjs → storage-leaf-ZUN6Y44X.mjs} +1481 -314
  117. package/dist/react-native/chunks/{store-leaf-QGLVSO5X.mjs → store-leaf-K5XG46FV.mjs} +6 -6
  118. package/dist/react-native/chunks/{suggestions-leaf-6GI5ZSW6.mjs → suggestions-leaf-KDPM3P27.mjs} +6 -6
  119. package/dist/react-native/chunks/{worker-copies-ALOD7TSP.mjs → worker-copies-FVFMBBH6.mjs} +5 -5
  120. package/dist/react-native/partial-words.mjs +4 -4
  121. package/dist/react-native/pattern.mjs +2 -2
  122. package/dist/react-native/query-suggestions.mjs +4 -4
  123. package/dist/react-native/sync.mjs +4 -4
  124. package/dist/react-native/synonyms.mjs +4 -4
  125. package/dist/react.d.ts +10 -10
  126. package/dist/react.mjs +0 -2
  127. package/dist/{rerank-HB6DU6GS.mjs → rerank-3QEJNMFH.mjs} +4 -4
  128. package/dist/rerankers/http.d.ts +1 -1
  129. package/dist/rerankers/http.mjs +19 -15
  130. package/dist/{restore-queue-VUQHGCXE.mjs → restore-queue-Q76Q4S35.mjs} +7 -7
  131. package/dist/{schema-CzG9OUw5.d.ts → schema-CFnQ-WI3.d.ts} +1 -1
  132. package/dist/{segment-codec-leaf-TW2FPAAT.mjs → segment-codec-leaf-BSDDR7DQ.mjs} +5 -5
  133. package/dist/{segment-merge-leaf-BS3QV5PZ.mjs → segment-merge-leaf-6TT66HC5.mjs} +5 -5
  134. package/dist/server.d.ts +86 -12
  135. package/dist/server.mjs +2243 -1918
  136. package/dist/{storage-leaf-5IG2HPNY.mjs → storage-leaf-R5JQ2X2M.mjs} +1193 -219
  137. package/dist/{store-leaf-ZWSOK2DV.mjs → store-leaf-NCGETZYY.mjs} +6 -6
  138. package/dist/{suggestions-leaf-MUZNKQVF.mjs → suggestions-leaf-TX3GTD6S.mjs} +8 -8
  139. package/dist/sync/filesystem.d.ts +4 -4
  140. package/dist/sync/filesystem.mjs +12 -12
  141. package/dist/sync/indexeddb.browser.mjs +1 -1
  142. package/dist/sync/indexeddb.d.ts +4 -4
  143. package/dist/sync/indexeddb.mjs +2 -2
  144. package/dist/sync.browser.mjs +4 -4
  145. package/dist/sync.d.ts +5 -5
  146. package/dist/sync.mjs +4 -4
  147. package/dist/synonyms.browser.mjs +3 -3
  148. package/dist/synonyms.mjs +4 -4
  149. package/dist/{tasks-DrLiolTG.d.ts → tasks-B2MDjPL0.d.ts} +4 -4
  150. package/dist/{thread-6ZBCENAP.mjs → thread-VLVPAUWC.mjs} +13 -13
  151. package/dist/{types-pS-mAJmX.d.ts → types-DGjk684l.d.ts} +2 -2
  152. package/dist/vector/search-worker.d.ts +2 -2
  153. package/dist/{worker-copies-32HWFUH7.mjs → worker-copies-NZSBQZKG.mjs} +7 -7
  154. package/dist/workers/entry.mjs +16 -16
  155. package/package.json +12 -12
  156. package/dist/chunk-HLPRHOSO.mjs +0 -17
  157. package/dist/msgpack-leaf-H5ZYW7S7.mjs +0 -9
  158. package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +0 -7
package/dist/server.mjs CHANGED
@@ -1,43 +1,43 @@
1
- import { answerRequest } from './chunk-AXDB6KUQ.mjs';
1
+ import { answerRequest } from './chunk-ZTB7AWTW.mjs';
2
2
  import { resolveVectorText } from './chunk-NJJN4FMM.mjs';
3
- import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, registerFeedRoute, SYNC_FEED_ROUTE, CHANGES_FEED_ROUTE, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, createSyncSnapshotHandler, acceptorOf, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, PROMETHEUS_TEXT_CONTENT_TYPE, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, createCloseAtExitWords, createDepartureWord, departureConfirmed, refusedRegex, toHttpError, parseJson, validateCreateIndex, rejectInvalid, isRemoveOnlyBatch, validateBatch, serializeBatchResult, parseWriteJson, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, requestSignal, withServerRerankWindow, respondAnswer, encodeQueryResult, parseTaskListQuery, parseJsonOptional, validateChangeAnalysis, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, respondPreparingError, parseRefusalOf, IMPORT_FLUSHES_IN_FLIGHT, isThreadFeedNote, clearGateSlot, clearBodyBudgetSlot, confirmDeparture, SERVER_CAPABILITIES, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-NCIGNQJE.mjs';
4
- export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, CHANGE_ANALYSIS_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-NCIGNQJE.mjs';
5
- import { startSyncRetentionSweep, publishedSyncSnapshot, publishedArchiveLayout } from './chunk-OGYSPNL2.mjs';
3
+ import { DEFAULT_MAX_RETAINED_TASKS, isTerminalTaskStatus, TASK_RECORD_LOAD_CONCURRENCY, loadUWebSockets, SERVER_CLOSE_DRAIN_DEADLINE_MS, registerFeedRoute, SYNC_FEED_ROUTE, CHANGES_FEED_ROUTE, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, isTaskType, isTaskStatus, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, createSyncSnapshotHandler, acceptorOf, createServerCloseSignal, signalServerClose, LeavingResponse, refuseAndClose, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, PROMETHEUS_TEXT_CONTENT_TYPE, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, createCloseAtExitWords, createDepartureWord, departureConfirmed, threadCloseAcknowledgedWithin, SERVER_CLOSE_ACKNOWLEDGE_GRACE_MS, threadCloseAcknowledged, refusedRegex, toHttpError, parseJsonObject, validateCreateIndex, rejectInvalid, isRemoveOnlyBatch, validateBatch, serializeBatchResult, parseWriteJsonObject, validateDocumentBody, validateSuggest, readQueryBody, withServerQueryLimits, requestSignal, respondAnswer, encodeQueryResult, parseTaskListQuery, parseJsonOptional, validateChangeAnalysis, validateRebalance, parseJson, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, respondPreparingError, parseRefusalOf, IMPORT_FLUSHES_IN_FLIGHT, isThreadFeedNote, clearGateSlot, clearBodyBudgetSlot, confirmDeparture, SERVER_CAPABILITIES, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-YGRJLI2Z.mjs';
4
+ export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, CHANGE_ANALYSIS_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-YGRJLI2Z.mjs';
5
+ import { startSyncRetentionSweep, publishedSyncSnapshot, publishedArchiveLayout } from './chunk-QNZJYMCF.mjs';
6
6
  import { SYNC_CLOSE_RESYNC, SYNC_CLOSE_CREDENTIAL_EXPIRED, SYNC_CLOSE_POLICY, MAX_DEVICE_MESSAGE_BYTES, SYNC_CLOSE_TOO_LARGE, deviceMessageOf, SYNC_CLOSE_SERVER_ERROR, positionList, positionsByPartition, CHANGES_CLOSE_RESYNC, CHANGES_CLOSE_POLICY, MAX_SUBSCRIBER_MESSAGE_BYTES, CHANGES_CLOSE_TOO_LARGE, closeReasonOf, CHANGES_CLOSE_CREDENTIAL_EXPIRED, CHANGES_CLOSE_SETTING, CHANGES_CLOSE_SERVER_ERROR, CHANGES_CLOSE_GOING_AWAY, SYNC_CLOSE_GOING_AWAY } from './chunk-TACP6VK6.mjs';
7
- import { syncServingSourceOf, changesServingSourceOf, writeBlockLookupOf, indexReadsOf, validateBackupName, readRestorePlacement } from './chunk-MI6UZGUR.mjs';
8
- import { isRecord, validateIndexName } from './chunk-2K24MEYV.mjs';
7
+ import { syncServingSourceOf, changesServingSourceOf, writeBlockLookupOf, indexReadsOf, validateBackupName, readRestorePlacement } from './chunk-SPA7GXST.mjs';
8
+ import { isRecord, validateIndexName } from './chunk-6LVQ3GLT.mjs';
9
9
  import { executeUntilAnswered } from './chunk-4H4RG2FR.mjs';
10
- import './chunk-WM5KJSQV.mjs';
11
- import { engineChangeSource, createSemaphore, createByteBudget, changesNotEnabled, subscriberMessageOf, ChangeCursor, changePositionList, seqNosByPartition } from './chunk-3IE3A74H.mjs';
10
+ import './chunk-ACNHJOGC.mjs';
11
+ import { engineChangeSource, createSemaphore, createByteBudget, changesNotEnabled, subscriberMessageOf, ChangeCursor, changePositionList, seqNosByPartition } from './chunk-PNFP5THV.mjs';
12
12
  import './chunk-M2Z5WVWW.mjs';
13
- import { DEFAULT_MAX_FEED_BUFFERED_BYTES, DEFAULT_MAX_FEED_CATCH_UPS, DEFAULT_MAX_FEED_CONNECTIONS, DEFAULT_FEED_WINDOW_BYTES, estimatedArchiveBytes, TIMER_DELAY_LIMIT_MS, FEED_HEAD_RETRY_MS, ENTRIES_MESSAGE_LIMIT_BYTES } from './chunk-6I6OO6B2.mjs';
13
+ import { DEFAULT_MAX_FEED_BUFFERED_BYTES, DEFAULT_MAX_FEED_CATCH_UPS, DEFAULT_MAX_FEED_CONNECTIONS, DEFAULT_FEED_WINDOW_BYTES, estimatedArchiveBytes, TIMER_DELAY_LIMIT_MS, FEED_HEAD_RETRY_MS, ENTRIES_MESSAGE_LIMIT_BYTES } from './chunk-LRIJ7U3T.mjs';
14
14
  import './chunk-JDDXV6UF.mjs';
15
- import { manifestFileRefs } from './chunk-B3MO37YF.mjs';
16
- import './chunk-MSIN4NM3.mjs';
15
+ import { manifestFileRefs } from './chunk-GTGQLCEI.mjs';
17
16
  import { encode as encode$1 } from './chunk-S5QMWECL.mjs';
18
17
  import './chunk-LNGE7SLY.mjs';
19
- import './chunk-R2PJKSNL.mjs';
20
- import './chunk-F26DOOUH.mjs';
21
- import './chunk-UHLFCGZ3.mjs';
18
+ import './chunk-5SHWA6PE.mjs';
19
+ import './chunk-PQWK7KXS.mjs';
20
+ import './chunk-ZO7VLAIT.mjs';
22
21
  import './chunk-R724DJSG.mjs';
23
22
  import './chunk-V6EUBAIT.mjs';
24
- import './chunk-CERPUGRK.mjs';
25
- import { querySuggestions } from './chunk-DBC4EJWB.mjs';
23
+ import './chunk-G3KVEHIW.mjs';
24
+ import { querySuggestions } from './chunk-R3PFPFME.mjs';
26
25
  import './chunk-S2POFTLN.mjs';
27
- import { rerankWindowCapOf, engineCoreOf, readMemoryFigures, listenForSyncChanges, MAX_RERANK_WINDOW, bindQuerySignal, clearQuerySignal, providedDocId, loadRerankRuntime, rulesOfStoredSynonyms } from './chunk-PP3W5PQQ.mjs';
26
+ import { rerankWindowCapOf, engineCoreOf, readMemoryFigures, listenForSyncChanges, MAX_RERANK_WINDOW, bindQuerySignal, clearQuerySignal, providedDocId, loadRerankRuntime, rulesOfStoredSynonyms } from './chunk-WYH3QHBK.mjs';
28
27
  import './chunk-2CKDCC65.mjs';
29
28
  import './chunk-2AFZE7Y3.mjs';
30
- import { decode } from './chunk-GXCDWLQI.mjs';
29
+ import { decodeBoundedMessage } from './chunk-Y7GXGJFZ.mjs';
31
30
  import './chunk-BUMREPLU.mjs';
32
31
  import './chunk-7T4LAYT4.mjs';
33
- import './chunk-IL5DRCNU.mjs';
34
- import './chunk-7P5YWDRQ.mjs';
32
+ import './chunk-4JLHHEBN.mjs';
33
+ import './chunk-EY5U4P5V.mjs';
35
34
  import './chunk-U2BKYEC7.mjs';
36
35
  import { detectRuntime } from './chunk-FJXJZUWF.mjs';
37
36
  import { startIdleHeapCleanup } from './chunk-5EI5CXO6.mjs';
38
37
  import './chunk-3ECIRFS5.mjs';
39
38
  import './chunk-ASZLNVWW.mjs';
40
39
  import './chunk-ZXX5GZEX.mjs';
40
+ import './chunk-S3JHYPH7.mjs';
41
41
  import './chunk-GR5GMFJJ.mjs';
42
42
  import './chunk-4PY4JYMY.mjs';
43
43
  import { ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-ZKOQR7EA.mjs';
@@ -45,29 +45,83 @@ export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilEr
45
45
  import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
46
46
  import { randomUUID } from 'crypto';
47
47
  import { encode } from '@msgpack/msgpack';
48
- import { createReadStream } from 'fs';
49
48
  import { MessageChannel } from 'worker_threads';
49
+ import { createReadStream } from 'fs';
50
50
 
51
- // src/server/changes/setting-handlers.ts
52
- function createChangesSettingHandlers(engine) {
53
- async function getChanges(ctx) {
54
- try {
55
- respondJson(ctx, await engine.getChanges(ctx.params[0]));
56
- } catch (err) {
57
- respondError(ctx, err);
51
+ // src/server/wall-clock-wait.ts
52
+ function waitUntilWallClock(deadline) {
53
+ let timer;
54
+ let cancelled = false;
55
+ const reached = new Promise((resolve) => {
56
+ const check = () => {
57
+ if (cancelled) return;
58
+ const remaining = deadline - Date.now();
59
+ if (remaining <= 0) {
60
+ resolve();
61
+ return;
62
+ }
63
+ timer = setTimeout(check, remaining);
64
+ };
65
+ check();
66
+ });
67
+ return {
68
+ reached,
69
+ cancel() {
70
+ cancelled = true;
71
+ clearTimeout(timer);
58
72
  }
73
+ };
74
+ }
75
+
76
+ // src/server/closing-requests.ts
77
+ var CLOSING_REFUSAL = {
78
+ error: {
79
+ code: ErrorCodes.WORKER_CRASHED,
80
+ message: "The server is closing, so retry the request on a new connection"
59
81
  }
60
- async function updateChanges(ctx) {
61
- const body = parseJson(ctx);
62
- if (body === void 0) return;
63
- try {
64
- await engine.updateChanges(ctx.params[0], body);
65
- respondJson(ctx, { ok: true });
66
- } catch (err) {
67
- respondError(ctx, err);
68
- }
82
+ };
83
+ function watchClosingRequests() {
84
+ let closing = false;
85
+ const running = /* @__PURE__ */ new Set();
86
+ const waitingForIdle = [];
87
+ function settle(request) {
88
+ if (!running.delete(request) || running.size > 0) return;
89
+ for (const resume of waitingForIdle.splice(0)) resume();
69
90
  }
70
- return { getChanges, updateChanges };
91
+ return {
92
+ accept(res) {
93
+ if (!closing) return true;
94
+ refuseAndClose(res, 503, CLOSING_REFUSAL);
95
+ return false;
96
+ },
97
+ respondThrough(res) {
98
+ return new LeavingResponse(res, () => closing);
99
+ },
100
+ track(res, abort) {
101
+ const request = {};
102
+ running.add(request);
103
+ let handled = false;
104
+ const finish = () => {
105
+ if (abort.aborted || handled && (!(res instanceof LeavingResponse) || res.isFinished)) settle(request);
106
+ };
107
+ if (res instanceof LeavingResponse) res.whenFinished(finish);
108
+ abort.onAbort(finish);
109
+ return () => {
110
+ handled = true;
111
+ finish();
112
+ };
113
+ },
114
+ async close(deadlineMs) {
115
+ closing = true;
116
+ if (running.size === 0) return;
117
+ const deadline = waitUntilWallClock(Date.now() + deadlineMs);
118
+ try {
119
+ await Promise.race([new Promise((resolve) => waitingForIdle.push(resolve)), deadline.reached]);
120
+ } finally {
121
+ deadline.cancel();
122
+ }
123
+ }
124
+ };
71
125
  }
72
126
 
73
127
  // src/changes/encode.ts
@@ -181,7 +235,7 @@ var ChangesSession = class {
181
235
  }
182
236
  let message;
183
237
  try {
184
- message = subscriberMessageOf(decode(bytes));
238
+ message = subscriberMessageOf(decodeBoundedMessage(bytes));
185
239
  } catch {
186
240
  message = null;
187
241
  }
@@ -630,7 +684,7 @@ var FeedSession = class {
630
684
  }
631
685
  let message;
632
686
  try {
633
- message = deviceMessageOf(decode(bytes));
687
+ message = deviceMessageOf(decodeBoundedMessage(bytes));
634
688
  } catch {
635
689
  message = null;
636
690
  }
@@ -1254,2145 +1308,2163 @@ function startFeeds(engine, core, settings) {
1254
1308
  if (hosted.sync === void 0 && hosted.changes === void 0) return null;
1255
1309
  return { host: createFeedHost(hosted, settings.authorize), syncServing };
1256
1310
  }
1257
- function snapshotHistoryOf(raw) {
1258
- if (raw === null || raw === "false") return false;
1259
- return raw === "true" ? true : null;
1311
+
1312
+ // src/server/options-validation.ts
1313
+ function isLoopback(host) {
1314
+ return host === "127.0.0.1" || host === "::1" || host === "localhost" || host.startsWith("127.");
1260
1315
  }
1261
- function createAdminHandlers(deps) {
1262
- const { engine, tasks, limits } = deps;
1263
- async function checkpoint(ctx) {
1264
- try {
1265
- await engine.checkpoint(ctx.params[0]);
1266
- respondJson(ctx, { ok: true });
1267
- } catch (err) {
1268
- respondError(ctx, err);
1269
- }
1270
- }
1271
- async function snapshot(ctx) {
1272
- const queryHistory = snapshotHistoryOf(ctx.query.get("queryHistory"));
1273
- if (queryHistory === null) {
1274
- badRequest(ctx.res, 'Query parameter "queryHistory" accepts only true or false', {
1275
- value: ctx.query.get("queryHistory")
1276
- });
1277
- return;
1278
- }
1279
- try {
1280
- const archive = await engine.snapshot(ctx.params[0], queryHistory ? { queryHistory } : void 0);
1281
- if (ctx.abort.aborted) {
1282
- await archive.cancel();
1283
- return;
1284
- }
1285
- await sendStream(ctx.res, ctx.abort, archive, [["Content-Type", SNAPSHOT_ARCHIVE_CONTENT_TYPE]]);
1286
- } catch (err) {
1287
- respondError(ctx, err);
1288
- }
1316
+ function assertSecureBinding(host, options) {
1317
+ if (isLoopback(host) || options.onRequest || options.allowInsecure) return;
1318
+ throw new NarsilError(
1319
+ ErrorCodes.CONFIG_INVALID,
1320
+ `Refusing to start: bound to a non-loopback address ("${host}") with no onRequest auth hook. The server exposes destructive admin endpoints (restore, drop, clear, rebalance, optimize). Resolve one of: (1) set options.onRequest to authenticate requests, (2) bind to 127.0.0.1, or (3) set options.allowInsecure: true if this address is on a trusted private network.`,
1321
+ { host }
1322
+ );
1323
+ }
1324
+ var ENGINE_METHOD_SAMPLE = ["createIndex", "listIndexes", "insert", "query", "get", "shutdown"];
1325
+ var LIMITS_UNBOUNDED_AT_ZERO = [
1326
+ "maxBodyBytes",
1327
+ "maxImportBytes",
1328
+ "maxRestoreBytes",
1329
+ "maxLineBytes",
1330
+ "maxImportErrors",
1331
+ "maxConcurrentRequests",
1332
+ "maxConcurrentTasks"
1333
+ ];
1334
+ var COUNTED_LIMITS = [
1335
+ "importBatchSize",
1336
+ "maxResultWindow",
1337
+ "maxRerankWindow",
1338
+ "maxFetchDocuments",
1339
+ "maxTaskPageSize"
1340
+ ];
1341
+ function assertServerEngine(engine) {
1342
+ const candidate = engine;
1343
+ if (typeof candidate !== "object" || candidate === null) {
1344
+ throw new NarsilError(
1345
+ ErrorCodes.CONFIG_INVALID,
1346
+ "The first argument to createServer must be the engine that createNarsil returns, with the options second",
1347
+ { received: candidate === null ? "null" : typeof candidate }
1348
+ );
1289
1349
  }
1290
- function vectorMaintenance(ctx) {
1291
- try {
1292
- respondJson(ctx, { fields: engine.vectorMaintenanceStatus(ctx.params[0]) });
1293
- } catch (err) {
1294
- respondError(ctx, err);
1350
+ const members = candidate;
1351
+ const missing = ENGINE_METHOD_SAMPLE.filter((method) => typeof members[method] !== "function");
1352
+ if (missing.length === 0) return;
1353
+ throw new NarsilError(
1354
+ ErrorCodes.CONFIG_INVALID,
1355
+ `The first argument to createServer offers no ${missing.join(", ")}, so it is some value other than the engine that createNarsil returns`,
1356
+ { missing }
1357
+ );
1358
+ }
1359
+ function assertLimit(field, value, minimum) {
1360
+ if (value === void 0) return;
1361
+ if (Number.isInteger(value) && value >= minimum) return;
1362
+ throw new NarsilError(
1363
+ ErrorCodes.CONFIG_INVALID,
1364
+ `limits.${field} must be a whole number of at least ${minimum}; however, this configuration sets it to ${String(value)}`,
1365
+ { field, value, minimum }
1366
+ );
1367
+ }
1368
+ function assertFraction(field, value) {
1369
+ if (value === void 0) return;
1370
+ if (Number.isFinite(value) && value >= 0 && value <= MAX_BODY_BYTES_IN_FLIGHT_FRACTION) return;
1371
+ throw new NarsilError(
1372
+ ErrorCodes.CONFIG_INVALID,
1373
+ `limits.${field} must be a fraction from 0 to ${MAX_BODY_BYTES_IN_FLIGHT_FRACTION}; however, this configuration sets it to ${String(value)}`,
1374
+ { field, value, maximum: MAX_BODY_BYTES_IN_FLIGHT_FRACTION }
1375
+ );
1376
+ }
1377
+ function assertRerankWindowCap(value, engineCap) {
1378
+ if (value === void 0) return;
1379
+ const cap = Math.min(MAX_RERANK_WINDOW, engineCap ?? MAX_RERANK_WINDOW);
1380
+ if (value <= cap) return;
1381
+ throw new NarsilError(
1382
+ ErrorCodes.CONFIG_INVALID,
1383
+ `limits.maxRerankWindow must be at most ${cap}, the largest window that the engine accepts; however, this configuration sets it to ${value}`,
1384
+ { field: "maxRerankWindow", value, maximum: cap }
1385
+ );
1386
+ }
1387
+ function assertServerLimits(limits, engineRerankWindow) {
1388
+ if (limits === void 0) return;
1389
+ for (const field of LIMITS_UNBOUNDED_AT_ZERO) assertLimit(field, limits[field], MIN_UNBOUNDED_SERVER_LIMIT);
1390
+ for (const field of COUNTED_LIMITS) assertLimit(field, limits[field], MIN_COUNTED_SERVER_LIMIT);
1391
+ assertFraction("maxBodyBytesInFlightFraction", limits.maxBodyBytesInFlightFraction);
1392
+ assertRerankWindowCap(limits.maxRerankWindow, engineRerankWindow);
1393
+ }
1394
+ function assertQuerySuggestionsOptions(options) {
1395
+ if (options === void 0) return;
1396
+ const identify = typeof options === "object" && options !== null ? Reflect.get(options, "identify") : void 0;
1397
+ if (typeof identify === "function") return;
1398
+ throw new NarsilError(
1399
+ ErrorCodes.CONFIG_INVALID,
1400
+ "querySuggestions must carry an identify function that returns the person and the group of a request",
1401
+ { field: "querySuggestions.identify" }
1402
+ );
1403
+ }
1404
+ function assertAllowRegex(allowRegex) {
1405
+ if (allowRegex === void 0 || typeof allowRegex === "boolean") return;
1406
+ throw new NarsilError(
1407
+ ErrorCodes.CONFIG_INVALID,
1408
+ `allowRegex must be true or false; however, this configuration sets it to ${String(allowRegex)}`,
1409
+ { field: "allowRegex", value: String(allowRegex) }
1410
+ );
1411
+ }
1412
+
1413
+ // package.json
1414
+ var version = "0.4.2-next.273";
1415
+
1416
+ // src/server/package-version.ts
1417
+ function publishedPackageVersion() {
1418
+ return version;
1419
+ }
1420
+
1421
+ // src/server/request-threads/close-deadline.ts
1422
+ async function closeThreadsByDeadline(words, closed, deadline) {
1423
+ const drained = waitUntilWallClock(deadline);
1424
+ try {
1425
+ await Promise.race([closed, drained.reached]);
1426
+ closeEveryThreadAtExit(words, 0);
1427
+ await Promise.all([...words].map((word) => threadCloseAcknowledgedWithin(word, SERVER_CLOSE_ACKNOWLEDGE_GRACE_MS)));
1428
+ const unacknowledged = [...words].filter((word) => !threadCloseAcknowledged(word)).length;
1429
+ if (unacknowledged > 0) {
1430
+ throw new NarsilError(
1431
+ ErrorCodes.WORKER_TIMEOUT,
1432
+ `Request threads that did not acknowledge closing their connections: ${unacknowledged}; server close deadline: ${SERVER_CLOSE_DRAIN_DEADLINE_MS}ms`,
1433
+ { threads: unacknowledged, deadlineMs: SERVER_CLOSE_DRAIN_DEADLINE_MS }
1434
+ );
1295
1435
  }
1436
+ } finally {
1437
+ drained.cancel();
1296
1438
  }
1297
- async function compact(ctx) {
1298
- const body = parseJsonOptional(ctx);
1299
- if (!body) return;
1300
- try {
1301
- await engine.compactVectors(ctx.params[0], body.field);
1302
- respondJson(ctx, { ok: true });
1303
- } catch (err) {
1304
- respondError(ctx, err);
1305
- }
1439
+ }
1440
+
1441
+ // src/server/request-threads/relay-answers.ts
1442
+ async function answerAuthorize(authorize, request) {
1443
+ if (authorize === void 0) {
1444
+ return { type: "authorized", requestId: request.requestId, denial: null, hookError: false, expiresAt: null };
1306
1445
  }
1307
- async function partitionConfig(ctx) {
1308
- const body = parseJsonOptional(ctx);
1309
- if (!body) return;
1310
- try {
1311
- await engine.updatePartitionConfig(ctx.params[0], body);
1312
- respondJson(ctx, { ok: true });
1313
- } catch (err) {
1314
- respondError(ctx, err);
1315
- }
1446
+ const authorization = await authorize(request.context);
1447
+ if (authorization.allowed) {
1448
+ return {
1449
+ type: "authorized",
1450
+ requestId: request.requestId,
1451
+ denial: null,
1452
+ hookError: false,
1453
+ expiresAt: authorization.expiresAt
1454
+ };
1316
1455
  }
1317
- async function readSynonyms(ctx) {
1318
- try {
1319
- respondJson(ctx, { synonyms: await engine.getSynonyms(ctx.params[0]) });
1320
- } catch (err) {
1321
- respondError(ctx, err);
1322
- }
1456
+ return {
1457
+ type: "authorized",
1458
+ requestId: request.requestId,
1459
+ denial: authorization.denial,
1460
+ hookError: authorization.denial === null,
1461
+ expiresAt: null
1462
+ };
1463
+ }
1464
+ async function answerEmbed(core, request, signal) {
1465
+ try {
1466
+ const entry = core.requireIndex(request.indexName);
1467
+ const resolved = await resolveVectorText(
1468
+ { vector: request.vector },
1469
+ entry.embeddingAdapter,
1470
+ signal === void 0 ? core.abortController.signal : AbortSignal.any([core.abortController.signal, signal]),
1471
+ entry.embeddingAdapterName
1472
+ );
1473
+ const value = resolved.vector?.value;
1474
+ return {
1475
+ type: "embedded",
1476
+ requestId: request.requestId,
1477
+ value: value === void 0 ? null : Array.from(value),
1478
+ error: null
1479
+ };
1480
+ } catch (err) {
1481
+ if (!isNarsilError(err)) console.warn("Embedding a query for a request thread failed:", err);
1482
+ return { type: "embedded", requestId: request.requestId, value: null, error: toHttpError(err).body };
1323
1483
  }
1324
- async function replaceSynonyms(ctx) {
1325
- const body = parseJson(ctx);
1326
- if (body === void 0) return;
1327
- if (body === null || typeof body !== "object" || Array.isArray(body) || !Object.hasOwn(body, "synonyms")) {
1328
- if (!ctx.abort.aborted) badRequest(ctx.res, 'Send the synonym set as { "synonyms": [ ...rules ] }');
1329
- return;
1330
- }
1331
- try {
1332
- await engine.updateSynonyms(ctx.params[0], Reflect.get(body, "synonyms"));
1333
- respondJson(ctx, { ok: true });
1334
- } catch (err) {
1335
- respondError(ctx, err);
1336
- }
1337
- }
1338
- async function updateBm25(ctx) {
1339
- const body = parseJson(ctx);
1340
- if (body === void 0) return;
1341
- try {
1342
- await engine.updateBM25(ctx.params[0], body);
1343
- respondJson(ctx, { ok: true });
1344
- } catch (err) {
1345
- respondError(ctx, err);
1346
- }
1347
- }
1348
- async function memory(ctx) {
1349
- try {
1350
- const stats = await engine.getMemoryStats();
1351
- const response = { ...stats, requestThreads: deps.requestThreadCount() };
1352
- respondJson(ctx, response);
1353
- } catch (err) {
1354
- respondError(ctx, err);
1355
- }
1356
- }
1357
- async function requireIndexExists(name) {
1358
- await indexReadsOf(engine).getStats(name);
1359
- }
1360
- async function optimize(ctx) {
1361
- const body = parseJsonOptional(ctx);
1362
- if (!body) return;
1363
- const name = ctx.params[0];
1364
- try {
1365
- await requireIndexExists(name);
1366
- } catch (err) {
1367
- respondError(ctx, err);
1368
- return;
1369
- }
1370
- const record = await tasks.start("optimizeVectors", name, () => engine.optimizeVectors(name, body.field));
1371
- respondJson(ctx, record, 202);
1372
- }
1373
- async function rebuildAnalysis(ctx) {
1374
- const name = ctx.params[0];
1375
- try {
1376
- await requireIndexExists(name);
1377
- } catch (err) {
1378
- respondError(ctx, err);
1379
- return;
1380
- }
1381
- const record = await tasks.start("rebuildAnalysis", name, () => engine.rebuildAnalysis(name));
1382
- respondJson(ctx, record, 202);
1383
- }
1384
- async function changeAnalysis(ctx) {
1385
- const body = parseJsonOptional(ctx);
1386
- if (!body) return;
1387
- const failure = validateChangeAnalysis(body);
1388
- if (failure) {
1389
- rejectInvalid(ctx, failure);
1390
- return;
1391
- }
1392
- const name = ctx.params[0];
1393
- try {
1394
- await requireIndexExists(name);
1395
- } catch (err) {
1396
- respondError(ctx, err);
1397
- return;
1398
- }
1399
- const record = await tasks.start("changeAnalysis", name, () => engine.changeAnalysis(name, body));
1400
- respondJson(ctx, record, 202);
1401
- }
1402
- async function rebalance(ctx) {
1403
- const body = parseJsonOptional(ctx);
1404
- if (!body) return;
1405
- const failure = validateRebalance(body);
1406
- if (failure) {
1407
- rejectInvalid(ctx, failure);
1408
- return;
1409
- }
1410
- const target = body.targetPartitionCount;
1411
- const name = ctx.params[0];
1412
- try {
1413
- await requireIndexExists(name);
1414
- } catch (err) {
1415
- respondError(ctx, err);
1416
- return;
1417
- }
1418
- const record = await tasks.start("rebalance", name, () => engine.rebalance(name, target));
1419
- respondJson(ctx, record, 202);
1420
- }
1421
- async function restore(ctx) {
1422
- const spooled = ctx.spooledBody;
1423
- if (spooled === null || spooled.bytes === 0) {
1424
- await spooled?.remove();
1425
- sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
1426
- return;
1427
- }
1428
- const name = ctx.params[0];
1429
- try {
1430
- const record = await tasks.start("restore", name, async () => {
1431
- try {
1432
- await engine.restore(name, createReadStream(spooled.path));
1433
- } finally {
1434
- await spooled.remove();
1435
- }
1436
- });
1437
- respondJson(ctx, record, 202);
1438
- } catch (err) {
1439
- await spooled.remove();
1440
- respondError(ctx, err);
1441
- }
1442
- }
1443
- async function getTask(ctx) {
1444
- const record = await tasks.get(ctx.params[0]);
1445
- if (!record) {
1446
- sendError(ctx.res, 404, ServerErrorCodes.TASK_NOT_FOUND, `Task "${ctx.params[0]}" not found`);
1447
- return;
1448
- }
1449
- respondJson(ctx, record);
1450
- }
1451
- async function listTasks(ctx) {
1452
- const parsed = parseTaskListQuery(ctx.query, limits.maxTaskPageSize);
1453
- if ("failure" in parsed) {
1454
- rejectInvalid(ctx, parsed.failure);
1455
- return;
1456
- }
1457
- respondJson(ctx, await tasks.list(parsed.query));
1458
- }
1459
- async function cancelTask(ctx) {
1460
- const taskId = ctx.params[0];
1461
- const { outcome, record } = await tasks.cancel(taskId);
1462
- if (outcome === "not-found") {
1463
- sendError(ctx.res, 404, ServerErrorCodes.TASK_NOT_FOUND, `Task "${taskId}" not found`);
1464
- return;
1465
- }
1466
- if (outcome === "already-finished") {
1467
- sendError(
1468
- ctx.res,
1469
- 409,
1470
- ServerErrorCodes.TASK_NOT_CANCELLABLE,
1471
- `Task "${taskId}" has already finished`,
1472
- record === null ? void 0 : { status: record.status }
1473
- );
1474
- return;
1475
- }
1476
- if (outcome === "owned-by-another-instance") {
1477
- sendError(
1478
- ctx.res,
1479
- 409,
1480
- ServerErrorCodes.TASK_OWNED_BY_ANOTHER_INSTANCE,
1481
- `Task "${taskId}" runs on another server instance, which is the only one that can stop it`,
1482
- record === null ? void 0 : { owner: record.owner }
1483
- );
1484
- return;
1485
- }
1486
- respondJson(ctx, record, 202);
1484
+ }
1485
+ async function answerRerank(core, request, signal) {
1486
+ try {
1487
+ const { rerankOnHost } = await loadRerankRuntime();
1488
+ const scores = await rerankOnHost(core, request.adapter, request.query, request.documents, signal);
1489
+ return { type: "reranked", requestId: request.requestId, scores, error: null };
1490
+ } catch (err) {
1491
+ if (!isNarsilError(err)) console.warn("Reranking a query for a request thread failed:", err);
1492
+ return { type: "reranked", requestId: request.requestId, scores: null, error: toHttpError(err).body };
1487
1493
  }
1494
+ }
1495
+ async function answerAwaitWrites(core, request) {
1496
+ await core.orchestrator.awaitWrites(request.indexName).catch(() => void 0);
1497
+ return { type: "writes-applied", requestId: request.requestId };
1498
+ }
1499
+ async function answerFeedAdmit(feeds, request) {
1500
+ if (feeds !== null) return feeds.admit(request);
1488
1501
  return {
1489
- checkpoint,
1490
- snapshot,
1491
- vectorMaintenance,
1492
- compact,
1493
- partitionConfig,
1494
- readSynonyms,
1495
- replaceSynonyms,
1496
- updateBm25,
1497
- memory,
1498
- optimize,
1499
- rebalance,
1500
- rebuildAnalysis,
1501
- changeAnalysis,
1502
- restore,
1503
- getTask,
1504
- listTasks,
1505
- cancelTask
1502
+ type: "feed-admitted",
1503
+ requestId: request.requestId,
1504
+ feedId: null,
1505
+ error: { code: ServerErrorCodes.NOT_FOUND, message: "Route not found" }
1506
1506
  };
1507
1507
  }
1508
1508
 
1509
- // src/server/handlers/cluster.ts
1510
- function requireCluster(deps) {
1511
- if (deps.cluster === void 0) {
1512
- throw new NarsilError(
1513
- ErrorCodes.CLUSTER_OPERATION_UNSUPPORTED,
1514
- "This server fronts a single engine, so it reports no cluster",
1515
- {}
1516
- );
1517
- }
1518
- return deps.cluster;
1509
+ // src/server/request-threads/host.ts
1510
+ function threadClosed(port, clearSlots) {
1511
+ return new Promise((resolve) => {
1512
+ port.once("close", () => {
1513
+ clearSlots();
1514
+ resolve();
1515
+ });
1516
+ });
1519
1517
  }
1520
- function describeNode(cluster) {
1521
- return { ...cluster.getNodeInfo(), readiness: cluster.getReadiness(), isController: cluster.isControllerActive() };
1518
+ var RELAYED_TYPES = /* @__PURE__ */ new Set([
1519
+ "request",
1520
+ "authorize",
1521
+ "embed",
1522
+ "rerank",
1523
+ "await-writes",
1524
+ "stream-credit",
1525
+ "stream-cancel",
1526
+ "feed-admit"
1527
+ ]);
1528
+ function isRelayRequest(message) {
1529
+ if (typeof message !== "object" || message === null) return false;
1530
+ const { type, requestId } = message;
1531
+ if (type === "touch" || type === "leaving") return true;
1532
+ return typeof requestId === "number" && typeof type === "string" && RELAYED_TYPES.has(type);
1522
1533
  }
1523
- function malformedIndexName(indexName) {
1524
- try {
1525
- validateIndexName(indexName);
1526
- return null;
1527
- } catch (error) {
1528
- return error instanceof Error ? error.message : String(error);
1529
- }
1534
+ function touchIntervalFor(copyIdleTimeoutMs) {
1535
+ return Math.min(MAX_INDEX_TOUCH_INTERVAL_MS, Math.floor(copyIdleTimeoutMs / INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT));
1530
1536
  }
1531
- function createClusterHandlers(deps) {
1532
- async function topology(ctx) {
1533
- try {
1534
- const cluster = requireCluster(deps);
1535
- const [nodes, controllerNodeId] = await Promise.all([cluster.listNodes(), cluster.getControllerNodeId()]);
1536
- respondJson(ctx, {
1537
- node: describeNode(cluster),
1538
- controllerNodeId,
1539
- nodes: nodes.map((node) => ({
1540
- nodeId: node.nodeId,
1541
- address: node.address,
1542
- roles: [...node.roles],
1543
- memory: node.memory ?? null
1544
- }))
1545
- });
1546
- } catch (err) {
1547
- respondError(ctx, err);
1548
- }
1537
+ async function startRequestThreads(options) {
1538
+ const found = acceptorOf(options.app);
1539
+ if (found === null) return null;
1540
+ const acceptor = found;
1541
+ const { core, handlers, authorize } = options;
1542
+ const searchHooks = core.pluginRegistry.hasHooks("beforeSearch") || core.pluginRegistry.hasHooks("afterSearch");
1543
+ const touchIntervalMs = touchIntervalFor(core.orchestrator.copyIdleTimeoutMs());
1544
+ const threads = /* @__PURE__ */ new Map();
1545
+ const slotUsers = /* @__PURE__ */ new Map();
1546
+ const longQueriesInFlight = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT);
1547
+ let longQueryThread = null;
1548
+ const closeAtExit = /* @__PURE__ */ new Set();
1549
+ const retiring = /* @__PURE__ */ new Set();
1550
+ const serverClose = createServerCloseSignal();
1551
+ let stopped = false;
1552
+ function closeThreadsAtExit() {
1553
+ closeEveryThreadAtExit(closeAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS);
1549
1554
  }
1550
- async function allocation(ctx) {
1551
- try {
1552
- const cluster = requireCluster(deps);
1553
- const indexName = ctx.params[0] ?? "";
1554
- const malformed = malformedIndexName(indexName);
1555
- if (malformed !== null) {
1556
- badRequest(ctx.res, malformed, { indexName });
1557
- return;
1558
- }
1559
- const table = await cluster.getAllocation(indexName);
1560
- if (table === null) {
1561
- respondJson(ctx, {
1562
- indexName,
1563
- allocated: false,
1564
- version: null,
1565
- partitions: []
1566
- });
1567
- return;
1568
- }
1569
- const partitions = [];
1570
- for (const [partitionId, assignment] of table.assignments) {
1571
- partitions.push({
1572
- partitionId,
1573
- state: assignment.state,
1574
- primary: assignment.primary,
1575
- primaryTerm: assignment.primaryTerm,
1576
- commitPoint: assignment.commitPoint,
1577
- replicas: [...assignment.replicas],
1578
- inSyncSet: [...assignment.inSyncSet],
1579
- lastHolders: [...assignment.lastHolders ?? []],
1580
- unassignedReason: assignment.unassignedReason ?? null
1581
- });
1555
+ function releaseDescriptor(thread) {
1556
+ thread.feeds?.closeAll();
1557
+ if (thread.descriptor !== null) {
1558
+ try {
1559
+ acceptor.removeChildAppDescriptor(thread.descriptor);
1560
+ } catch {
1582
1561
  }
1583
- partitions.sort((left, right) => left.partitionId - right.partitionId);
1584
- respondJson(ctx, {
1585
- indexName,
1586
- allocated: partitions.length > 0,
1587
- version: table.version,
1588
- partitions
1589
- });
1590
- } catch (err) {
1591
- respondError(ctx, err);
1562
+ thread.descriptor = null;
1592
1563
  }
1564
+ confirmDeparture(thread.departure);
1593
1565
  }
1594
- async function replicationFactor(ctx) {
1595
- try {
1596
- const cluster = requireCluster(deps);
1597
- const indexName = ctx.params[0] ?? "";
1598
- const malformed = malformedIndexName(indexName);
1599
- if (malformed !== null) {
1600
- badRequest(ctx.res, malformed, { indexName });
1566
+ function relayFrom(thread) {
1567
+ const { port } = thread;
1568
+ const streams = /* @__PURE__ */ new Map();
1569
+ const calls = /* @__PURE__ */ new Map();
1570
+ port.on("close", () => {
1571
+ thread.feeds?.closeAll();
1572
+ for (const stream of streams.values()) stream.cancel();
1573
+ for (const call of calls.values()) call.abort();
1574
+ calls.clear();
1575
+ });
1576
+ port.on("message", (message) => {
1577
+ if (isThreadFeedNote(message)) {
1578
+ thread.feeds?.handle(message);
1601
1579
  return;
1602
1580
  }
1603
- const body = parseJson(ctx);
1604
- if (body === void 0) return;
1605
- const factor = isRecord(body) ? body.replicationFactor : void 0;
1606
- if (typeof factor !== "number") {
1607
- badRequest(ctx.res, "The body must carry a numeric replicationFactor", { indexName });
1581
+ if (!isRelayRequest(message)) return;
1582
+ if (message.type === "leaving") {
1583
+ releaseDescriptor(thread);
1608
1584
  return;
1609
1585
  }
1610
- await cluster.setReplicationFactor(indexName, factor);
1611
- respondJson(ctx, { indexName, replicationFactor: factor });
1612
- } catch (err) {
1613
- respondError(ctx, err);
1614
- }
1615
- }
1616
- async function health(ctx) {
1617
- try {
1618
- respondJson(ctx, await requireCluster(deps).getHealth());
1619
- } catch (err) {
1620
- respondError(ctx, err);
1621
- }
1622
- }
1623
- return { topology, allocation, replicationFactor, health };
1624
- }
1625
-
1626
- // src/server/handlers/backups.ts
1627
- function stepsOf(context) {
1628
- return { onProgress: (progress) => context.reportSteps({ done: progress.done, total: progress.total }) };
1629
- }
1630
- function restoreOptionsOf(body) {
1631
- const { indexName } = body;
1632
- if (typeof indexName !== "string") return 'The body must carry the new index name as a string in "indexName"';
1633
- const placement = readRestorePlacement(body);
1634
- return typeof placement === "string" ? placement : { indexName, ...placement };
1635
- }
1636
- function readBody(ctx) {
1637
- const body = parseJson(ctx);
1638
- if (body === void 0) return void 0;
1639
- if (!isRecord(body)) {
1640
- badRequest(ctx.res, "The body must be a JSON object");
1641
- return void 0;
1642
- }
1643
- return body;
1644
- }
1645
- function createBackupHandlers(deps) {
1646
- async function create(ctx) {
1647
- try {
1648
- const cluster = requireCluster(deps);
1649
- const indexName = ctx.params[0] ?? "";
1650
- const body = readBody(ctx);
1651
- if (body === void 0) return;
1652
- if (typeof body.backup !== "string") {
1653
- badRequest(ctx.res, 'The body must carry the backup name as a string in "backup"', { indexName });
1586
+ if (message.type === "touch") {
1587
+ core.orchestrator.noteAccess(message.indexName);
1654
1588
  return;
1655
1589
  }
1656
- if (body.queryHistory !== void 0 && typeof body.queryHistory !== "boolean") {
1657
- badRequest(ctx.res, 'The body accepts only true or false in "queryHistory"', { indexName });
1590
+ if (message.type === "stream-credit") {
1591
+ streams.get(message.requestId)?.credit();
1658
1592
  return;
1659
1593
  }
1660
- const backupName = validateBackupName(body.backup);
1661
- const queryHistory = body.queryHistory === true;
1662
- const record = await deps.tasks.start(
1663
- "backup",
1664
- indexName,
1665
- async (context) => {
1666
- await cluster.createBackup(indexName, backupName, { ...stepsOf(context), queryHistory });
1667
- },
1668
- void 0,
1669
- backupName
1670
- );
1671
- respondJson(ctx, record, 202);
1672
- } catch (err) {
1673
- respondError(ctx, err);
1674
- }
1675
- }
1676
- async function list(ctx) {
1677
- try {
1678
- const cluster = requireCluster(deps);
1679
- respondJson(ctx, { backups: await cluster.listBackups() });
1680
- } catch (err) {
1681
- respondError(ctx, err);
1682
- }
1683
- }
1684
- async function restore(ctx) {
1685
- try {
1686
- const cluster = requireCluster(deps);
1687
- const backupName = validateBackupName(ctx.params[0]);
1688
- const body = readBody(ctx);
1689
- if (body === void 0) return;
1690
- const options = restoreOptionsOf(body);
1691
- if (typeof options === "string") {
1692
- badRequest(ctx.res, options, { backupName });
1594
+ if (message.type === "stream-cancel") {
1595
+ streams.get(message.requestId)?.cancel();
1596
+ calls.get(message.requestId)?.abort();
1597
+ calls.delete(message.requestId);
1693
1598
  return;
1694
1599
  }
1695
- const record = await deps.tasks.start(
1696
- "backupRestore",
1697
- options.indexName,
1698
- async (context) => {
1699
- await cluster.restoreBackup(backupName, { ...options, ...stepsOf(context) });
1600
+ const call = message.type === "embed" || message.type === "rerank" ? new AbortController() : void 0;
1601
+ if (call !== void 0) calls.set(message.requestId, call);
1602
+ const reply = message.type === "request" ? answerRequest(handlers[message.route], message, port, streams) : message.type === "authorize" ? answerAuthorize(authorize, message) : message.type === "feed-admit" ? answerFeedAdmit(thread.feeds, message) : message.type === "embed" ? answerEmbed(core, message, call?.signal) : message.type === "await-writes" ? answerAwaitWrites(core, message) : answerRerank(core, message, call?.signal);
1603
+ void reply.then(
1604
+ (answer) => {
1605
+ calls.delete(message.requestId);
1606
+ if (answer !== null) port.postMessage(answer);
1700
1607
  },
1701
- void 0,
1702
- backupName
1608
+ () => {
1609
+ calls.delete(message.requestId);
1610
+ }
1703
1611
  );
1704
- respondJson(ctx, record, 202);
1705
- } catch (err) {
1706
- respondError(ctx, err);
1707
- }
1612
+ });
1708
1613
  }
1709
- async function remove(ctx) {
1710
- try {
1711
- const cluster = requireCluster(deps);
1712
- const backupName = validateBackupName(ctx.params[0]);
1713
- const record = await deps.tasks.start(
1714
- "backupDelete",
1715
- "",
1716
- async (context) => {
1717
- await cluster.deleteBackup(backupName, stepsOf(context));
1718
- },
1719
- void 0,
1720
- backupName
1721
- );
1722
- respondJson(ctx, record, 202);
1723
- } catch (err) {
1724
- respondError(ctx, err);
1725
- }
1614
+ function connectToLongQueries(thread) {
1615
+ if (longQueryThread === null) return;
1616
+ const channel = new MessageChannel();
1617
+ thread.port.postMessage({ type: "long-query-port", port: channel.port1 }, [channel.port1]);
1618
+ longQueryThread.port.postMessage({ type: "long-query-port", port: channel.port2 }, [channel.port2]);
1726
1619
  }
1727
- return { create, list, restore, remove };
1728
- }
1729
-
1730
- // src/server/handlers/capabilities.ts
1731
- function createCapabilitiesHandler() {
1732
- function report(ctx) {
1733
- respondJson(ctx, { capabilities: [...SERVER_CAPABILITIES] });
1620
+ function dropLongQueryThread() {
1621
+ if (longQueryThread === null) return;
1622
+ longQueryThread.port.close();
1623
+ longQueryThread = null;
1734
1624
  }
1735
- return { report };
1736
- }
1737
- function importResultOf(result, maxErrors) {
1738
- const errors = result.refused.slice(0, maxErrors);
1739
- return {
1740
- indexed: result.imported,
1741
- failed: result.refusedCount,
1742
- errors,
1743
- errorsTruncated: result.refusedCount > errors.length
1744
- };
1745
- }
1746
- function createDocumentExportHandlers(deps) {
1747
- const { engine, tasks, limits } = deps;
1748
- async function exportDocuments(ctx) {
1749
- try {
1750
- const lines = await engine.exportDocuments(ctx.params[0]);
1751
- if (ctx.abort.aborted) {
1752
- await lines.cancel();
1753
- return;
1754
- }
1755
- await sendStream(ctx.res, ctx.abort, lines, [["Content-Type", DOCUMENT_EXPORT_CONTENT_TYPE]]);
1756
- } catch (err) {
1757
- respondPreparingError(ctx, err, ErrorCodes.CHANGES_EXPORT_PREPARING);
1758
- }
1625
+ function retireThread(workerId, thread, refuseNewRequests = false) {
1626
+ releaseDescriptor(thread);
1627
+ thread.port.unref();
1628
+ retiring.add(thread.closed);
1629
+ void thread.closed.then(() => retiring.delete(thread.closed));
1630
+ void executeUntilAnswered(thread.executor, {
1631
+ type: "stopServing",
1632
+ requestId: `stop-serving-${workerId}`,
1633
+ refuseNewRequests
1634
+ }).catch(() => void 0);
1635
+ return thread.closed;
1759
1636
  }
1760
- async function importExport(ctx) {
1761
- const spooled = ctx.spooledBody;
1762
- if (spooled === null || spooled.bytes === 0) {
1763
- await spooled?.remove();
1764
- sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
1765
- return;
1637
+ const listener = {
1638
+ async onLongQueryWorkerReady(workerId, executor) {
1639
+ if (stopped) return;
1640
+ dropLongQueryThread();
1641
+ const channel = new MessageChannel();
1642
+ const relaying = {
1643
+ feeds: null,
1644
+ descriptor: null,
1645
+ port: channel.port2,
1646
+ executor,
1647
+ departure: createDepartureWord(),
1648
+ closeAtExit: createCloseAtExitWords(),
1649
+ closed: new Promise((resolve) => channel.port2.once("close", resolve))
1650
+ };
1651
+ relayFrom(relaying);
1652
+ try {
1653
+ await executor.execute(
1654
+ {
1655
+ type: "serveLongQueries",
1656
+ requestId: `serve-long-queries-${workerId}`,
1657
+ settings: {
1658
+ port: channel.port1,
1659
+ touchIntervalMs,
1660
+ limits: options.limits,
1661
+ patternWorkCap: options.core.patternWorkCap,
1662
+ maxRerankWindow: options.core.rerankSettings.maxWindow,
1663
+ allowRegex: options.allowRegex
1664
+ }
1665
+ },
1666
+ [channel.port1]
1667
+ );
1668
+ } catch (err) {
1669
+ channel.port2.close();
1670
+ console.warn(`Worker ${workerId} could not start answering long queries:`, err);
1671
+ return;
1672
+ }
1673
+ if (stopped) {
1674
+ channel.port2.close();
1675
+ return;
1676
+ }
1677
+ longQueryThread = { workerId, port: channel.port2, executor };
1678
+ for (const thread of threads.values()) connectToLongQueries(thread);
1679
+ },
1680
+ async onWorkerReady(workerId, executor) {
1681
+ if (stopped) return;
1682
+ const channel = new MessageChannel();
1683
+ const slot = gateSlotOfWorker(workerId);
1684
+ slotUsers.set(slot, (slotUsers.get(slot) ?? 0) + 1);
1685
+ const thread = {
1686
+ feeds: options.feeds?.forThread(channel.port2) ?? null,
1687
+ descriptor: null,
1688
+ port: channel.port2,
1689
+ executor,
1690
+ departure: createDepartureWord(),
1691
+ closeAtExit: createCloseAtExitWords(),
1692
+ closed: threadClosed(channel.port2, () => {
1693
+ const remaining = (slotUsers.get(slot) ?? 1) - 1;
1694
+ if (remaining > 0) {
1695
+ slotUsers.set(slot, remaining);
1696
+ return;
1697
+ }
1698
+ slotUsers.delete(slot);
1699
+ clearGateSlot(options.gate, slot);
1700
+ clearBodyBudgetSlot(options.bodyBudget, slot);
1701
+ })
1702
+ };
1703
+ relayFrom(thread);
1704
+ closeAtExit.add(thread.closeAtExit);
1705
+ void thread.closed.then(() => closeAtExit.delete(thread.closeAtExit));
1706
+ const longQueries = longQueryThread;
1707
+ const longQueryChannel = longQueries === null ? null : new MessageChannel();
1708
+ const transfer = longQueryChannel === null ? [channel.port1] : [channel.port1, longQueryChannel.port1];
1709
+ let result;
1710
+ try {
1711
+ result = await executor.execute(
1712
+ {
1713
+ type: "serveRequests",
1714
+ requestId: `serve-${workerId}`,
1715
+ settings: {
1716
+ port: channel.port1,
1717
+ departure: thread.departure.buffer,
1718
+ serverClose: serverClose.buffer,
1719
+ closeAtExit: thread.closeAtExit.buffer,
1720
+ gate: options.gate,
1721
+ gateSlot: gateSlotOfWorker(workerId),
1722
+ maxConcurrentRequests: options.limits.maxConcurrentRequests,
1723
+ bodyBudget: options.bodyBudget,
1724
+ touchIntervalMs,
1725
+ cors: options.cors,
1726
+ limits: options.limits,
1727
+ build: options.build,
1728
+ authorizes: authorize !== void 0,
1729
+ searchHooks,
1730
+ patternWorkCap: options.core.patternWorkCap,
1731
+ maxRerankWindow: options.core.rerankSettings.maxWindow,
1732
+ allowRegex: options.allowRegex,
1733
+ feeds: Object.keys(options.feeds?.gateways ?? {}),
1734
+ longQueriesInFlight,
1735
+ longQueryPort: longQueryChannel?.port1 ?? null
1736
+ }
1737
+ },
1738
+ transfer
1739
+ );
1740
+ } catch (err) {
1741
+ closeAtExit.delete(thread.closeAtExit);
1742
+ channel.port2.close();
1743
+ longQueryChannel?.port2.close();
1744
+ console.warn(`Worker ${workerId} could not start taking requests:`, err);
1745
+ return;
1746
+ }
1747
+ thread.descriptor = result.descriptor;
1748
+ if (stopped || departureConfirmed(thread.departure)) {
1749
+ longQueryChannel?.port2.close();
1750
+ await retireThread(workerId, thread);
1751
+ return;
1752
+ }
1753
+ if (longQueryChannel !== null && longQueries !== null && longQueryThread === longQueries) {
1754
+ longQueries.port.postMessage({ type: "long-query-port", port: longQueryChannel.port2 }, [
1755
+ longQueryChannel.port2
1756
+ ]);
1757
+ } else {
1758
+ longQueryChannel?.port2.close();
1759
+ connectToLongQueries(thread);
1760
+ }
1761
+ acceptor.addChildAppDescriptor(result.descriptor);
1762
+ threads.set(workerId, thread);
1763
+ },
1764
+ onWorkerGone(workerId) {
1765
+ if (longQueryThread?.workerId === workerId) {
1766
+ dropLongQueryThread();
1767
+ return;
1768
+ }
1769
+ const thread = threads.get(workerId);
1770
+ if (thread === void 0) return;
1771
+ threads.delete(workerId);
1772
+ void retireThread(workerId, thread);
1766
1773
  }
1767
- const name = ctx.params[0];
1768
- try {
1769
- const record = await tasks.start("importExport", name, async (context) => {
1770
- try {
1771
- const result = await engine.importDocumentExport(name, createReadStream(spooled.path));
1772
- context.reportResult(importResultOf(result, limits.maxImportErrors));
1773
- } finally {
1774
- await spooled.remove();
1775
- }
1774
+ };
1775
+ process.on("exit", closeThreadsAtExit);
1776
+ await core.orchestrator.serveRequestsOnWorkers(listener);
1777
+ return {
1778
+ get threadCount() {
1779
+ return threads.size;
1780
+ },
1781
+ async stop() {
1782
+ const deadline = signalServerClose(serverClose, SERVER_CLOSE_DRAIN_DEADLINE_MS);
1783
+ stopped = true;
1784
+ process.off("exit", closeThreadsAtExit);
1785
+ const stopping = [...threads.entries()].map(([workerId, thread]) => {
1786
+ threads.delete(workerId);
1787
+ return retireThread(workerId, thread, true);
1776
1788
  });
1777
- respondJson(ctx, record, 202);
1778
- } catch (err) {
1779
- await spooled.remove();
1780
- respondError(ctx, err);
1789
+ core.orchestrator.stopRequestThreads();
1790
+ const longQueries = longQueryThread;
1791
+ dropLongQueryThread();
1792
+ if (longQueries !== null) {
1793
+ stopping.push(
1794
+ longQueries.executor.execute({ type: "stopServing", requestId: `stop-long-queries-${longQueries.workerId}` }).then(
1795
+ () => void 0,
1796
+ () => void 0
1797
+ )
1798
+ );
1799
+ }
1800
+ await closeThreadsByDeadline(closeAtExit, Promise.all([...stopping, ...retiring]), deadline);
1781
1801
  }
1802
+ };
1803
+ }
1804
+
1805
+ // src/server/sync/options.ts
1806
+ var FEED_OPTION_FIELDS = ["windowBytes", "maxConnections", "maxCatchUps", "maxBufferedBytes"];
1807
+ var FEED_REFUSALS = {
1808
+ sync: "Refusing to start: the device sync routes are on with no onRequest hook. A device receives the whole index, so set options.onRequest to admit only the user whose documents the index holds. allowInsecure does not lift this refusal.",
1809
+ changes: "Refusing to start: the change feed is on with no onRequest hook. A subscriber receives every document of the index, so set options.onRequest to admit only the services that may read the whole index. allowInsecure does not lift this refusal."
1810
+ };
1811
+ function assertCount(feed, field, value) {
1812
+ if (value === void 0) return;
1813
+ if (typeof value === "number" && Number.isSafeInteger(value) && value >= 1) return;
1814
+ throw new NarsilError(
1815
+ ErrorCodes.CONFIG_INVALID,
1816
+ `${feed}.${field} must be a whole number of at least 1; however, this configuration sets it to ${String(value)}`,
1817
+ { field, value: String(value) }
1818
+ );
1819
+ }
1820
+ function resolveFeedOptions(feed, value) {
1821
+ if (value === void 0 || value === false) return null;
1822
+ if (value !== true && (typeof value !== "object" || value === null)) {
1823
+ throw new NarsilError(
1824
+ ErrorCodes.CONFIG_INVALID,
1825
+ `${feed} must be true, false, or an object of settings; however, this configuration sets it to ${String(value)}`,
1826
+ { field: feed, value: String(value) }
1827
+ );
1782
1828
  }
1783
- return { exportDocuments, importExport };
1829
+ const settings = value === true ? {} : value;
1830
+ for (const field of FEED_OPTION_FIELDS) assertCount(feed, field, settings[field]);
1831
+ return {
1832
+ windowBytes: settings.windowBytes ?? DEFAULT_FEED_WINDOW_BYTES,
1833
+ maxConnections: settings.maxConnections ?? DEFAULT_MAX_FEED_CONNECTIONS,
1834
+ maxCatchUps: settings.maxCatchUps ?? DEFAULT_MAX_FEED_CATCH_UPS,
1835
+ maxBufferedBytes: settings.maxBufferedBytes ?? DEFAULT_MAX_FEED_BUFFERED_BYTES
1836
+ };
1837
+ }
1838
+ function assertFeedHasAuthentication(feed, resolved, options) {
1839
+ if (resolved === null || options.onRequest !== void 0) return;
1840
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, FEED_REFUSALS[feed], { field: feed });
1784
1841
  }
1785
1842
 
1786
- // src/server/handlers/documents.ts
1787
- function createDocumentHandlers(deps) {
1788
- const { engine } = deps;
1789
- const writeBlock = writeBlockLookupOf(engine);
1790
- const reads = createDocumentReadHandlers(deps);
1791
- async function insert(ctx) {
1792
- const body = await parseWriteJson(ctx, writeBlock);
1793
- if (!body) return;
1794
- const failure = validateDocumentBody(body);
1795
- if (failure) {
1796
- rejectInvalid(ctx, failure);
1797
- return;
1798
- }
1799
- try {
1800
- const id = await engine.insert(ctx.params[0], body.document, body.id, body.options);
1801
- respondJson(ctx, { id }, 201);
1802
- } catch (err) {
1803
- respondError(ctx, err);
1804
- }
1843
+ // src/server/task-store.ts
1844
+ var InMemoryTaskStore = class {
1845
+ /**
1846
+ * Builds a store that keeps task records in this process alone.
1847
+ *
1848
+ * @param maxRetained - Records to keep before the oldest finished ones are
1849
+ * dropped. Defaults to 1000.
1850
+ */
1851
+ constructor(maxRetained = DEFAULT_MAX_RETAINED_TASKS) {
1852
+ this.maxRetained = maxRetained;
1805
1853
  }
1806
- async function put(ctx) {
1807
- const body = await parseWriteJson(ctx, writeBlock);
1808
- if (!body) return;
1809
- const failure = validateDocumentBody(body);
1810
- if (failure) {
1811
- rejectInvalid(ctx, failure);
1812
- return;
1854
+ records = /* @__PURE__ */ new Map();
1855
+ /**
1856
+ * Stores a record, replacing whatever its id already held.
1857
+ *
1858
+ * @param record - The record to store. A copy is kept, so changing the
1859
+ * object afterwards leaves the store untouched.
1860
+ */
1861
+ async set(record) {
1862
+ this.records.set(record.id, { ...record });
1863
+ this.prune();
1864
+ }
1865
+ /**
1866
+ * Reads one record back.
1867
+ *
1868
+ * @param id - The task to read.
1869
+ * @returns A copy of the record, or `null` when the store holds no such id.
1870
+ */
1871
+ async get(id) {
1872
+ const record = this.records.get(id);
1873
+ return record ? { ...record } : null;
1874
+ }
1875
+ /**
1876
+ * Lists every record the store holds.
1877
+ *
1878
+ * @returns A copy of each record, oldest first.
1879
+ */
1880
+ async list() {
1881
+ return [...this.records.values()].map((record) => ({ ...record }));
1882
+ }
1883
+ /**
1884
+ * Removes one record. An id the store never held is not an error.
1885
+ *
1886
+ * @param id - The task to remove.
1887
+ */
1888
+ async delete(id) {
1889
+ this.records.delete(id);
1890
+ }
1891
+ prune() {
1892
+ if (this.records.size <= this.maxRetained) return;
1893
+ for (const [id, record] of this.records) {
1894
+ if (this.records.size <= this.maxRetained) break;
1895
+ if (isTerminalTaskStatus(record.status)) this.records.delete(id);
1813
1896
  }
1814
- const [name, id] = ctx.params;
1815
- try {
1816
- if (await engine.has(name, id)) {
1817
- await engine.update(name, id, body.document, body.options);
1818
- respondJson(ctx, { id, created: false });
1819
- } else {
1820
- const created = await engine.insert(name, body.document, id, body.options);
1821
- respondJson(ctx, { id: created, created: true }, 201);
1897
+ }
1898
+ };
1899
+ function byNewestFirst(left, right) {
1900
+ if (left.createdAt !== right.createdAt) return right.createdAt - left.createdAt;
1901
+ return left.id.localeCompare(right.id);
1902
+ }
1903
+ function matchesQuery(record, query) {
1904
+ if (query.indexName !== void 0 && record.indexName !== query.indexName) return false;
1905
+ if (query.type !== void 0 && query.type.length > 0 && !query.type.includes(record.type)) return false;
1906
+ if (query.status !== void 0 && query.status.length > 0 && !query.status.includes(record.status)) return false;
1907
+ return true;
1908
+ }
1909
+ var TaskRegistry = class {
1910
+ constructor(store, instanceId, maxConcurrent = 0) {
1911
+ this.store = store;
1912
+ this.instanceId = instanceId;
1913
+ this.maxConcurrent = maxConcurrent;
1914
+ }
1915
+ live = /* @__PURE__ */ new Map();
1916
+ /** Records a task as running and drives `op` to completion in the background.
1917
+ * Returns the record once it is persisted so the caller can respond 202.
1918
+ * @throws A `NarsilError` with `TOO_MANY_REQUESTS` where this instance
1919
+ * already drives {@link ServerLimits.maxConcurrentTasks} tasks, because each
1920
+ * one holds its own working set until it finishes. */
1921
+ async start(type, indexName, op, progress, backupName) {
1922
+ if (this.maxConcurrent > 0 && this.live.size >= this.maxConcurrent) {
1923
+ throw new NarsilError(
1924
+ ServerErrorCodes.TOO_MANY_REQUESTS,
1925
+ `This instance already drives ${this.maxConcurrent} tasks; poll one to completion before starting another`,
1926
+ { running: this.live.size, limit: this.maxConcurrent }
1927
+ );
1928
+ }
1929
+ const now = Date.now();
1930
+ const record = {
1931
+ id: randomUUID(),
1932
+ type,
1933
+ indexName,
1934
+ ...backupName === void 0 ? {} : { backupName },
1935
+ owner: this.instanceId,
1936
+ status: "running",
1937
+ createdAt: now,
1938
+ startedAt: now,
1939
+ ...progress === void 0 ? {} : { progress }
1940
+ };
1941
+ const task = {
1942
+ record,
1943
+ controller: new AbortController(),
1944
+ writes: Promise.resolve(),
1945
+ lastProgressWriteAt: now,
1946
+ settled: false,
1947
+ finished: Promise.resolve()
1948
+ };
1949
+ this.live.set(record.id, task);
1950
+ const accepted = Promise.resolve().then(() => this.store.set(record, RUNNING_TASK_TTL_MS));
1951
+ task.writes = accepted;
1952
+ task.finished = accepted.then(
1953
+ () => this.drive(task, op),
1954
+ async () => {
1955
+ this.live.delete(record.id);
1956
+ await this.store.delete(record.id).catch(() => void 0);
1822
1957
  }
1958
+ );
1959
+ try {
1960
+ await accepted;
1823
1961
  } catch (err) {
1824
- respondError(ctx, err);
1962
+ await task.finished;
1963
+ throw err;
1825
1964
  }
1965
+ return record;
1826
1966
  }
1827
- async function patch(ctx) {
1828
- const body = await parseWriteJson(ctx, writeBlock);
1829
- if (!body) return;
1830
- const failure = validateDocumentBody(body);
1831
- if (failure) {
1832
- rejectInvalid(ctx, failure);
1967
+ get(id) {
1968
+ return this.store.get(id);
1969
+ }
1970
+ async shutdown() {
1971
+ const tasks = [...this.live.values()];
1972
+ for (const task of tasks) task.controller.abort(new Error("The server is shutting down"));
1973
+ await Promise.all(tasks.map((task) => task.finished));
1974
+ }
1975
+ /** Reads one page of task records, newest first, keeping only what the query
1976
+ * matches. Filtering happens here rather than in the store, so every custom
1977
+ * {@link TaskStore} works unchanged. */
1978
+ async list(query = {}) {
1979
+ const from = Math.max(0, Math.trunc(query.from ?? 0));
1980
+ const limit = Math.max(1, Math.trunc(query.limit ?? DEFAULT_TASK_PAGE_SIZE));
1981
+ const matched = (await this.store.list()).filter((record) => matchesQuery(record, query)).sort(byNewestFirst);
1982
+ const page = matched.slice(from, from + limit);
1983
+ const consumed = from + page.length;
1984
+ return { tasks: page, total: matched.length, from, limit, next: consumed < matched.length ? consumed : null };
1985
+ }
1986
+ /** Asks a running task to stop. The work stops between units, so the task
1987
+ * reaches `cancelled` only once it actually stopped, and a request that
1988
+ * arrives too late leaves it `succeeded`. */
1989
+ async cancel(id) {
1990
+ const stored = await this.store.get(id);
1991
+ if (!stored) return { outcome: "not-found", record: null };
1992
+ if (isTerminalTaskStatus(stored.status)) {
1993
+ return { outcome: "already-finished", record: stored };
1994
+ }
1995
+ const task = this.live.get(id);
1996
+ if (!task) {
1997
+ const outcome = stored.owner === this.instanceId ? "already-finished" : "owned-by-another-instance";
1998
+ return { outcome, record: stored };
1999
+ }
2000
+ task.controller.abort(new Error("The task was cancelled"));
2001
+ const record = { ...task.record, cancelRequestedAt: Date.now() };
2002
+ task.record = record;
2003
+ await this.enqueueWrite(task, record, RUNNING_TASK_TTL_MS);
2004
+ return { outcome: "cancelling", record };
2005
+ }
2006
+ /** Fails this instance's previously-running tasks after a restart. A task can
2007
+ * only be advanced by the process that owns it, so its own running tasks from
2008
+ * a prior life are dead and must not keep showing a stale running status. This
2009
+ * is effective only with a stable instanceId; with the default random id it
2010
+ * finds none of its own prior tasks, which is also correct for the in-memory
2011
+ * default (which starts empty). */
2012
+ async reconcile(cancelled = () => false) {
2013
+ let records;
2014
+ try {
2015
+ records = await this.store.list();
2016
+ } catch {
1833
2017
  return;
1834
2018
  }
2019
+ for (const record of records) {
2020
+ if (cancelled()) return;
2021
+ if (record.owner !== this.instanceId) continue;
2022
+ if (isTerminalTaskStatus(record.status)) continue;
2023
+ const failed = {
2024
+ ...record,
2025
+ status: "failed",
2026
+ completedAt: Date.now(),
2027
+ error: { code: ServerErrorCodes.TASK_INTERRUPTED, message: "The server restarted while this task was running" }
2028
+ };
2029
+ try {
2030
+ await this.store.set(failed, TERMINAL_TASK_TTL_MS);
2031
+ } catch {
2032
+ }
2033
+ }
2034
+ }
2035
+ enqueueWrite(task, record, ttlMs) {
2036
+ task.writes = task.writes.then(
2037
+ () => this.store.set(record, ttlMs),
2038
+ () => this.store.set(record, ttlMs)
2039
+ );
2040
+ return task.writes.catch(() => {
2041
+ });
2042
+ }
2043
+ reportProgress(task, change) {
2044
+ if (task.settled) return;
2045
+ task.record = { ...task.record, ...change };
2046
+ const now = Date.now();
2047
+ if (now - task.lastProgressWriteAt < TASK_PROGRESS_WRITE_INTERVAL_MS) return;
2048
+ task.lastProgressWriteAt = now;
2049
+ void this.enqueueWrite(task, task.record, RUNNING_TASK_TTL_MS);
2050
+ }
2051
+ async drive(task, op) {
2052
+ const context = {
2053
+ signal: task.controller.signal,
2054
+ reportProgress: (progress) => this.reportProgress(task, { progress }),
2055
+ reportSteps: (steps) => this.reportProgress(task, { steps }),
2056
+ reportResult: (result) => {
2057
+ task.record = { ...task.record, result };
2058
+ },
2059
+ reportBuild: (build) => {
2060
+ task.record = { ...task.record, build };
2061
+ }
2062
+ };
2063
+ let settled;
2064
+ try {
2065
+ await op(context);
2066
+ settled = { ...task.record, status: "succeeded" };
2067
+ } catch (err) {
2068
+ settled = task.controller.signal.aborted ? { ...task.record, status: "cancelled" } : {
2069
+ ...task.record,
2070
+ status: "failed",
2071
+ error: err instanceof NarsilError ? serializeNarsilError(err) : { code: "INTERNAL_ERROR", message: "The operation failed" }
2072
+ };
2073
+ }
2074
+ task.settled = true;
2075
+ task.record = { ...settled, completedAt: Date.now() };
2076
+ await this.enqueueWrite(task, task.record, TERMINAL_TASK_TTL_MS);
2077
+ this.live.delete(task.record.id);
2078
+ }
2079
+ };
2080
+
2081
+ // src/server/changes/setting-handlers.ts
2082
+ function createChangesSettingHandlers(engine) {
2083
+ async function getChanges(ctx) {
1835
2084
  try {
1836
- await engine.update(ctx.params[0], ctx.params[1], body.document, body.options);
1837
- respondJson(ctx, { id: ctx.params[1] });
2085
+ respondJson(ctx, await engine.getChanges(ctx.params[0]));
1838
2086
  } catch (err) {
1839
2087
  respondError(ctx, err);
1840
2088
  }
1841
2089
  }
1842
- async function remove(ctx) {
2090
+ async function updateChanges(ctx) {
2091
+ const body = parseJson(ctx);
2092
+ if (body === void 0) return;
1843
2093
  try {
1844
- await engine.remove(ctx.params[0], ctx.params[1], { wait: ctx.query.get("wait") === "true" });
1845
- respondJson(ctx, { id: ctx.params[1], removed: true });
2094
+ await engine.updateChanges(ctx.params[0], body);
2095
+ respondJson(ctx, { ok: true });
1846
2096
  } catch (err) {
1847
2097
  respondError(ctx, err);
1848
2098
  }
1849
2099
  }
1850
- async function waitForWrites(ctx) {
2100
+ return { getChanges, updateChanges };
2101
+ }
2102
+ function snapshotHistoryOf(raw) {
2103
+ if (raw === null || raw === "false") return false;
2104
+ return raw === "true" ? true : null;
2105
+ }
2106
+ function createAdminHandlers(deps) {
2107
+ const { engine, tasks, limits } = deps;
2108
+ async function checkpoint(ctx) {
1851
2109
  try {
1852
- await engine.waitForWrites(ctx.params[0]);
2110
+ await engine.checkpoint(ctx.params[0]);
1853
2111
  respondJson(ctx, { ok: true });
1854
2112
  } catch (err) {
1855
2113
  respondError(ctx, err);
1856
2114
  }
1857
2115
  }
1858
- async function batch(ctx) {
1859
- const body = parseJson(ctx, ctx.rawBody && isRemoveOnlyBatch(ctx.rawBody) ? "read" : "write");
1860
- if (!body) return;
1861
- const failure = validateBatch(body);
1862
- if (failure) {
1863
- rejectInvalid(ctx, failure);
2116
+ async function snapshot(ctx) {
2117
+ const queryHistory = snapshotHistoryOf(ctx.query.get("queryHistory"));
2118
+ if (queryHistory === null) {
2119
+ badRequest(ctx.res, 'Query parameter "queryHistory" accepts only true or false', {
2120
+ value: ctx.query.get("queryHistory")
2121
+ });
1864
2122
  return;
1865
2123
  }
1866
- const action = body.action ?? "insert";
1867
2124
  try {
1868
- if (action === "update") {
1869
- respondJson(
1870
- ctx,
1871
- serializeBatchResult(await engine.updateBatch(ctx.params[0], body.updates ?? [], body.options))
1872
- );
1873
- } else if (action === "delete") {
1874
- respondJson(ctx, serializeBatchResult(await engine.removeBatch(ctx.params[0], body.docIds ?? [], body.options)));
1875
- } else {
1876
- respondJson(
1877
- ctx,
1878
- serializeBatchResult(await engine.insertBatch(ctx.params[0], body.documents ?? [], body.options))
1879
- );
2125
+ const archive = await engine.snapshot(ctx.params[0], queryHistory ? { queryHistory } : void 0);
2126
+ if (ctx.abort.aborted) {
2127
+ await archive.cancel();
2128
+ return;
1880
2129
  }
2130
+ await sendStream(ctx.res, ctx.abort, archive, [["Content-Type", SNAPSHOT_ARCHIVE_CONTENT_TYPE]]);
1881
2131
  } catch (err) {
1882
2132
  respondError(ctx, err);
1883
2133
  }
1884
2134
  }
1885
- return { ...reads, insert, put, patch, remove, batch, waitForWrites };
1886
- }
1887
-
1888
- // src/server/handlers/health.ts
1889
- function createHealthHandlers(deps) {
1890
- function livez(ctx) {
1891
- respondJson(ctx, { status: "ok" });
2135
+ function vectorMaintenance(ctx) {
2136
+ try {
2137
+ respondJson(ctx, { fields: engine.vectorMaintenanceStatus(ctx.params[0]) });
2138
+ } catch (err) {
2139
+ respondError(ctx, err);
2140
+ }
1892
2141
  }
1893
- function readyz(ctx) {
1894
- const cluster = deps.cluster;
1895
- if (cluster === void 0) {
1896
- if (deps.isReady()) {
1897
- respondJson(ctx, { status: "ready" });
1898
- } else {
1899
- respondJson(ctx, { status: "unavailable" }, 503);
1900
- }
1901
- return;
2142
+ async function compact(ctx) {
2143
+ const body = parseJsonOptional(ctx);
2144
+ if (!body) return;
2145
+ try {
2146
+ await engine.compactVectors(ctx.params[0], body.field);
2147
+ respondJson(ctx, { ok: true });
2148
+ } catch (err) {
2149
+ respondError(ctx, err);
1902
2150
  }
1903
- const readiness = cluster.getReadiness();
1904
- const ready = deps.isReady() && readiness === "SERVING";
1905
- respondJson(
1906
- ctx,
1907
- {
1908
- status: ready ? "ready" : "unavailable",
1909
- cluster: describeNode(cluster)
1910
- },
1911
- ready ? 200 : 503
1912
- );
1913
2151
  }
1914
- return { livez, readyz };
1915
- }
1916
-
1917
- // src/server/ndjson.ts
1918
- var NEWLINE = 10;
1919
- var CARRIAGE_RETURN = 13;
1920
- var NdjsonLineTooLongError = class extends NarsilError {
1921
- constructor(lineNumber) {
1922
- super(ServerErrorCodes.PAYLOAD_TOO_LARGE, `NDJSON line ${lineNumber} exceeds the configured per-line size limit`, {
1923
- line: lineNumber
1924
- });
1925
- this.name = "NdjsonLineTooLongError";
2152
+ async function partitionConfig(ctx) {
2153
+ const body = parseJsonOptional(ctx);
2154
+ if (!body) return;
2155
+ try {
2156
+ await engine.updatePartitionConfig(ctx.params[0], body);
2157
+ respondJson(ctx, { ok: true });
2158
+ } catch (err) {
2159
+ respondError(ctx, err);
2160
+ }
1926
2161
  }
1927
- };
1928
- function* iterateNdjson(buffer, maxLineBytes) {
1929
- let start = 0;
1930
- let lineNumber = 0;
1931
- const length = buffer.length;
1932
- while (start < length) {
1933
- let end = buffer.indexOf(NEWLINE, start);
1934
- if (end === -1) end = length;
1935
- if (end - start > maxLineBytes) throw new NdjsonLineTooLongError(lineNumber + 1);
1936
- let contentEnd = end;
1937
- if (contentEnd > start && buffer[contentEnd - 1] === CARRIAGE_RETURN) contentEnd--;
1938
- if (contentEnd > start) {
1939
- lineNumber++;
1940
- yield { lineNumber, bytes: buffer.subarray(start, contentEnd), bytesConsumed: Math.min(end + 1, length) };
2162
+ async function readSynonyms(ctx) {
2163
+ try {
2164
+ respondJson(ctx, { synonyms: await engine.getSynonyms(ctx.params[0]) });
2165
+ } catch (err) {
2166
+ respondError(ctx, err);
1941
2167
  }
1942
- start = end + 1;
1943
2168
  }
1944
- }
1945
-
1946
- // src/server/handlers/import.ts
1947
- function yieldToEventLoop() {
1948
- return new Promise((resolve) => setImmediate(resolve));
1949
- }
1950
- function isPlainObject(value) {
1951
- return typeof value === "object" && value !== null && !Array.isArray(value);
1952
- }
1953
- async function runImport(engine, options) {
1954
- const { indexName, body, maxLineBytes, batchSize, maxErrors, signal, onProgress } = options;
1955
- const bytesTotal = body.length;
1956
- const writeBlock = writeBlockLookupOf(engine);
1957
- const errors = [];
1958
- let indexed = 0;
1959
- let failed = 0;
1960
- let bytesProcessed = 0;
1961
- let pending = [];
1962
- const recordFailure = (error) => {
1963
- failed += 1;
1964
- if (errors.length < maxErrors) errors.push(error);
1965
- };
1966
- const flushDocuments = async (documents) => {
1967
- const result = await engine.insertBatch(indexName, documents, { skipClone: true });
1968
- indexed += result.succeeded.length;
1969
- for (const failure of result.failed) {
1970
- const serialized = failure.error instanceof NarsilError ? serializeNarsilError(failure.error) : { code: ServerErrorCodes.INTERNAL_ERROR, message: String(failure.error) };
1971
- recordFailure({
1972
- docId: failure.docId,
1973
- code: serialized.code,
1974
- message: serialized.message,
1975
- ...serialized.details === void 0 ? {} : { details: serialized.details }
1976
- });
2169
+ async function replaceSynonyms(ctx) {
2170
+ const body = parseJson(ctx);
2171
+ if (body === void 0) return;
2172
+ if (body === null || typeof body !== "object" || Array.isArray(body) || !Object.hasOwn(body, "synonyms")) {
2173
+ if (!ctx.abort.aborted) badRequest(ctx.res, 'Send the synonym set as { "synonyms": [ ...rules ] }');
2174
+ return;
1977
2175
  }
1978
- };
1979
- const inFlight = /* @__PURE__ */ new Set();
1980
- let flushError;
1981
- const startFlush = () => {
1982
- if (pending.length === 0) return;
1983
- const documents = pending.map((entry) => entry.document);
1984
- pending = [];
1985
- const run = flushDocuments(documents).catch((err) => {
1986
- flushError ??= err;
1987
- }).finally(() => inFlight.delete(run));
1988
- inFlight.add(run);
1989
- };
1990
- const settleFlushes = async () => {
1991
- await Promise.all(inFlight);
1992
- if (flushError !== void 0) throw flushError;
1993
- };
1994
- const report = () => {
1995
- onProgress?.({ indexed, failed, bytesProcessed, bytesTotal });
1996
- };
1997
- const seenIds = /* @__PURE__ */ new Set();
1998
- const repeated = [];
1999
- try {
2000
- for (const line of iterateNdjson(body, maxLineBytes)) {
2001
- bytesProcessed = line.bytesConsumed;
2002
- const refusal = parseRefusalOf(line.bytes);
2003
- if (refusal !== null) {
2004
- const block = refusal.status === 429 && writeBlock !== void 0 ? await writeBlock(indexName).catch(() => null) : null;
2005
- recordFailure(
2006
- block === null ? { line: line.lineNumber, code: refusal.code, message: refusal.message, details: refusal.details } : { line: line.lineNumber, ...serializeNarsilError(block) }
2007
- );
2008
- continue;
2009
- }
2010
- let parsed;
2011
- try {
2012
- parsed = JSON.parse(line.bytes.toString("utf8"));
2013
- } catch {
2014
- recordFailure({ line: line.lineNumber, code: ServerErrorCodes.INVALID_JSON, message: "Line is not valid JSON" });
2015
- continue;
2016
- }
2017
- if (!isPlainObject(parsed)) {
2018
- recordFailure({
2019
- line: line.lineNumber,
2020
- code: ServerErrorCodes.INVALID_REQUEST,
2021
- message: "Line is not a JSON object"
2022
- });
2023
- continue;
2024
- }
2025
- const docId = providedDocId(parsed);
2026
- if (docId !== void 0) {
2027
- if (seenIds.has(docId)) {
2028
- repeated.push({ document: parsed, line: line.lineNumber });
2029
- continue;
2030
- }
2031
- seenIds.add(docId);
2032
- }
2033
- pending.push({ document: parsed, line: line.lineNumber });
2034
- if (pending.length >= batchSize) {
2035
- signal?.throwIfAborted();
2036
- startFlush();
2037
- if (inFlight.size >= IMPORT_FLUSHES_IN_FLIGHT) await Promise.race(inFlight);
2038
- if (flushError !== void 0) await settleFlushes();
2039
- report();
2040
- await yieldToEventLoop();
2041
- signal?.throwIfAborted();
2042
- }
2176
+ try {
2177
+ await engine.updateSynonyms(ctx.params[0], Reflect.get(body, "synonyms"));
2178
+ respondJson(ctx, { ok: true });
2179
+ } catch (err) {
2180
+ respondError(ctx, err);
2043
2181
  }
2044
- startFlush();
2045
- await settleFlushes();
2046
- for (let start = 0; start < repeated.length; start += batchSize) {
2047
- signal?.throwIfAborted();
2048
- await flushDocuments(repeated.slice(start, start + batchSize).map((entry) => entry.document));
2182
+ }
2183
+ async function updateBm25(ctx) {
2184
+ const body = parseJson(ctx);
2185
+ if (body === void 0) return;
2186
+ try {
2187
+ await engine.updateBM25(ctx.params[0], body);
2188
+ respondJson(ctx, { ok: true });
2189
+ } catch (err) {
2190
+ respondError(ctx, err);
2049
2191
  }
2050
- } catch (err) {
2051
- await Promise.allSettled(inFlight);
2052
- report();
2053
- throw err;
2054
2192
  }
2055
- bytesProcessed = bytesTotal;
2056
- report();
2057
- return { indexed, failed, errors, errorsTruncated: failed > errors.length };
2058
- }
2059
- function createImportHandler(deps) {
2060
- const { engine, limits, tasks } = deps;
2061
- return async function importNdjson(ctx) {
2062
- const raw = ctx.rawBody;
2063
- if (!raw || raw.length === 0) {
2064
- sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
2193
+ async function memory(ctx) {
2194
+ try {
2195
+ const stats = await engine.getMemoryStats();
2196
+ const response = { ...stats, requestThreads: deps.requestThreadCount() };
2197
+ respondJson(ctx, response);
2198
+ } catch (err) {
2199
+ respondError(ctx, err);
2200
+ }
2201
+ }
2202
+ async function requireIndexExists(name) {
2203
+ await indexReadsOf(engine).getStats(name);
2204
+ }
2205
+ async function optimize(ctx) {
2206
+ const body = parseJsonOptional(ctx);
2207
+ if (!body) return;
2208
+ const name = ctx.params[0];
2209
+ try {
2210
+ await requireIndexExists(name);
2211
+ } catch (err) {
2212
+ respondError(ctx, err);
2065
2213
  return;
2066
2214
  }
2067
- const indexName = ctx.params[0];
2068
- const runOptions = {
2069
- indexName,
2070
- body: raw,
2071
- maxLineBytes: limits.maxLineBytes,
2072
- batchSize: limits.importBatchSize,
2073
- maxErrors: limits.maxImportErrors
2074
- };
2075
- if (ctx.query.get("async") === "true") {
2076
- try {
2077
- await engine.countDocuments(indexName);
2078
- } catch (err) {
2079
- respondError(ctx, err);
2080
- return;
2081
- }
2082
- const record = await tasks.start(
2083
- "import",
2084
- indexName,
2085
- async (context) => {
2086
- const result = await runImport(engine, {
2087
- ...runOptions,
2088
- signal: context.signal,
2089
- onProgress: context.reportProgress
2090
- });
2091
- context.reportResult(result);
2092
- },
2093
- { indexed: 0, failed: 0, bytesProcessed: 0, bytesTotal: raw.length }
2094
- );
2095
- respondJson(ctx, record, 202);
2096
- return;
2097
- }
2098
- const controller = new AbortController();
2099
- ctx.abort.onAbort(() => controller.abort());
2215
+ const record = await tasks.start("optimizeVectors", name, () => engine.optimizeVectors(name, body.field));
2216
+ respondJson(ctx, record, 202);
2217
+ }
2218
+ async function rebuildAnalysis(ctx) {
2219
+ const name = ctx.params[0];
2100
2220
  try {
2101
- respondJson(ctx, await runImport(engine, { ...runOptions, signal: controller.signal }));
2221
+ await requireIndexExists(name);
2102
2222
  } catch (err) {
2103
- if (ctx.abort.aborted) return;
2104
2223
  respondError(ctx, err);
2224
+ return;
2105
2225
  }
2106
- };
2107
- }
2108
-
2109
- // src/server/config-mapping.ts
2110
- function mapHttpIndexConfig(http) {
2111
- if (typeof http !== "object" || http === null) {
2112
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index config must be an object");
2113
- }
2114
- if (typeof http.schema !== "object" || http.schema === null || Array.isArray(http.schema)) {
2115
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Index config requires a "schema" object');
2116
- }
2117
- const config = { schema: http.schema };
2118
- if (http.language !== void 0) config.language = http.language;
2119
- if (http.partitions !== void 0) config.partitions = http.partitions;
2120
- if (http.defaultScoring !== void 0) config.defaultScoring = http.defaultScoring;
2121
- if (http.bm25 !== void 0) config.bm25 = http.bm25;
2122
- if (http.trackPositions !== void 0) config.trackPositions = http.trackPositions;
2123
- if (http.textStorage !== void 0) config.textStorage = http.textStorage;
2124
- if (http.surfaceForms !== void 0) config.surfaceForms = http.surfaceForms === true;
2125
- if (http.vectorPromotion !== void 0) config.vectorPromotion = http.vectorPromotion;
2126
- if (http.strict !== void 0) config.strict = http.strict;
2127
- if (http.required !== void 0) config.required = http.required;
2128
- if (http.patternValueLimit !== void 0) config.patternValueLimit = http.patternValueLimit;
2129
- if (http.synonyms !== void 0) config.synonyms = http.synonyms;
2130
- if (http.tokenizer !== void 0) {
2131
- if (typeof http.tokenizer !== "string") {
2132
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Field "tokenizer" must be a registered name');
2133
- }
2134
- config.tokenizer = http.tokenizer;
2135
- }
2136
- if (typeof http.stopWords === "string") {
2137
- config.stopWords = http.stopWords;
2138
- } else if (http.stopWords !== void 0) {
2139
- if (!Array.isArray(http.stopWords) || http.stopWords.some((w) => typeof w !== "string")) {
2140
- throw new NarsilError(
2141
- ErrorCodes.CONFIG_INVALID,
2142
- 'Field "stopWords" must be a registered name or an array of strings'
2143
- );
2144
- }
2145
- config.stopWords = new Set(http.stopWords);
2146
- }
2147
- if (http.embedding !== void 0) {
2148
- config.embedding = mapEmbedding(http.embedding);
2149
- }
2150
- return config;
2151
- }
2152
- function mapEmbedding(embedding) {
2153
- if (typeof embedding !== "object" || embedding === null) {
2154
- throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, 'Field "embedding" must be an object');
2155
- }
2156
- if (typeof embedding.fields !== "object" || embedding.fields === null) {
2157
- throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, 'Field "embedding.fields" is required');
2158
- }
2159
- const mapped = { fields: embedding.fields };
2160
- if (embedding.adapter !== void 0) {
2161
- if (typeof embedding.adapter !== "string" || embedding.adapter.length === 0) {
2162
- throw new NarsilError(
2163
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
2164
- 'Field "embedding.adapter" must be a non-empty adapter name'
2165
- );
2166
- }
2167
- mapped.adapter = embedding.adapter;
2226
+ const record = await tasks.start("rebuildAnalysis", name, () => engine.rebuildAnalysis(name));
2227
+ respondJson(ctx, record, 202);
2168
2228
  }
2169
- return mapped;
2170
- }
2171
-
2172
- // src/server/handlers/indexes.ts
2173
- function createIndexHandlers(deps) {
2174
- const { engine } = deps;
2175
- async function create(ctx) {
2176
- const body = parseJson(ctx);
2229
+ async function changeAnalysis(ctx) {
2230
+ const body = parseJsonOptional(ctx);
2177
2231
  if (!body) return;
2178
- const failure = validateCreateIndex(body);
2232
+ const failure = validateChangeAnalysis(body);
2179
2233
  if (failure) {
2180
2234
  rejectInvalid(ctx, failure);
2181
2235
  return;
2182
2236
  }
2237
+ const name = ctx.params[0];
2183
2238
  try {
2184
- const config = mapHttpIndexConfig(body.config);
2185
- await engine.createIndex(body.name, config);
2186
- respondJson(ctx, { name: body.name }, 201);
2239
+ await requireIndexExists(name);
2187
2240
  } catch (err) {
2188
2241
  respondError(ctx, err);
2242
+ return;
2189
2243
  }
2244
+ const record = await tasks.start("changeAnalysis", name, () => engine.changeAnalysis(name, body));
2245
+ respondJson(ctx, record, 202);
2190
2246
  }
2191
- async function list(ctx) {
2192
- try {
2193
- respondJson(ctx, { indexes: await indexReadsOf(engine).listIndexes() });
2194
- } catch (err) {
2195
- respondError(ctx, err);
2247
+ async function rebalance(ctx) {
2248
+ const body = parseJsonOptional(ctx);
2249
+ if (!body) return;
2250
+ const failure = validateRebalance(body);
2251
+ if (failure) {
2252
+ rejectInvalid(ctx, failure);
2253
+ return;
2196
2254
  }
2197
- }
2198
- async function drop(ctx) {
2255
+ const target = body.targetPartitionCount;
2256
+ const name = ctx.params[0];
2199
2257
  try {
2200
- await engine.dropIndex(ctx.params[0]);
2201
- respondJson(ctx, { name: ctx.params[0], dropped: true });
2258
+ await requireIndexExists(name);
2202
2259
  } catch (err) {
2203
2260
  respondError(ctx, err);
2261
+ return;
2204
2262
  }
2263
+ const record = await tasks.start("rebalance", name, () => engine.rebalance(name, target));
2264
+ respondJson(ctx, record, 202);
2205
2265
  }
2206
- async function stats(ctx) {
2207
- try {
2208
- respondJson(ctx, await indexReadsOf(engine).getStats(ctx.params[0]));
2209
- } catch (err) {
2210
- respondError(ctx, err);
2266
+ async function restore(ctx) {
2267
+ const spooled = ctx.spooledBody;
2268
+ if (spooled === null || spooled.bytes === 0) {
2269
+ await spooled?.remove();
2270
+ sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
2271
+ return;
2211
2272
  }
2212
- }
2213
- async function partitions(ctx) {
2273
+ const name = ctx.params[0];
2214
2274
  try {
2215
- respondJson(ctx, {
2216
- partitions: await indexReadsOf(engine).getPartitionStats(ctx.params[0])
2275
+ const record = await tasks.start("restore", name, async () => {
2276
+ try {
2277
+ await engine.restore(name, createReadStream(spooled.path));
2278
+ } finally {
2279
+ await spooled.remove();
2280
+ }
2217
2281
  });
2282
+ respondJson(ctx, record, 202);
2218
2283
  } catch (err) {
2284
+ await spooled.remove();
2219
2285
  respondError(ctx, err);
2220
2286
  }
2221
2287
  }
2222
- async function clear(ctx) {
2223
- try {
2224
- await engine.clear(ctx.params[0]);
2225
- respondJson(ctx, { name: ctx.params[0], cleared: true });
2226
- } catch (err) {
2227
- respondError(ctx, err);
2288
+ async function getTask(ctx) {
2289
+ const record = await tasks.get(ctx.params[0]);
2290
+ if (!record) {
2291
+ sendError(ctx.res, 404, ServerErrorCodes.TASK_NOT_FOUND, `Task "${ctx.params[0]}" not found`);
2292
+ return;
2228
2293
  }
2294
+ respondJson(ctx, record);
2229
2295
  }
2230
- async function open(ctx) {
2231
- try {
2232
- await engine.open(ctx.params[0]);
2233
- respondJson(ctx, { name: ctx.params[0], state: "open" });
2234
- } catch (err) {
2235
- respondError(ctx, err);
2296
+ async function listTasks(ctx) {
2297
+ const parsed = parseTaskListQuery(ctx.query, limits.maxTaskPageSize);
2298
+ if ("failure" in parsed) {
2299
+ rejectInvalid(ctx, parsed.failure);
2300
+ return;
2236
2301
  }
2302
+ respondJson(ctx, await tasks.list(parsed.query));
2237
2303
  }
2238
- async function close(ctx) {
2239
- try {
2240
- await engine.close(ctx.params[0]);
2241
- respondJson(ctx, { name: ctx.params[0], state: "closed" });
2242
- } catch (err) {
2243
- respondError(ctx, err);
2304
+ async function cancelTask(ctx) {
2305
+ const taskId = ctx.params[0];
2306
+ const { outcome, record } = await tasks.cancel(taskId);
2307
+ if (outcome === "not-found") {
2308
+ sendError(ctx.res, 404, ServerErrorCodes.TASK_NOT_FOUND, `Task "${taskId}" not found`);
2309
+ return;
2310
+ }
2311
+ if (outcome === "already-finished") {
2312
+ sendError(
2313
+ ctx.res,
2314
+ 409,
2315
+ ServerErrorCodes.TASK_NOT_CANCELLABLE,
2316
+ `Task "${taskId}" has already finished`,
2317
+ record === null ? void 0 : { status: record.status }
2318
+ );
2319
+ return;
2320
+ }
2321
+ if (outcome === "owned-by-another-instance") {
2322
+ sendError(
2323
+ ctx.res,
2324
+ 409,
2325
+ ServerErrorCodes.TASK_OWNED_BY_ANOTHER_INSTANCE,
2326
+ `Task "${taskId}" runs on another server instance, which is the only one that can stop it`,
2327
+ record === null ? void 0 : { owner: record.owner }
2328
+ );
2329
+ return;
2244
2330
  }
2331
+ respondJson(ctx, record, 202);
2245
2332
  }
2246
- return { create, list, drop, stats, partitions, clear, open, close };
2333
+ return {
2334
+ checkpoint,
2335
+ snapshot,
2336
+ vectorMaintenance,
2337
+ compact,
2338
+ partitionConfig,
2339
+ readSynonyms,
2340
+ replaceSynonyms,
2341
+ updateBm25,
2342
+ memory,
2343
+ optimize,
2344
+ rebalance,
2345
+ rebuildAnalysis,
2346
+ changeAnalysis,
2347
+ restore,
2348
+ getTask,
2349
+ listTasks,
2350
+ cancelTask
2351
+ };
2247
2352
  }
2248
2353
 
2249
- // src/server/handlers/metrics.ts
2250
- function labelValue(value) {
2251
- return value.replaceAll("\\", "\\\\").replaceAll('"', '\\"').replaceAll("\n", "\\n");
2252
- }
2253
- function labels(pairs) {
2254
- return `{${pairs.map(([name, value]) => `${name}="${labelValue(value)}"`).join(",")}}`;
2255
- }
2256
- function family(name, type, help, samples) {
2257
- return [`# HELP ${name} ${help}`, `# TYPE ${name} ${type}`, ...samples];
2258
- }
2259
- function prometheusText(metrics) {
2260
- const lines = [
2261
- ...family(
2262
- "narsil_replication_lag",
2263
- "gauge",
2264
- "Entries that the replica has yet to acknowledge, as the primary's last sequence number minus the replica's.",
2265
- metrics.replicationLag.map(
2266
- (figure) => `narsil_replication_lag${labels([
2267
- ["index", figure.indexName],
2268
- ["partition", String(figure.partitionId)],
2269
- ["replica", figure.replica]
2270
- ])} ${figure.lag}`
2271
- )
2272
- ),
2273
- ...family(
2274
- "narsil_commit_point",
2275
- "gauge",
2276
- "The highest sequence number that the leader of the partition has acknowledged to a client.",
2277
- metrics.commitPoints.map(
2278
- (figure) => `narsil_commit_point${labels([
2279
- ["index", figure.indexName],
2280
- ["partition", String(figure.partitionId)]
2281
- ])} ${figure.commitPoint}`
2282
- )
2283
- ),
2284
- ...family(
2285
- "narsil_commit_point_floor",
2286
- "gauge",
2287
- "The floor on the commit point that the allocation table stores, raised when the controller admits a replica.",
2288
- metrics.partitions.map(
2289
- (figure) => `narsil_commit_point_floor${labels([
2290
- ["index", figure.indexName],
2291
- ["partition", String(figure.partitionId)]
2292
- ])} ${figure.commitPointFloor}`
2293
- )
2294
- ),
2295
- ...family(
2296
- "narsil_in_sync_replicas",
2297
- "gauge",
2298
- "The replicas besides the primary that are in the in-sync set of the partition.",
2299
- metrics.partitions.map(
2300
- (figure) => `narsil_in_sync_replicas${labels([
2301
- ["index", figure.indexName],
2302
- ["partition", String(figure.partitionId)]
2303
- ])} ${figure.inSyncReplicas}`
2304
- )
2305
- ),
2306
- ...family(
2307
- "narsil_vector_rebuild_waiting_for_memory",
2308
- "gauge",
2309
- "A vector field whose rebuild waits for the node to have memory for a second graph, reported as 1.",
2310
- metrics.vectorRebuildsWaitingForMemory.map(
2311
- (figure) => `narsil_vector_rebuild_waiting_for_memory${labels([
2312
- ["index", figure.indexName],
2313
- ["field", figure.fieldName]
2314
- ])} 1`
2315
- )
2316
- ),
2317
- ...family(
2318
- "narsil_coordinator_request_seconds",
2319
- "histogram",
2320
- "How long the calls from this node to the cluster coordinator took.",
2321
- metrics.coordinatorRequests.flatMap((figure) => {
2322
- const operation = ["operation", figure.operation];
2323
- return [
2324
- ...figure.bucketBoundsSeconds.map(
2325
- (bound, position) => `narsil_coordinator_request_seconds_bucket${labels([operation, ["le", String(bound)]])} ${figure.bucketCounts[position] ?? 0}`
2326
- ),
2327
- `narsil_coordinator_request_seconds_bucket${labels([operation, ["le", "+Inf"]])} ${figure.count}`,
2328
- `narsil_coordinator_request_seconds_sum${labels([operation])} ${figure.sumSeconds}`,
2329
- `narsil_coordinator_request_seconds_count${labels([operation])} ${figure.count}`
2330
- ];
2331
- })
2332
- )
2333
- ];
2334
- return `${lines.join("\n")}
2335
- `;
2336
- }
2337
- function createMetricsHandler(deps) {
2338
- return async function metrics(ctx) {
2339
- try {
2340
- if (deps.cluster === void 0) {
2341
- throw new NarsilError(
2342
- ErrorCodes.CLUSTER_OPERATION_UNSUPPORTED,
2343
- "This server fronts a single engine, so it reports no cluster metrics",
2344
- {}
2345
- );
2346
- }
2347
- const text = prometheusText(await deps.cluster.getMetrics());
2348
- if (ctx.abort.aborted) return;
2349
- sendText(ctx.res, text, PROMETHEUS_TEXT_CONTENT_TYPE, ctx.abort);
2350
- } catch (err) {
2351
- respondError(ctx, err);
2352
- }
2353
- };
2354
- }
2355
-
2356
- // src/server/handlers/query-suggestions.ts
2357
- var PREVIEW_LIMIT_PATTERN = /^\d{1,7}$/;
2358
- function readObject(ctx) {
2359
- const body = parseJson(ctx);
2360
- if (body === void 0) return void 0;
2361
- if (!isRecord(body)) {
2362
- badRequest(ctx.res, "The body must be a JSON object");
2363
- return void 0;
2354
+ // src/server/handlers/cluster.ts
2355
+ function requireCluster(deps) {
2356
+ if (deps.cluster === void 0) {
2357
+ throw new NarsilError(
2358
+ ErrorCodes.CLUSTER_OPERATION_UNSUPPORTED,
2359
+ "This server fronts a single engine, so it reports no cluster",
2360
+ {}
2361
+ );
2364
2362
  }
2365
- return body;
2366
- }
2367
- function readQuery(ctx, body) {
2368
- if (typeof body.query === "string") return body.query;
2369
- badRequest(ctx.res, 'The body must carry the phrase as a string in "query"');
2370
- return void 0;
2371
- }
2372
- function isStringList(value) {
2373
- return Array.isArray(value) && value.every((entry) => typeof entry === "string");
2374
- }
2375
- function isIdentity(value) {
2376
- return isRecord(value) && typeof value.person === "string" && typeof value.group === "string";
2363
+ return deps.cluster;
2377
2364
  }
2378
- function previewLimitOf(raw) {
2379
- if (raw === null) return void 0;
2380
- return PREVIEW_LIMIT_PATTERN.test(raw) ? Number(raw) : null;
2365
+ function describeNode(cluster) {
2366
+ return { ...cluster.getNodeInfo(), readiness: cluster.getReadiness(), isController: cluster.isControllerActive() };
2381
2367
  }
2382
- async function identityOf(options, ctx) {
2383
- if (ctx.hookContext === null) return void 0;
2368
+ function malformedIndexName(indexName) {
2384
2369
  try {
2385
- const identity = await options.identify(ctx.hookContext);
2386
- if (identity === null) return null;
2387
- return isIdentity(identity) ? { person: identity.person, group: identity.group } : void 0;
2388
- } catch {
2389
- return void 0;
2370
+ validateIndexName(indexName);
2371
+ return null;
2372
+ } catch (error) {
2373
+ return error instanceof Error ? error.message : String(error);
2390
2374
  }
2391
2375
  }
2392
- function noIdentify(indexName) {
2393
- return new NarsilError(
2394
- ErrorCodes.QUERY_SUGGESTIONS_NOT_ENABLED,
2395
- "This server has no identify function for popular query suggestions, so it counts no search",
2396
- { indexName }
2397
- );
2398
- }
2399
- function createQuerySuggestionHandlers(deps, options) {
2400
- const { engine, tasks } = deps;
2401
- const suggestions = () => querySuggestions(engine);
2402
- async function update(ctx) {
2403
- const body = parseJson(ctx);
2404
- if (body === void 0) return;
2376
+ function createClusterHandlers(deps) {
2377
+ async function topology(ctx) {
2405
2378
  try {
2406
- await suggestions().updateSetting(ctx.params[0], body);
2407
- respondJson(ctx, { ok: true });
2379
+ const cluster = requireCluster(deps);
2380
+ const [nodes, controllerNodeId] = await Promise.all([cluster.listNodes(), cluster.getControllerNodeId()]);
2381
+ respondJson(ctx, {
2382
+ node: describeNode(cluster),
2383
+ controllerNodeId,
2384
+ nodes: nodes.map((node) => ({
2385
+ nodeId: node.nodeId,
2386
+ address: node.address,
2387
+ roles: [...node.roles],
2388
+ memory: node.memory ?? null
2389
+ }))
2390
+ });
2408
2391
  } catch (err) {
2409
2392
  respondError(ctx, err);
2410
2393
  }
2411
2394
  }
2412
- async function status(ctx) {
2395
+ async function allocation(ctx) {
2413
2396
  try {
2414
- respondJson(ctx, await suggestions().status(ctx.params[0]));
2397
+ const cluster = requireCluster(deps);
2398
+ const indexName = ctx.params[0] ?? "";
2399
+ const malformed = malformedIndexName(indexName);
2400
+ if (malformed !== null) {
2401
+ badRequest(ctx.res, malformed, { indexName });
2402
+ return;
2403
+ }
2404
+ const table = await cluster.getAllocation(indexName);
2405
+ if (table === null) {
2406
+ respondJson(ctx, {
2407
+ indexName,
2408
+ allocated: false,
2409
+ version: null,
2410
+ partitions: []
2411
+ });
2412
+ return;
2413
+ }
2414
+ const partitions = [];
2415
+ for (const [partitionId, assignment] of table.assignments) {
2416
+ partitions.push({
2417
+ partitionId,
2418
+ state: assignment.state,
2419
+ primary: assignment.primary,
2420
+ primaryTerm: assignment.primaryTerm,
2421
+ commitPoint: assignment.commitPoint,
2422
+ replicas: [...assignment.replicas],
2423
+ inSyncSet: [...assignment.inSyncSet],
2424
+ lastHolders: [...assignment.lastHolders ?? []],
2425
+ unassignedReason: assignment.unassignedReason ?? null
2426
+ });
2427
+ }
2428
+ partitions.sort((left, right) => left.partitionId - right.partitionId);
2429
+ respondJson(ctx, {
2430
+ indexName,
2431
+ allocated: partitions.length > 0,
2432
+ version: table.version,
2433
+ partitions
2434
+ });
2415
2435
  } catch (err) {
2416
2436
  respondError(ctx, err);
2417
2437
  }
2418
2438
  }
2419
- async function record(ctx) {
2420
- const body = readObject(ctx);
2421
- if (body === void 0) return;
2422
- const extra = Object.keys(body).filter((key) => key !== "query");
2423
- if (extra.length > 0) {
2424
- badRequest(ctx.res, 'The body carries the search in "query" alone, because the server identifies the person', {
2425
- fields: extra
2426
- });
2427
- return;
2428
- }
2429
- const query = readQuery(ctx, body);
2430
- if (query === void 0) return;
2431
- const indexName = ctx.params[0];
2439
+ async function replicationFactor(ctx) {
2432
2440
  try {
2433
- if (options === void 0) throw noIdentify(indexName);
2434
- const identity = await identityOf(options, ctx);
2435
- if (identity === void 0) {
2436
- if (!ctx.abort.aborted) {
2437
- sendError(ctx.res, 500, ServerErrorCodes.HOOK_ERROR, "The identify function of query suggestions failed");
2438
- }
2441
+ const cluster = requireCluster(deps);
2442
+ const indexName = ctx.params[0] ?? "";
2443
+ const malformed = malformedIndexName(indexName);
2444
+ if (malformed !== null) {
2445
+ badRequest(ctx.res, malformed, { indexName });
2439
2446
  return;
2440
2447
  }
2441
- if (identity === null) {
2442
- respondJson(ctx, { recorded: false, outcome: "unidentified" });
2448
+ const body = parseJson(ctx);
2449
+ if (body === void 0) return;
2450
+ const factor = isRecord(body) ? body.replicationFactor : void 0;
2451
+ if (typeof factor !== "number") {
2452
+ badRequest(ctx.res, "The body must carry a numeric replicationFactor", { indexName });
2443
2453
  return;
2444
2454
  }
2445
- respondJson(ctx, await suggestions().recordQuery(indexName, { query, ...identity }));
2455
+ await cluster.setReplicationFactor(indexName, factor);
2456
+ respondJson(ctx, { indexName, replicationFactor: factor });
2446
2457
  } catch (err) {
2447
2458
  respondError(ctx, err);
2448
2459
  }
2449
2460
  }
2450
- async function build(ctx) {
2451
- const indexName = ctx.params[0];
2461
+ async function health(ctx) {
2452
2462
  try {
2453
- const operations = suggestions();
2454
- const current = await operations.status(indexName);
2455
- if (current.setting?.enabled !== true) {
2456
- throw new NarsilError(
2457
- ErrorCodes.QUERY_SUGGESTIONS_NOT_ENABLED,
2458
- `Index "${indexName}" has popular query suggestions off`,
2459
- { indexName }
2460
- );
2461
- }
2462
- const task = await tasks.start("querySuggestionsBuild", indexName, async (context) => {
2463
- const finished = await operations.build(indexName);
2464
- context.reportBuild(finished);
2465
- if (finished.status === "failed") {
2466
- throw new NarsilError(
2467
- finished.errorCode ?? ErrorCodes.WORKER_ACTION_FAILED,
2468
- "The build of popular query suggestions failed",
2469
- { indexName, buildId: finished.buildId }
2470
- );
2471
- }
2472
- });
2473
- respondJson(ctx, task, 202);
2463
+ respondJson(ctx, await requireCluster(deps).getHealth());
2474
2464
  } catch (err) {
2475
2465
  respondError(ctx, err);
2476
2466
  }
2477
2467
  }
2478
- async function preview(ctx) {
2479
- const limit = previewLimitOf(ctx.query.get("limit"));
2480
- if (limit === null) {
2481
- badRequest(ctx.res, 'Query parameter "limit" must be a whole number', { value: ctx.query.get("limit") });
2482
- return;
2483
- }
2468
+ return { topology, allocation, replicationFactor, health };
2469
+ }
2470
+
2471
+ // src/server/handlers/backups.ts
2472
+ function stepsOf(context) {
2473
+ return { onProgress: (progress) => context.reportSteps({ done: progress.done, total: progress.total }) };
2474
+ }
2475
+ function restoreOptionsOf(body) {
2476
+ const { indexName } = body;
2477
+ if (typeof indexName !== "string") return 'The body must carry the new index name as a string in "indexName"';
2478
+ const placement = readRestorePlacement(body);
2479
+ return typeof placement === "string" ? placement : { indexName, ...placement };
2480
+ }
2481
+ function readBody(ctx) {
2482
+ const body = parseJson(ctx);
2483
+ if (body === void 0) return void 0;
2484
+ if (!isRecord(body)) {
2485
+ badRequest(ctx.res, "The body must be a JSON object");
2486
+ return void 0;
2487
+ }
2488
+ return body;
2489
+ }
2490
+ function createBackupHandlers(deps) {
2491
+ async function create(ctx) {
2484
2492
  try {
2485
- respondJson(ctx, await suggestions().preview(ctx.params[0], limit));
2493
+ const cluster = requireCluster(deps);
2494
+ const indexName = ctx.params[0] ?? "";
2495
+ const body = readBody(ctx);
2496
+ if (body === void 0) return;
2497
+ if (typeof body.backup !== "string") {
2498
+ badRequest(ctx.res, 'The body must carry the backup name as a string in "backup"', { indexName });
2499
+ return;
2500
+ }
2501
+ if (body.queryHistory !== void 0 && typeof body.queryHistory !== "boolean") {
2502
+ badRequest(ctx.res, 'The body accepts only true or false in "queryHistory"', { indexName });
2503
+ return;
2504
+ }
2505
+ const backupName = validateBackupName(body.backup);
2506
+ const queryHistory = body.queryHistory === true;
2507
+ const record = await deps.tasks.start(
2508
+ "backup",
2509
+ indexName,
2510
+ async (context) => {
2511
+ await cluster.createBackup(indexName, backupName, { ...stepsOf(context), queryHistory });
2512
+ },
2513
+ void 0,
2514
+ backupName
2515
+ );
2516
+ respondJson(ctx, record, 202);
2486
2517
  } catch (err) {
2487
2518
  respondError(ctx, err);
2488
2519
  }
2489
2520
  }
2490
- async function readBans(ctx) {
2521
+ async function list(ctx) {
2491
2522
  try {
2492
- respondJson(ctx, { bans: await suggestions().bans(ctx.params[0]) });
2523
+ const cluster = requireCluster(deps);
2524
+ respondJson(ctx, { backups: await cluster.listBackups() });
2493
2525
  } catch (err) {
2494
2526
  respondError(ctx, err);
2495
2527
  }
2496
2528
  }
2497
- async function replaceBans(ctx) {
2498
- const body = readObject(ctx);
2499
- if (body === void 0) return;
2500
- if (!isStringList(body.bans)) {
2501
- badRequest(ctx.res, 'The body must carry the ban list as an array of strings in "bans"');
2502
- return;
2503
- }
2529
+ async function restore(ctx) {
2504
2530
  try {
2505
- respondJson(ctx, { bans: await suggestions().updateBans(ctx.params[0], body.bans) });
2531
+ const cluster = requireCluster(deps);
2532
+ const backupName = validateBackupName(ctx.params[0]);
2533
+ const body = readBody(ctx);
2534
+ if (body === void 0) return;
2535
+ const options = restoreOptionsOf(body);
2536
+ if (typeof options === "string") {
2537
+ badRequest(ctx.res, options, { backupName });
2538
+ return;
2539
+ }
2540
+ const record = await deps.tasks.start(
2541
+ "backupRestore",
2542
+ options.indexName,
2543
+ async (context) => {
2544
+ await cluster.restoreBackup(backupName, { ...options, ...stepsOf(context) });
2545
+ },
2546
+ void 0,
2547
+ backupName
2548
+ );
2549
+ respondJson(ctx, record, 202);
2506
2550
  } catch (err) {
2507
2551
  respondError(ctx, err);
2508
2552
  }
2509
2553
  }
2510
2554
  async function remove(ctx) {
2511
- const body = readObject(ctx);
2512
- if (body === void 0) return;
2513
- const query = readQuery(ctx, body);
2514
- if (query === void 0) return;
2515
2555
  try {
2516
- const held = await suggestions().removeQuery(ctx.params[0], query);
2517
- respondJson(ctx, { suggestionIndexes: held });
2556
+ const cluster = requireCluster(deps);
2557
+ const backupName = validateBackupName(ctx.params[0]);
2558
+ const record = await deps.tasks.start(
2559
+ "backupDelete",
2560
+ "",
2561
+ async (context) => {
2562
+ await cluster.deleteBackup(backupName, stepsOf(context));
2563
+ },
2564
+ void 0,
2565
+ backupName
2566
+ );
2567
+ respondJson(ctx, record, 202);
2518
2568
  } catch (err) {
2519
2569
  respondError(ctx, err);
2520
2570
  }
2521
2571
  }
2522
- async function manual(ctx) {
2523
- const body = readObject(ctx);
2524
- if (body === void 0) return;
2525
- const query = readQuery(ctx, body);
2526
- if (query === void 0) return;
2527
- const { group, count } = body;
2528
- if (typeof group !== "string" || count !== void 0 && typeof count !== "number") {
2529
- badRequest(ctx.res, 'The body must carry the group as a string in "group", and any count as a number in "count"');
2572
+ return { create, list, restore, remove };
2573
+ }
2574
+
2575
+ // src/server/handlers/capabilities.ts
2576
+ function createCapabilitiesHandler() {
2577
+ function report(ctx) {
2578
+ respondJson(ctx, { capabilities: [...SERVER_CAPABILITIES] });
2579
+ }
2580
+ return { report };
2581
+ }
2582
+ function importResultOf(result, maxErrors) {
2583
+ const errors = result.refused.slice(0, maxErrors);
2584
+ return {
2585
+ indexed: result.imported,
2586
+ failed: result.refusedCount,
2587
+ errors,
2588
+ errorsTruncated: result.refusedCount > errors.length
2589
+ };
2590
+ }
2591
+ function createDocumentExportHandlers(deps) {
2592
+ const { engine, tasks, limits } = deps;
2593
+ async function exportDocuments(ctx) {
2594
+ try {
2595
+ const lines = await engine.exportDocuments(ctx.params[0]);
2596
+ if (ctx.abort.aborted) {
2597
+ await lines.cancel();
2598
+ return;
2599
+ }
2600
+ await sendStream(ctx.res, ctx.abort, lines, [["Content-Type", DOCUMENT_EXPORT_CONTENT_TYPE]]);
2601
+ } catch (err) {
2602
+ respondPreparingError(ctx, err, ErrorCodes.CHANGES_EXPORT_PREPARING);
2603
+ }
2604
+ }
2605
+ async function importExport(ctx) {
2606
+ const spooled = ctx.spooledBody;
2607
+ if (spooled === null || spooled.bytes === 0) {
2608
+ await spooled?.remove();
2609
+ sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
2530
2610
  return;
2531
2611
  }
2612
+ const name = ctx.params[0];
2532
2613
  try {
2533
- const name = await suggestions().addSuggestion(ctx.params[0], {
2534
- group,
2535
- query,
2536
- ...count === void 0 ? {} : { count }
2614
+ const record = await tasks.start("importExport", name, async (context) => {
2615
+ try {
2616
+ const result = await engine.importDocumentExport(name, createReadStream(spooled.path));
2617
+ context.reportResult(importResultOf(result, limits.maxImportErrors));
2618
+ } finally {
2619
+ await spooled.remove();
2620
+ }
2537
2621
  });
2538
- respondJson(ctx, { indexName: name });
2622
+ respondJson(ctx, record, 202);
2539
2623
  } catch (err) {
2624
+ await spooled.remove();
2540
2625
  respondError(ctx, err);
2541
2626
  }
2542
2627
  }
2543
- return { update, status, record, build, preview, readBans, replaceBans, remove, manual };
2628
+ return { exportDocuments, importExport };
2544
2629
  }
2545
2630
 
2546
- // src/server/handlers/search.ts
2547
- function createSearchHandlers(deps) {
2548
- const { engine, limits, allowRegex } = deps;
2549
- async function search(ctx) {
2550
- const params = parseJson(ctx, "read");
2551
- if (!params) return;
2552
- const failure = validateQuery(params, limits);
2631
+ // src/server/handlers/documents.ts
2632
+ function createDocumentHandlers(deps) {
2633
+ const { engine } = deps;
2634
+ const writeBlock = writeBlockLookupOf(engine);
2635
+ const reads = createDocumentReadHandlers(deps);
2636
+ async function insert(ctx) {
2637
+ const body = await parseWriteJsonObject(ctx, writeBlock);
2638
+ if (!body) return;
2639
+ const failure = validateDocumentBody(body);
2553
2640
  if (failure) {
2554
2641
  rejectInvalid(ctx, failure);
2555
2642
  return;
2556
2643
  }
2557
- if (refuseRegex(ctx, params.filters, allowRegex)) return;
2558
2644
  try {
2559
- const query = withServerRerankWindow(params, limits);
2560
- const result = await searchWithSignal(ctx, query);
2561
- await respondAnswer(ctx, encodeQueryResult(result), "hits", `a search on index '${ctx.params[0]}'`);
2645
+ const id = await engine.insert(ctx.params[0], body.document, body.id, body.options);
2646
+ respondJson(ctx, { id }, 201);
2562
2647
  } catch (err) {
2563
2648
  respondError(ctx, err);
2564
2649
  }
2565
2650
  }
2566
- async function searchWithSignal(ctx, params) {
2567
- bindQuerySignal(params, requestSignal(ctx.abort));
2568
- try {
2569
- return await engine.query(ctx.params[0], params);
2570
- } finally {
2571
- clearQuerySignal(params);
2572
- }
2573
- }
2574
- async function preflight(ctx) {
2575
- const params = parseJson(ctx, "read");
2576
- if (!params) return;
2577
- const failure = validateQuery(params, limits);
2651
+ async function put(ctx) {
2652
+ const body = await parseWriteJsonObject(ctx, writeBlock);
2653
+ if (!body) return;
2654
+ const failure = validateDocumentBody(body);
2578
2655
  if (failure) {
2579
2656
  rejectInvalid(ctx, failure);
2580
2657
  return;
2581
2658
  }
2582
- if (refuseRegex(ctx, params.filters, allowRegex)) return;
2659
+ const [name, id] = ctx.params;
2583
2660
  try {
2584
- bindQuerySignal(params, requestSignal(ctx.abort));
2585
- try {
2586
- respondJson(ctx, await engine.preflight(ctx.params[0], params));
2587
- } finally {
2588
- clearQuerySignal(params);
2661
+ if (await engine.has(name, id)) {
2662
+ await engine.update(name, id, body.document, body.options);
2663
+ respondJson(ctx, { id, created: false });
2664
+ } else {
2665
+ const created = await engine.insert(name, body.document, id, body.options);
2666
+ respondJson(ctx, { id: created, created: true }, 201);
2589
2667
  }
2590
2668
  } catch (err) {
2591
2669
  respondError(ctx, err);
2592
2670
  }
2593
2671
  }
2594
- async function suggest(ctx) {
2595
- const params = parseJson(ctx, "read");
2596
- if (!params) return;
2597
- const failure = validateSuggest(params);
2672
+ async function patch(ctx) {
2673
+ const body = await parseWriteJsonObject(ctx, writeBlock);
2674
+ if (!body) return;
2675
+ const failure = validateDocumentBody(body);
2598
2676
  if (failure) {
2599
2677
  rejectInvalid(ctx, failure);
2600
2678
  return;
2601
2679
  }
2602
2680
  try {
2603
- respondJson(ctx, await engine.suggest(ctx.params[0], params));
2681
+ await engine.update(ctx.params[0], ctx.params[1], body.document, body.options);
2682
+ respondJson(ctx, { id: ctx.params[1] });
2604
2683
  } catch (err) {
2605
2684
  respondError(ctx, err);
2606
2685
  }
2607
2686
  }
2608
- return { search, preflight, suggest };
2609
- }
2610
-
2611
- // src/server/options-validation.ts
2612
- var ENGINE_METHOD_SAMPLE = ["createIndex", "listIndexes", "insert", "query", "get", "shutdown"];
2613
- var LIMITS_UNBOUNDED_AT_ZERO = [
2614
- "maxBodyBytes",
2615
- "maxImportBytes",
2616
- "maxRestoreBytes",
2617
- "maxLineBytes",
2618
- "maxImportErrors",
2619
- "maxConcurrentRequests",
2620
- "maxConcurrentTasks"
2621
- ];
2622
- var COUNTED_LIMITS = [
2623
- "importBatchSize",
2624
- "maxResultWindow",
2625
- "maxRerankWindow",
2626
- "maxFetchDocuments",
2627
- "maxTaskPageSize"
2628
- ];
2629
- function assertServerEngine(engine) {
2630
- const candidate = engine;
2631
- if (typeof candidate !== "object" || candidate === null) {
2632
- throw new NarsilError(
2633
- ErrorCodes.CONFIG_INVALID,
2634
- "The first argument to createServer must be the engine that createNarsil returns, with the options second",
2635
- { received: candidate === null ? "null" : typeof candidate }
2636
- );
2687
+ async function remove(ctx) {
2688
+ try {
2689
+ await engine.remove(ctx.params[0], ctx.params[1], { wait: ctx.query.get("wait") === "true" });
2690
+ respondJson(ctx, { id: ctx.params[1], removed: true });
2691
+ } catch (err) {
2692
+ respondError(ctx, err);
2693
+ }
2637
2694
  }
2638
- const members = candidate;
2639
- const missing = ENGINE_METHOD_SAMPLE.filter((method) => typeof members[method] !== "function");
2640
- if (missing.length === 0) return;
2641
- throw new NarsilError(
2642
- ErrorCodes.CONFIG_INVALID,
2643
- `The first argument to createServer offers no ${missing.join(", ")}, so it is some value other than the engine that createNarsil returns`,
2644
- { missing }
2645
- );
2646
- }
2647
- function assertLimit(field, value, minimum) {
2648
- if (value === void 0) return;
2649
- if (Number.isInteger(value) && value >= minimum) return;
2650
- throw new NarsilError(
2651
- ErrorCodes.CONFIG_INVALID,
2652
- `limits.${field} must be a whole number of at least ${minimum}; however, this configuration sets it to ${String(value)}`,
2653
- { field, value, minimum }
2654
- );
2655
- }
2656
- function assertFraction(field, value) {
2657
- if (value === void 0) return;
2658
- if (Number.isFinite(value) && value >= 0 && value <= MAX_BODY_BYTES_IN_FLIGHT_FRACTION) return;
2659
- throw new NarsilError(
2660
- ErrorCodes.CONFIG_INVALID,
2661
- `limits.${field} must be a fraction from 0 to ${MAX_BODY_BYTES_IN_FLIGHT_FRACTION}; however, this configuration sets it to ${String(value)}`,
2662
- { field, value, maximum: MAX_BODY_BYTES_IN_FLIGHT_FRACTION }
2663
- );
2664
- }
2665
- function assertRerankWindowCap(value, engineCap) {
2666
- if (value === void 0) return;
2667
- const cap = Math.min(MAX_RERANK_WINDOW, engineCap ?? MAX_RERANK_WINDOW);
2668
- if (value <= cap) return;
2669
- throw new NarsilError(
2670
- ErrorCodes.CONFIG_INVALID,
2671
- `limits.maxRerankWindow must be at most ${cap}, the largest window that the engine accepts; however, this configuration sets it to ${value}`,
2672
- { field: "maxRerankWindow", value, maximum: cap }
2673
- );
2674
- }
2675
- function assertServerLimits(limits, engineRerankWindow) {
2676
- if (limits === void 0) return;
2677
- for (const field of LIMITS_UNBOUNDED_AT_ZERO) assertLimit(field, limits[field], MIN_UNBOUNDED_SERVER_LIMIT);
2678
- for (const field of COUNTED_LIMITS) assertLimit(field, limits[field], MIN_COUNTED_SERVER_LIMIT);
2679
- assertFraction("maxBodyBytesInFlightFraction", limits.maxBodyBytesInFlightFraction);
2680
- assertRerankWindowCap(limits.maxRerankWindow, engineRerankWindow);
2681
- }
2682
- function assertQuerySuggestionsOptions(options) {
2683
- if (options === void 0) return;
2684
- const identify = typeof options === "object" && options !== null ? Reflect.get(options, "identify") : void 0;
2685
- if (typeof identify === "function") return;
2686
- throw new NarsilError(
2687
- ErrorCodes.CONFIG_INVALID,
2688
- "querySuggestions must carry an identify function that returns the person and the group of a request",
2689
- { field: "querySuggestions.identify" }
2690
- );
2691
- }
2692
- function assertAllowRegex(allowRegex) {
2693
- if (allowRegex === void 0 || typeof allowRegex === "boolean") return;
2694
- throw new NarsilError(
2695
- ErrorCodes.CONFIG_INVALID,
2696
- `allowRegex must be true or false; however, this configuration sets it to ${String(allowRegex)}`,
2697
- { field: "allowRegex", value: String(allowRegex) }
2698
- );
2699
- }
2700
-
2701
- // package.json
2702
- var version = "0.4.1";
2703
-
2704
- // src/server/package-version.ts
2705
- function publishedPackageVersion() {
2706
- return version;
2707
- }
2708
-
2709
- // src/server/request-threads/relay-answers.ts
2710
- async function answerAuthorize(authorize, request) {
2711
- if (authorize === void 0) {
2712
- return { type: "authorized", requestId: request.requestId, denial: null, hookError: false, expiresAt: null };
2695
+ async function waitForWrites(ctx) {
2696
+ try {
2697
+ await engine.waitForWrites(ctx.params[0]);
2698
+ respondJson(ctx, { ok: true });
2699
+ } catch (err) {
2700
+ respondError(ctx, err);
2701
+ }
2713
2702
  }
2714
- const authorization = await authorize(request.context);
2715
- if (authorization.allowed) {
2716
- return {
2717
- type: "authorized",
2718
- requestId: request.requestId,
2719
- denial: null,
2720
- hookError: false,
2721
- expiresAt: authorization.expiresAt
2722
- };
2703
+ async function batch(ctx) {
2704
+ const body = parseJsonObject(ctx, ctx.rawBody && isRemoveOnlyBatch(ctx.rawBody) ? "read" : "write");
2705
+ if (!body) return;
2706
+ const failure = validateBatch(body);
2707
+ if (failure) {
2708
+ rejectInvalid(ctx, failure);
2709
+ return;
2710
+ }
2711
+ const action = body.action ?? "insert";
2712
+ try {
2713
+ if (action === "update") {
2714
+ respondJson(
2715
+ ctx,
2716
+ serializeBatchResult(await engine.updateBatch(ctx.params[0], body.updates ?? [], body.options))
2717
+ );
2718
+ } else if (action === "delete") {
2719
+ respondJson(ctx, serializeBatchResult(await engine.removeBatch(ctx.params[0], body.docIds ?? [], body.options)));
2720
+ } else {
2721
+ respondJson(
2722
+ ctx,
2723
+ serializeBatchResult(await engine.insertBatch(ctx.params[0], body.documents ?? [], body.options))
2724
+ );
2725
+ }
2726
+ } catch (err) {
2727
+ respondError(ctx, err);
2728
+ }
2723
2729
  }
2724
- return {
2725
- type: "authorized",
2726
- requestId: request.requestId,
2727
- denial: authorization.denial,
2728
- hookError: authorization.denial === null,
2729
- expiresAt: null
2730
- };
2730
+ return { ...reads, insert, put, patch, remove, batch, waitForWrites };
2731
2731
  }
2732
- async function answerEmbed(core, request, signal) {
2733
- try {
2734
- const entry = core.requireIndex(request.indexName);
2735
- const resolved = await resolveVectorText(
2736
- { vector: request.vector },
2737
- entry.embeddingAdapter,
2738
- signal === void 0 ? core.abortController.signal : AbortSignal.any([core.abortController.signal, signal]),
2739
- entry.embeddingAdapterName
2740
- );
2741
- const value = resolved.vector?.value;
2742
- return {
2743
- type: "embedded",
2744
- requestId: request.requestId,
2745
- value: value === void 0 ? null : Array.from(value),
2746
- error: null
2747
- };
2748
- } catch (err) {
2749
- if (!isNarsilError(err)) console.warn("Embedding a query for a request thread failed:", err);
2750
- return { type: "embedded", requestId: request.requestId, value: null, error: toHttpError(err).body };
2732
+
2733
+ // src/server/handlers/health.ts
2734
+ function createHealthHandlers(deps) {
2735
+ function livez(ctx) {
2736
+ respondJson(ctx, { status: "ok" });
2751
2737
  }
2752
- }
2753
- async function answerRerank(core, request, signal) {
2754
- try {
2755
- const { rerankOnHost } = await loadRerankRuntime();
2756
- const scores = await rerankOnHost(core, request.adapter, request.query, request.documents, signal);
2757
- return { type: "reranked", requestId: request.requestId, scores, error: null };
2758
- } catch (err) {
2759
- if (!isNarsilError(err)) console.warn("Reranking a query for a request thread failed:", err);
2760
- return { type: "reranked", requestId: request.requestId, scores: null, error: toHttpError(err).body };
2738
+ function readyz(ctx) {
2739
+ const cluster = deps.cluster;
2740
+ if (cluster === void 0) {
2741
+ if (deps.isReady()) {
2742
+ respondJson(ctx, { status: "ready" });
2743
+ } else {
2744
+ respondJson(ctx, { status: "unavailable" }, 503);
2745
+ }
2746
+ return;
2747
+ }
2748
+ const readiness = cluster.getReadiness();
2749
+ const ready = deps.isReady() && readiness === "SERVING";
2750
+ respondJson(
2751
+ ctx,
2752
+ {
2753
+ status: ready ? "ready" : "unavailable",
2754
+ cluster: describeNode(cluster)
2755
+ },
2756
+ ready ? 200 : 503
2757
+ );
2761
2758
  }
2762
- }
2763
- async function answerAwaitWrites(core, request) {
2764
- await core.orchestrator.awaitWrites(request.indexName).catch(() => void 0);
2765
- return { type: "writes-applied", requestId: request.requestId };
2766
- }
2767
- async function answerFeedAdmit(feeds, request) {
2768
- if (feeds !== null) return feeds.admit(request);
2769
- return {
2770
- type: "feed-admitted",
2771
- requestId: request.requestId,
2772
- feedId: null,
2773
- error: { code: ServerErrorCodes.NOT_FOUND, message: "Route not found" }
2774
- };
2759
+ return { livez, readyz };
2775
2760
  }
2776
2761
 
2777
- // src/server/request-threads/host.ts
2778
- function threadClosed(port, executor, clearSlots) {
2779
- return new Promise((resolve) => {
2780
- port.once("close", () => {
2781
- clearSlots();
2782
- resolve();
2762
+ // src/server/ndjson.ts
2763
+ var NEWLINE = 10;
2764
+ var CARRIAGE_RETURN = 13;
2765
+ var NdjsonLineTooLongError = class extends NarsilError {
2766
+ constructor(lineNumber) {
2767
+ super(ServerErrorCodes.PAYLOAD_TOO_LARGE, `NDJSON line ${lineNumber} exceeds the configured per-line size limit`, {
2768
+ line: lineNumber
2783
2769
  });
2784
- void executor.whenAbandoned?.().then(resolve);
2785
- });
2770
+ this.name = "NdjsonLineTooLongError";
2771
+ }
2772
+ };
2773
+ function* iterateNdjson(buffer, maxLineBytes) {
2774
+ let start = 0;
2775
+ let lineNumber = 0;
2776
+ const length = buffer.length;
2777
+ while (start < length) {
2778
+ let end = buffer.indexOf(NEWLINE, start);
2779
+ if (end === -1) end = length;
2780
+ if (end - start > maxLineBytes) throw new NdjsonLineTooLongError(lineNumber + 1);
2781
+ let contentEnd = end;
2782
+ if (contentEnd > start && buffer[contentEnd - 1] === CARRIAGE_RETURN) contentEnd--;
2783
+ if (contentEnd > start) {
2784
+ lineNumber++;
2785
+ yield { lineNumber, bytes: buffer.subarray(start, contentEnd), bytesConsumed: Math.min(end + 1, length) };
2786
+ }
2787
+ start = end + 1;
2788
+ }
2786
2789
  }
2787
- var RELAYED_TYPES = /* @__PURE__ */ new Set([
2788
- "request",
2789
- "authorize",
2790
- "embed",
2791
- "rerank",
2792
- "await-writes",
2793
- "stream-credit",
2794
- "stream-cancel",
2795
- "feed-admit"
2796
- ]);
2797
- function isRelayRequest(message) {
2798
- if (typeof message !== "object" || message === null) return false;
2799
- const { type, requestId } = message;
2800
- if (type === "touch" || type === "leaving") return true;
2801
- return typeof requestId === "number" && typeof type === "string" && RELAYED_TYPES.has(type);
2790
+
2791
+ // src/server/handlers/import.ts
2792
+ function yieldToEventLoop() {
2793
+ return new Promise((resolve) => setImmediate(resolve));
2802
2794
  }
2803
- function touchIntervalFor(copyIdleTimeoutMs) {
2804
- return Math.min(MAX_INDEX_TOUCH_INTERVAL_MS, Math.floor(copyIdleTimeoutMs / INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT));
2795
+ function isPlainObject(value) {
2796
+ return typeof value === "object" && value !== null && !Array.isArray(value);
2805
2797
  }
2806
- async function startRequestThreads(options) {
2807
- const found = acceptorOf(options.app);
2808
- if (found === null) return null;
2809
- const acceptor = found;
2810
- const { core, handlers, authorize } = options;
2811
- const searchHooks = core.pluginRegistry.hasHooks("beforeSearch") || core.pluginRegistry.hasHooks("afterSearch");
2812
- const touchIntervalMs = touchIntervalFor(core.orchestrator.copyIdleTimeoutMs());
2813
- const threads = /* @__PURE__ */ new Map();
2814
- const slotUsers = /* @__PURE__ */ new Map();
2815
- const longQueriesInFlight = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT);
2816
- let longQueryThread = null;
2817
- const closeAtExit = /* @__PURE__ */ new Set();
2818
- const retiring = /* @__PURE__ */ new Set();
2819
- let stopped = false;
2820
- function closeThreadsAtExit() {
2821
- closeEveryThreadAtExit(closeAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS);
2822
- }
2823
- function releaseDescriptor(thread) {
2824
- thread.feeds?.closeAll();
2825
- closeAtExit.delete(thread.closeAtExit);
2826
- if (thread.descriptor !== null) {
2827
- try {
2828
- acceptor.removeChildAppDescriptor(thread.descriptor);
2829
- } catch {
2830
- }
2831
- thread.descriptor = null;
2798
+ async function runImport(engine, options) {
2799
+ const { indexName, body, maxLineBytes, batchSize, maxErrors, signal, onProgress } = options;
2800
+ const bytesTotal = body.length;
2801
+ const writeBlock = writeBlockLookupOf(engine);
2802
+ const errors = [];
2803
+ let indexed = 0;
2804
+ let failed = 0;
2805
+ let bytesProcessed = 0;
2806
+ let pending = [];
2807
+ const recordFailure = (error) => {
2808
+ failed += 1;
2809
+ if (errors.length < maxErrors) errors.push(error);
2810
+ };
2811
+ const flushDocuments = async (documents) => {
2812
+ const result = await engine.insertBatch(indexName, documents, { skipClone: true });
2813
+ indexed += result.succeeded.length;
2814
+ for (const failure of result.failed) {
2815
+ const serialized = failure.error instanceof NarsilError ? serializeNarsilError(failure.error) : { code: ServerErrorCodes.INTERNAL_ERROR, message: String(failure.error) };
2816
+ recordFailure({
2817
+ docId: failure.docId,
2818
+ code: serialized.code,
2819
+ message: serialized.message,
2820
+ ...serialized.details === void 0 ? {} : { details: serialized.details }
2821
+ });
2832
2822
  }
2833
- confirmDeparture(thread.departure);
2834
- }
2835
- function relayFrom(thread) {
2836
- const { port } = thread;
2837
- const streams = /* @__PURE__ */ new Map();
2838
- const calls = /* @__PURE__ */ new Map();
2839
- port.on("close", () => {
2840
- thread.feeds?.closeAll();
2841
- for (const stream of streams.values()) stream.cancel();
2842
- for (const call of calls.values()) call.abort();
2843
- calls.clear();
2844
- });
2845
- port.on("message", (message) => {
2846
- if (isThreadFeedNote(message)) {
2847
- thread.feeds?.handle(message);
2848
- return;
2849
- }
2850
- if (!isRelayRequest(message)) return;
2851
- if (message.type === "leaving") {
2852
- releaseDescriptor(thread);
2853
- return;
2854
- }
2855
- if (message.type === "touch") {
2856
- core.orchestrator.noteAccess(message.indexName);
2857
- return;
2858
- }
2859
- if (message.type === "stream-credit") {
2860
- streams.get(message.requestId)?.credit();
2861
- return;
2862
- }
2863
- if (message.type === "stream-cancel") {
2864
- streams.get(message.requestId)?.cancel();
2865
- calls.get(message.requestId)?.abort();
2866
- calls.delete(message.requestId);
2867
- return;
2868
- }
2869
- const call = message.type === "embed" || message.type === "rerank" ? new AbortController() : void 0;
2870
- if (call !== void 0) calls.set(message.requestId, call);
2871
- const reply = message.type === "request" ? answerRequest(handlers[message.route], message, port, streams) : message.type === "authorize" ? answerAuthorize(authorize, message) : message.type === "feed-admit" ? answerFeedAdmit(thread.feeds, message) : message.type === "embed" ? answerEmbed(core, message, call?.signal) : message.type === "await-writes" ? answerAwaitWrites(core, message) : answerRerank(core, message, call?.signal);
2872
- void reply.then(
2873
- (answer) => {
2874
- calls.delete(message.requestId);
2875
- if (answer !== null) port.postMessage(answer);
2876
- },
2877
- () => {
2878
- calls.delete(message.requestId);
2879
- }
2880
- );
2881
- });
2882
- }
2883
- function connectToLongQueries(thread) {
2884
- if (longQueryThread === null) return;
2885
- const channel = new MessageChannel();
2886
- thread.port.postMessage({ type: "long-query-port", port: channel.port1 }, [channel.port1]);
2887
- longQueryThread.port.postMessage({ type: "long-query-port", port: channel.port2 }, [channel.port2]);
2888
- }
2889
- function dropLongQueryThread() {
2890
- if (longQueryThread === null) return;
2891
- longQueryThread.port.close();
2892
- longQueryThread = null;
2893
- }
2894
- function retireThread(workerId, thread) {
2895
- releaseDescriptor(thread);
2896
- thread.port.unref();
2897
- retiring.add(thread.closed);
2898
- void thread.closed.then(() => retiring.delete(thread.closed));
2899
- void executeUntilAnswered(thread.executor, { type: "stopServing", requestId: `stop-serving-${workerId}` }).catch(
2900
- () => void 0
2901
- );
2902
- return thread.closed;
2903
- }
2904
- const listener = {
2905
- async onLongQueryWorkerReady(workerId, executor) {
2906
- if (stopped) return;
2907
- dropLongQueryThread();
2908
- const channel = new MessageChannel();
2909
- const relaying = {
2910
- feeds: null,
2911
- descriptor: null,
2912
- port: channel.port2,
2913
- executor,
2914
- departure: createDepartureWord(),
2915
- closeAtExit: createCloseAtExitWords(),
2916
- closed: new Promise((resolve) => channel.port2.once("close", resolve))
2917
- };
2918
- relayFrom(relaying);
2919
- try {
2920
- await executor.execute(
2921
- {
2922
- type: "serveLongQueries",
2923
- requestId: `serve-long-queries-${workerId}`,
2924
- settings: {
2925
- port: channel.port1,
2926
- touchIntervalMs,
2927
- limits: options.limits,
2928
- patternWorkCap: options.core.patternWorkCap,
2929
- maxRerankWindow: options.core.rerankSettings.maxWindow,
2930
- allowRegex: options.allowRegex
2931
- }
2932
- },
2933
- [channel.port1]
2934
- );
2935
- } catch (err) {
2936
- channel.port2.close();
2937
- console.warn(`Worker ${workerId} could not start answering long queries:`, err);
2938
- return;
2939
- }
2940
- if (stopped) {
2941
- channel.port2.close();
2942
- return;
2943
- }
2944
- longQueryThread = { workerId, port: channel.port2, executor };
2945
- for (const thread of threads.values()) connectToLongQueries(thread);
2946
- },
2947
- async onWorkerReady(workerId, executor) {
2948
- if (stopped) return;
2949
- const channel = new MessageChannel();
2950
- const slot = gateSlotOfWorker(workerId);
2951
- slotUsers.set(slot, (slotUsers.get(slot) ?? 0) + 1);
2952
- const thread = {
2953
- feeds: options.feeds?.forThread(channel.port2) ?? null,
2954
- descriptor: null,
2955
- port: channel.port2,
2956
- executor,
2957
- departure: createDepartureWord(),
2958
- closeAtExit: createCloseAtExitWords(),
2959
- closed: threadClosed(channel.port2, executor, () => {
2960
- const remaining = (slotUsers.get(slot) ?? 1) - 1;
2961
- if (remaining > 0) {
2962
- slotUsers.set(slot, remaining);
2963
- return;
2964
- }
2965
- slotUsers.delete(slot);
2966
- clearGateSlot(options.gate, slot);
2967
- clearBodyBudgetSlot(options.bodyBudget, slot);
2968
- })
2969
- };
2970
- relayFrom(thread);
2971
- closeAtExit.add(thread.closeAtExit);
2972
- const longQueries = longQueryThread;
2973
- const longQueryChannel = longQueries === null ? null : new MessageChannel();
2974
- const transfer = longQueryChannel === null ? [channel.port1] : [channel.port1, longQueryChannel.port1];
2975
- let result;
2976
- try {
2977
- result = await executor.execute(
2978
- {
2979
- type: "serveRequests",
2980
- requestId: `serve-${workerId}`,
2981
- settings: {
2982
- port: channel.port1,
2983
- departure: thread.departure.buffer,
2984
- closeAtExit: thread.closeAtExit.buffer,
2985
- gate: options.gate,
2986
- gateSlot: gateSlotOfWorker(workerId),
2987
- maxConcurrentRequests: options.limits.maxConcurrentRequests,
2988
- bodyBudget: options.bodyBudget,
2989
- touchIntervalMs,
2990
- cors: options.cors,
2991
- limits: options.limits,
2992
- build: options.build,
2993
- authorizes: authorize !== void 0,
2994
- searchHooks,
2995
- patternWorkCap: options.core.patternWorkCap,
2996
- maxRerankWindow: options.core.rerankSettings.maxWindow,
2997
- allowRegex: options.allowRegex,
2998
- feeds: Object.keys(options.feeds?.gateways ?? {}),
2999
- longQueriesInFlight,
3000
- longQueryPort: longQueryChannel?.port1 ?? null
3001
- }
3002
- },
3003
- transfer
2823
+ };
2824
+ const inFlight = /* @__PURE__ */ new Set();
2825
+ let flushError;
2826
+ const startFlush = () => {
2827
+ if (pending.length === 0) return;
2828
+ const documents = pending.map((entry) => entry.document);
2829
+ pending = [];
2830
+ const run = flushDocuments(documents).catch((err) => {
2831
+ flushError ??= err;
2832
+ }).finally(() => inFlight.delete(run));
2833
+ inFlight.add(run);
2834
+ };
2835
+ const settleFlushes = async () => {
2836
+ await Promise.all(inFlight);
2837
+ if (flushError !== void 0) throw flushError;
2838
+ };
2839
+ const report = () => {
2840
+ onProgress?.({ indexed, failed, bytesProcessed, bytesTotal });
2841
+ };
2842
+ const seenIds = /* @__PURE__ */ new Set();
2843
+ const repeated = [];
2844
+ try {
2845
+ for (const line of iterateNdjson(body, maxLineBytes)) {
2846
+ bytesProcessed = line.bytesConsumed;
2847
+ const refusal = parseRefusalOf(line.bytes);
2848
+ if (refusal !== null) {
2849
+ const block = refusal.status === 429 && writeBlock !== void 0 ? await writeBlock(indexName).catch(() => null) : null;
2850
+ recordFailure(
2851
+ block === null ? { line: line.lineNumber, code: refusal.code, message: refusal.message, details: refusal.details } : { line: line.lineNumber, ...serializeNarsilError(block) }
3004
2852
  );
3005
- } catch (err) {
3006
- closeAtExit.delete(thread.closeAtExit);
3007
- channel.port2.close();
3008
- longQueryChannel?.port2.close();
3009
- console.warn(`Worker ${workerId} could not start taking requests:`, err);
3010
- return;
2853
+ continue;
3011
2854
  }
3012
- thread.descriptor = result.descriptor;
3013
- if (stopped || departureConfirmed(thread.departure)) {
3014
- longQueryChannel?.port2.close();
3015
- await retireThread(workerId, thread);
3016
- return;
2855
+ let parsed;
2856
+ try {
2857
+ parsed = JSON.parse(line.bytes.toString("utf8"));
2858
+ } catch {
2859
+ recordFailure({ line: line.lineNumber, code: ServerErrorCodes.INVALID_JSON, message: "Line is not valid JSON" });
2860
+ continue;
3017
2861
  }
3018
- if (longQueryChannel !== null && longQueries !== null && longQueryThread === longQueries) {
3019
- longQueries.port.postMessage({ type: "long-query-port", port: longQueryChannel.port2 }, [
3020
- longQueryChannel.port2
3021
- ]);
3022
- } else {
3023
- longQueryChannel?.port2.close();
3024
- connectToLongQueries(thread);
2862
+ if (!isPlainObject(parsed)) {
2863
+ recordFailure({
2864
+ line: line.lineNumber,
2865
+ code: ServerErrorCodes.INVALID_REQUEST,
2866
+ message: "Line is not a JSON object"
2867
+ });
2868
+ continue;
3025
2869
  }
3026
- acceptor.addChildAppDescriptor(result.descriptor);
3027
- threads.set(workerId, thread);
3028
- },
3029
- onWorkerGone(workerId) {
3030
- if (longQueryThread?.workerId === workerId) {
3031
- dropLongQueryThread();
2870
+ const docId = providedDocId(parsed);
2871
+ if (docId !== void 0) {
2872
+ if (seenIds.has(docId)) {
2873
+ repeated.push({ document: parsed, line: line.lineNumber });
2874
+ continue;
2875
+ }
2876
+ seenIds.add(docId);
2877
+ }
2878
+ pending.push({ document: parsed, line: line.lineNumber });
2879
+ if (pending.length >= batchSize) {
2880
+ signal?.throwIfAborted();
2881
+ startFlush();
2882
+ if (inFlight.size >= IMPORT_FLUSHES_IN_FLIGHT) await Promise.race(inFlight);
2883
+ if (flushError !== void 0) await settleFlushes();
2884
+ report();
2885
+ await yieldToEventLoop();
2886
+ signal?.throwIfAborted();
2887
+ }
2888
+ }
2889
+ startFlush();
2890
+ await settleFlushes();
2891
+ for (let start = 0; start < repeated.length; start += batchSize) {
2892
+ signal?.throwIfAborted();
2893
+ await flushDocuments(repeated.slice(start, start + batchSize).map((entry) => entry.document));
2894
+ }
2895
+ } catch (err) {
2896
+ await Promise.allSettled(inFlight);
2897
+ report();
2898
+ throw err;
2899
+ }
2900
+ bytesProcessed = bytesTotal;
2901
+ report();
2902
+ return { indexed, failed, errors, errorsTruncated: failed > errors.length };
2903
+ }
2904
+ function createImportHandler(deps) {
2905
+ const { engine, limits, tasks } = deps;
2906
+ return async function importNdjson(ctx) {
2907
+ const raw = ctx.rawBody;
2908
+ if (!raw || raw.length === 0) {
2909
+ sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
2910
+ return;
2911
+ }
2912
+ const indexName = ctx.params[0];
2913
+ const runOptions = {
2914
+ indexName,
2915
+ body: raw,
2916
+ maxLineBytes: limits.maxLineBytes,
2917
+ batchSize: limits.importBatchSize,
2918
+ maxErrors: limits.maxImportErrors
2919
+ };
2920
+ if (ctx.query.get("async") === "true") {
2921
+ try {
2922
+ await engine.countDocuments(indexName);
2923
+ } catch (err) {
2924
+ respondError(ctx, err);
3032
2925
  return;
3033
2926
  }
3034
- const thread = threads.get(workerId);
3035
- if (thread === void 0) return;
3036
- threads.delete(workerId);
3037
- void retireThread(workerId, thread);
2927
+ const record = await tasks.start(
2928
+ "import",
2929
+ indexName,
2930
+ async (context) => {
2931
+ const result = await runImport(engine, {
2932
+ ...runOptions,
2933
+ signal: context.signal,
2934
+ onProgress: context.reportProgress
2935
+ });
2936
+ context.reportResult(result);
2937
+ },
2938
+ { indexed: 0, failed: 0, bytesProcessed: 0, bytesTotal: raw.length }
2939
+ );
2940
+ respondJson(ctx, record, 202);
2941
+ return;
2942
+ }
2943
+ const controller = new AbortController();
2944
+ ctx.abort.onAbort(() => controller.abort());
2945
+ try {
2946
+ respondJson(ctx, await runImport(engine, { ...runOptions, signal: controller.signal }));
2947
+ } catch (err) {
2948
+ if (ctx.abort.aborted) return;
2949
+ respondError(ctx, err);
3038
2950
  }
3039
2951
  };
3040
- process.on("exit", closeThreadsAtExit);
3041
- await core.orchestrator.serveRequestsOnWorkers(listener);
3042
- return {
3043
- get threadCount() {
3044
- return threads.size;
3045
- },
3046
- async stop() {
3047
- stopped = true;
3048
- process.off("exit", closeThreadsAtExit);
3049
- const stopping = [...threads.entries()].map(([workerId, thread]) => {
3050
- threads.delete(workerId);
3051
- return retireThread(workerId, thread);
2952
+ }
2953
+
2954
+ // src/server/config-mapping.ts
2955
+ function mapHttpIndexConfig(http) {
2956
+ if (typeof http !== "object" || http === null) {
2957
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index config must be an object");
2958
+ }
2959
+ if (typeof http.schema !== "object" || http.schema === null || Array.isArray(http.schema)) {
2960
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Index config requires a "schema" object');
2961
+ }
2962
+ const config = { schema: http.schema };
2963
+ if (http.language !== void 0) config.language = http.language;
2964
+ if (http.partitions !== void 0) config.partitions = http.partitions;
2965
+ if (http.defaultScoring !== void 0) config.defaultScoring = http.defaultScoring;
2966
+ if (http.bm25 !== void 0) config.bm25 = http.bm25;
2967
+ if (http.trackPositions !== void 0) config.trackPositions = http.trackPositions;
2968
+ if (http.textStorage !== void 0) config.textStorage = http.textStorage;
2969
+ if (http.surfaceForms !== void 0) config.surfaceForms = http.surfaceForms === true;
2970
+ if (http.vectorPromotion !== void 0) config.vectorPromotion = http.vectorPromotion;
2971
+ if (http.strict !== void 0) config.strict = http.strict;
2972
+ if (http.required !== void 0) config.required = http.required;
2973
+ if (http.patternValueLimit !== void 0) config.patternValueLimit = http.patternValueLimit;
2974
+ if (http.synonyms !== void 0) config.synonyms = http.synonyms;
2975
+ if (http.tokenizer !== void 0) {
2976
+ if (typeof http.tokenizer !== "string") {
2977
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Field "tokenizer" must be a registered name');
2978
+ }
2979
+ config.tokenizer = http.tokenizer;
2980
+ }
2981
+ if (typeof http.stopWords === "string") {
2982
+ config.stopWords = http.stopWords;
2983
+ } else if (http.stopWords !== void 0) {
2984
+ if (!Array.isArray(http.stopWords) || http.stopWords.some((w) => typeof w !== "string")) {
2985
+ throw new NarsilError(
2986
+ ErrorCodes.CONFIG_INVALID,
2987
+ 'Field "stopWords" must be a registered name or an array of strings'
2988
+ );
2989
+ }
2990
+ config.stopWords = new Set(http.stopWords);
2991
+ }
2992
+ if (http.embedding !== void 0) {
2993
+ config.embedding = mapEmbedding(http.embedding);
2994
+ }
2995
+ return config;
2996
+ }
2997
+ function mapEmbedding(embedding) {
2998
+ if (typeof embedding !== "object" || embedding === null) {
2999
+ throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, 'Field "embedding" must be an object');
3000
+ }
3001
+ if (typeof embedding.fields !== "object" || embedding.fields === null) {
3002
+ throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, 'Field "embedding.fields" is required');
3003
+ }
3004
+ const mapped = { fields: embedding.fields };
3005
+ if (embedding.adapter !== void 0) {
3006
+ if (typeof embedding.adapter !== "string" || embedding.adapter.length === 0) {
3007
+ throw new NarsilError(
3008
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
3009
+ 'Field "embedding.adapter" must be a non-empty adapter name'
3010
+ );
3011
+ }
3012
+ mapped.adapter = embedding.adapter;
3013
+ }
3014
+ return mapped;
3015
+ }
3016
+
3017
+ // src/server/handlers/indexes.ts
3018
+ function createIndexHandlers(deps) {
3019
+ const { engine } = deps;
3020
+ async function create(ctx) {
3021
+ const body = parseJsonObject(ctx);
3022
+ if (!body) return;
3023
+ const failure = validateCreateIndex(body);
3024
+ if (failure) {
3025
+ rejectInvalid(ctx, failure);
3026
+ return;
3027
+ }
3028
+ try {
3029
+ const config = mapHttpIndexConfig(body.config);
3030
+ await engine.createIndex(body.name, config);
3031
+ respondJson(ctx, { name: body.name }, 201);
3032
+ } catch (err) {
3033
+ respondError(ctx, err);
3034
+ }
3035
+ }
3036
+ async function list(ctx) {
3037
+ try {
3038
+ respondJson(ctx, { indexes: await indexReadsOf(engine).listIndexes() });
3039
+ } catch (err) {
3040
+ respondError(ctx, err);
3041
+ }
3042
+ }
3043
+ async function drop(ctx) {
3044
+ try {
3045
+ await engine.dropIndex(ctx.params[0]);
3046
+ respondJson(ctx, { name: ctx.params[0], dropped: true });
3047
+ } catch (err) {
3048
+ respondError(ctx, err);
3049
+ }
3050
+ }
3051
+ async function stats(ctx) {
3052
+ try {
3053
+ respondJson(ctx, await indexReadsOf(engine).getStats(ctx.params[0]));
3054
+ } catch (err) {
3055
+ respondError(ctx, err);
3056
+ }
3057
+ }
3058
+ async function partitions(ctx) {
3059
+ try {
3060
+ respondJson(ctx, {
3061
+ partitions: await indexReadsOf(engine).getPartitionStats(ctx.params[0])
3052
3062
  });
3053
- core.orchestrator.stopRequestThreads();
3054
- const longQueries = longQueryThread;
3055
- dropLongQueryThread();
3056
- if (longQueries !== null) {
3057
- stopping.push(
3058
- longQueries.executor.execute({ type: "stopServing", requestId: `stop-long-queries-${longQueries.workerId}` }).then(
3059
- () => void 0,
3060
- () => void 0
3061
- )
3063
+ } catch (err) {
3064
+ respondError(ctx, err);
3065
+ }
3066
+ }
3067
+ async function clear(ctx) {
3068
+ try {
3069
+ await engine.clear(ctx.params[0]);
3070
+ respondJson(ctx, { name: ctx.params[0], cleared: true });
3071
+ } catch (err) {
3072
+ respondError(ctx, err);
3073
+ }
3074
+ }
3075
+ async function open(ctx) {
3076
+ try {
3077
+ await engine.open(ctx.params[0]);
3078
+ respondJson(ctx, { name: ctx.params[0], state: "open" });
3079
+ } catch (err) {
3080
+ respondError(ctx, err);
3081
+ }
3082
+ }
3083
+ async function close(ctx) {
3084
+ try {
3085
+ await engine.close(ctx.params[0]);
3086
+ respondJson(ctx, { name: ctx.params[0], state: "closed" });
3087
+ } catch (err) {
3088
+ respondError(ctx, err);
3089
+ }
3090
+ }
3091
+ return { create, list, drop, stats, partitions, clear, open, close };
3092
+ }
3093
+
3094
+ // src/server/handlers/metrics.ts
3095
+ function labelValue(value) {
3096
+ return value.replaceAll("\\", "\\\\").replaceAll('"', '\\"').replaceAll("\n", "\\n");
3097
+ }
3098
+ function labels(pairs) {
3099
+ return `{${pairs.map(([name, value]) => `${name}="${labelValue(value)}"`).join(",")}}`;
3100
+ }
3101
+ function family(name, type, help, samples) {
3102
+ return [`# HELP ${name} ${help}`, `# TYPE ${name} ${type}`, ...samples];
3103
+ }
3104
+ function prometheusText(metrics) {
3105
+ const lines = [
3106
+ ...family(
3107
+ "narsil_replication_lag",
3108
+ "gauge",
3109
+ "Entries that the replica has yet to acknowledge, as the primary's last sequence number minus the replica's.",
3110
+ metrics.replicationLag.map(
3111
+ (figure) => `narsil_replication_lag${labels([
3112
+ ["index", figure.indexName],
3113
+ ["partition", String(figure.partitionId)],
3114
+ ["replica", figure.replica]
3115
+ ])} ${figure.lag}`
3116
+ )
3117
+ ),
3118
+ ...family(
3119
+ "narsil_commit_point",
3120
+ "gauge",
3121
+ "The highest sequence number that the leader of the partition has acknowledged to a client.",
3122
+ metrics.commitPoints.map(
3123
+ (figure) => `narsil_commit_point${labels([
3124
+ ["index", figure.indexName],
3125
+ ["partition", String(figure.partitionId)]
3126
+ ])} ${figure.commitPoint}`
3127
+ )
3128
+ ),
3129
+ ...family(
3130
+ "narsil_commit_point_floor",
3131
+ "gauge",
3132
+ "The floor on the commit point that the allocation table stores, raised when the controller admits a replica.",
3133
+ metrics.partitions.map(
3134
+ (figure) => `narsil_commit_point_floor${labels([
3135
+ ["index", figure.indexName],
3136
+ ["partition", String(figure.partitionId)]
3137
+ ])} ${figure.commitPointFloor}`
3138
+ )
3139
+ ),
3140
+ ...family(
3141
+ "narsil_in_sync_replicas",
3142
+ "gauge",
3143
+ "The replicas besides the primary that are in the in-sync set of the partition.",
3144
+ metrics.partitions.map(
3145
+ (figure) => `narsil_in_sync_replicas${labels([
3146
+ ["index", figure.indexName],
3147
+ ["partition", String(figure.partitionId)]
3148
+ ])} ${figure.inSyncReplicas}`
3149
+ )
3150
+ ),
3151
+ ...family(
3152
+ "narsil_vector_rebuild_waiting_for_memory",
3153
+ "gauge",
3154
+ "A vector field whose rebuild waits for the node to have memory for a second graph, reported as 1.",
3155
+ metrics.vectorRebuildsWaitingForMemory.map(
3156
+ (figure) => `narsil_vector_rebuild_waiting_for_memory${labels([
3157
+ ["index", figure.indexName],
3158
+ ["field", figure.fieldName]
3159
+ ])} 1`
3160
+ )
3161
+ ),
3162
+ ...family(
3163
+ "narsil_coordinator_request_seconds",
3164
+ "histogram",
3165
+ "How long the calls from this node to the cluster coordinator took.",
3166
+ metrics.coordinatorRequests.flatMap((figure) => {
3167
+ const operation = ["operation", figure.operation];
3168
+ return [
3169
+ ...figure.bucketBoundsSeconds.map(
3170
+ (bound, position) => `narsil_coordinator_request_seconds_bucket${labels([operation, ["le", String(bound)]])} ${figure.bucketCounts[position] ?? 0}`
3171
+ ),
3172
+ `narsil_coordinator_request_seconds_bucket${labels([operation, ["le", "+Inf"]])} ${figure.count}`,
3173
+ `narsil_coordinator_request_seconds_sum${labels([operation])} ${figure.sumSeconds}`,
3174
+ `narsil_coordinator_request_seconds_count${labels([operation])} ${figure.count}`
3175
+ ];
3176
+ })
3177
+ )
3178
+ ];
3179
+ return `${lines.join("\n")}
3180
+ `;
3181
+ }
3182
+ function createMetricsHandler(deps) {
3183
+ return async function metrics(ctx) {
3184
+ try {
3185
+ if (deps.cluster === void 0) {
3186
+ throw new NarsilError(
3187
+ ErrorCodes.CLUSTER_OPERATION_UNSUPPORTED,
3188
+ "This server fronts a single engine, so it reports no cluster metrics",
3189
+ {}
3062
3190
  );
3063
3191
  }
3064
- await Promise.all([...stopping, ...retiring]);
3192
+ const text = prometheusText(await deps.cluster.getMetrics());
3193
+ if (ctx.abort.aborted) return;
3194
+ sendText(ctx.res, text, PROMETHEUS_TEXT_CONTENT_TYPE, ctx.abort);
3195
+ } catch (err) {
3196
+ respondError(ctx, err);
3065
3197
  }
3066
3198
  };
3067
3199
  }
3068
3200
 
3069
- // src/server/sync/options.ts
3070
- var FEED_OPTION_FIELDS = ["windowBytes", "maxConnections", "maxCatchUps", "maxBufferedBytes"];
3071
- var FEED_REFUSALS = {
3072
- sync: "Refusing to start: the device sync routes are on with no onRequest hook. A device receives the whole index, so set options.onRequest to admit only the user whose documents the index holds. allowInsecure does not lift this refusal.",
3073
- changes: "Refusing to start: the change feed is on with no onRequest hook. A subscriber receives every document of the index, so set options.onRequest to admit only the services that may read the whole index. allowInsecure does not lift this refusal."
3074
- };
3075
- function assertCount(feed, field, value) {
3076
- if (value === void 0) return;
3077
- if (typeof value === "number" && Number.isSafeInteger(value) && value >= 1) return;
3078
- throw new NarsilError(
3079
- ErrorCodes.CONFIG_INVALID,
3080
- `${feed}.${field} must be a whole number of at least 1; however, this configuration sets it to ${String(value)}`,
3081
- { field, value: String(value) }
3082
- );
3201
+ // src/server/handlers/query-suggestions.ts
3202
+ var PREVIEW_LIMIT_PATTERN = /^\d{1,7}$/;
3203
+ function readObject(ctx) {
3204
+ const body = parseJson(ctx);
3205
+ if (body === void 0) return void 0;
3206
+ if (!isRecord(body)) {
3207
+ badRequest(ctx.res, "The body must be a JSON object");
3208
+ return void 0;
3209
+ }
3210
+ return body;
3083
3211
  }
3084
- function resolveFeedOptions(feed, value) {
3085
- if (value === void 0 || value === false) return null;
3086
- if (value !== true && (typeof value !== "object" || value === null)) {
3087
- throw new NarsilError(
3088
- ErrorCodes.CONFIG_INVALID,
3089
- `${feed} must be true, false, or an object of settings; however, this configuration sets it to ${String(value)}`,
3090
- { field: feed, value: String(value) }
3091
- );
3212
+ function readQuery(ctx, body) {
3213
+ if (typeof body.query === "string") return body.query;
3214
+ badRequest(ctx.res, 'The body must carry the phrase as a string in "query"');
3215
+ return void 0;
3216
+ }
3217
+ function isStringList(value) {
3218
+ return Array.isArray(value) && value.every((entry) => typeof entry === "string");
3219
+ }
3220
+ function isIdentity(value) {
3221
+ return isRecord(value) && typeof value.person === "string" && typeof value.group === "string";
3222
+ }
3223
+ function previewLimitOf(raw) {
3224
+ if (raw === null) return void 0;
3225
+ return PREVIEW_LIMIT_PATTERN.test(raw) ? Number(raw) : null;
3226
+ }
3227
+ async function identityOf(options, ctx) {
3228
+ if (ctx.hookContext === null) return void 0;
3229
+ try {
3230
+ const identity = await options.identify(ctx.hookContext);
3231
+ if (identity === null) return null;
3232
+ return isIdentity(identity) ? { person: identity.person, group: identity.group } : void 0;
3233
+ } catch {
3234
+ return void 0;
3092
3235
  }
3093
- const settings = value === true ? {} : value;
3094
- for (const field of FEED_OPTION_FIELDS) assertCount(feed, field, settings[field]);
3095
- return {
3096
- windowBytes: settings.windowBytes ?? DEFAULT_FEED_WINDOW_BYTES,
3097
- maxConnections: settings.maxConnections ?? DEFAULT_MAX_FEED_CONNECTIONS,
3098
- maxCatchUps: settings.maxCatchUps ?? DEFAULT_MAX_FEED_CATCH_UPS,
3099
- maxBufferedBytes: settings.maxBufferedBytes ?? DEFAULT_MAX_FEED_BUFFERED_BYTES
3100
- };
3101
3236
  }
3102
- function assertFeedHasAuthentication(feed, resolved, options) {
3103
- if (resolved === null || options.onRequest !== void 0) return;
3104
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, FEED_REFUSALS[feed], { field: feed });
3237
+ function noIdentify(indexName) {
3238
+ return new NarsilError(
3239
+ ErrorCodes.QUERY_SUGGESTIONS_NOT_ENABLED,
3240
+ "This server has no identify function for popular query suggestions, so it counts no search",
3241
+ { indexName }
3242
+ );
3105
3243
  }
3106
-
3107
- // src/server/sync/setting-handlers.ts
3108
- function createSyncSettingHandlers(engine) {
3109
- async function getSync(ctx) {
3244
+ function createQuerySuggestionHandlers(deps, options) {
3245
+ const { engine, tasks } = deps;
3246
+ const suggestions = () => querySuggestions(engine);
3247
+ async function update(ctx) {
3248
+ const body = parseJson(ctx);
3249
+ if (body === void 0) return;
3250
+ try {
3251
+ await suggestions().updateSetting(ctx.params[0], body);
3252
+ respondJson(ctx, { ok: true });
3253
+ } catch (err) {
3254
+ respondError(ctx, err);
3255
+ }
3256
+ }
3257
+ async function status(ctx) {
3258
+ try {
3259
+ respondJson(ctx, await suggestions().status(ctx.params[0]));
3260
+ } catch (err) {
3261
+ respondError(ctx, err);
3262
+ }
3263
+ }
3264
+ async function record(ctx) {
3265
+ const body = readObject(ctx);
3266
+ if (body === void 0) return;
3267
+ const extra = Object.keys(body).filter((key) => key !== "query");
3268
+ if (extra.length > 0) {
3269
+ badRequest(ctx.res, 'The body carries the search in "query" alone, because the server identifies the person', {
3270
+ fields: extra
3271
+ });
3272
+ return;
3273
+ }
3274
+ const query = readQuery(ctx, body);
3275
+ if (query === void 0) return;
3276
+ const indexName = ctx.params[0];
3110
3277
  try {
3111
- respondJson(ctx, await engine.getSync(ctx.params[0]));
3278
+ if (options === void 0) throw noIdentify(indexName);
3279
+ const identity = await identityOf(options, ctx);
3280
+ if (identity === void 0) {
3281
+ if (!ctx.abort.aborted) {
3282
+ sendError(ctx.res, 500, ServerErrorCodes.HOOK_ERROR, "The identify function of query suggestions failed");
3283
+ }
3284
+ return;
3285
+ }
3286
+ if (identity === null) {
3287
+ respondJson(ctx, { recorded: false, outcome: "unidentified" });
3288
+ return;
3289
+ }
3290
+ respondJson(ctx, await suggestions().recordQuery(indexName, { query, ...identity }));
3112
3291
  } catch (err) {
3113
3292
  respondError(ctx, err);
3114
3293
  }
3115
3294
  }
3116
- async function updateSync(ctx) {
3117
- const body = parseJson(ctx);
3118
- if (body === void 0) return;
3295
+ async function build(ctx) {
3296
+ const indexName = ctx.params[0];
3119
3297
  try {
3120
- await engine.updateSync(ctx.params[0], body);
3121
- respondJson(ctx, { ok: true });
3298
+ const operations = suggestions();
3299
+ const current = await operations.status(indexName);
3300
+ if (current.setting?.enabled !== true) {
3301
+ throw new NarsilError(
3302
+ ErrorCodes.QUERY_SUGGESTIONS_NOT_ENABLED,
3303
+ `Index "${indexName}" has popular query suggestions off`,
3304
+ { indexName }
3305
+ );
3306
+ }
3307
+ const task = await tasks.start("querySuggestionsBuild", indexName, async (context) => {
3308
+ const finished = await operations.build(indexName);
3309
+ context.reportBuild(finished);
3310
+ if (finished.status === "failed") {
3311
+ throw new NarsilError(
3312
+ finished.errorCode ?? ErrorCodes.WORKER_ACTION_FAILED,
3313
+ "The build of popular query suggestions failed",
3314
+ { indexName, buildId: finished.buildId }
3315
+ );
3316
+ }
3317
+ });
3318
+ respondJson(ctx, task, 202);
3122
3319
  } catch (err) {
3123
3320
  respondError(ctx, err);
3124
3321
  }
3125
3322
  }
3126
- return { getSync, updateSync };
3127
- }
3128
-
3129
- // src/server/task-status.ts
3130
- var TERMINAL_TASK_STATUSES = /* @__PURE__ */ new Set(["succeeded", "failed", "cancelled"]);
3131
- function isTerminalTaskStatus(status) {
3132
- return TERMINAL_TASK_STATUSES.has(status);
3133
- }
3134
-
3135
- // src/server/task-store.ts
3136
- var InMemoryTaskStore = class {
3137
- /**
3138
- * Builds a store that keeps task records in this process alone.
3139
- *
3140
- * @param maxRetained - Records to keep before the oldest finished ones are
3141
- * dropped. Defaults to 1000.
3142
- */
3143
- constructor(maxRetained = DEFAULT_MAX_RETAINED_TASKS) {
3144
- this.maxRetained = maxRetained;
3145
- }
3146
- records = /* @__PURE__ */ new Map();
3147
- /**
3148
- * Stores a record, replacing whatever its id already held.
3149
- *
3150
- * @param record - The record to store. A copy is kept, so changing the
3151
- * object afterwards leaves the store untouched.
3152
- */
3153
- async set(record) {
3154
- this.records.set(record.id, { ...record });
3155
- this.prune();
3156
- }
3157
- /**
3158
- * Reads one record back.
3159
- *
3160
- * @param id - The task to read.
3161
- * @returns A copy of the record, or `null` when the store holds no such id.
3162
- */
3163
- async get(id) {
3164
- const record = this.records.get(id);
3165
- return record ? { ...record } : null;
3166
- }
3167
- /**
3168
- * Lists every record the store holds.
3169
- *
3170
- * @returns A copy of each record, oldest first.
3171
- */
3172
- async list() {
3173
- return [...this.records.values()].map((record) => ({ ...record }));
3174
- }
3175
- /**
3176
- * Removes one record. An id the store never held is not an error.
3177
- *
3178
- * @param id - The task to remove.
3179
- */
3180
- async delete(id) {
3181
- this.records.delete(id);
3182
- }
3183
- prune() {
3184
- if (this.records.size <= this.maxRetained) return;
3185
- for (const [id, record] of this.records) {
3186
- if (this.records.size <= this.maxRetained) break;
3187
- if (isTerminalTaskStatus(record.status)) this.records.delete(id);
3323
+ async function preview(ctx) {
3324
+ const limit = previewLimitOf(ctx.query.get("limit"));
3325
+ if (limit === null) {
3326
+ badRequest(ctx.res, 'Query parameter "limit" must be a whole number', { value: ctx.query.get("limit") });
3327
+ return;
3328
+ }
3329
+ try {
3330
+ respondJson(ctx, await suggestions().preview(ctx.params[0], limit));
3331
+ } catch (err) {
3332
+ respondError(ctx, err);
3188
3333
  }
3189
3334
  }
3190
- };
3191
- function byNewestFirst(left, right) {
3192
- if (left.createdAt !== right.createdAt) return right.createdAt - left.createdAt;
3193
- return left.id.localeCompare(right.id);
3194
- }
3195
- function matchesQuery(record, query) {
3196
- if (query.indexName !== void 0 && record.indexName !== query.indexName) return false;
3197
- if (query.type !== void 0 && query.type.length > 0 && !query.type.includes(record.type)) return false;
3198
- if (query.status !== void 0 && query.status.length > 0 && !query.status.includes(record.status)) return false;
3199
- return true;
3200
- }
3201
- var TaskRegistry = class {
3202
- constructor(store, instanceId, maxConcurrent = 0) {
3203
- this.store = store;
3204
- this.instanceId = instanceId;
3205
- this.maxConcurrent = maxConcurrent;
3206
- }
3207
- live = /* @__PURE__ */ new Map();
3208
- /** Records a task as running and drives `op` to completion in the background.
3209
- * Returns the record once it is persisted so the caller can respond 202.
3210
- * @throws A `NarsilError` with `TOO_MANY_REQUESTS` where this instance
3211
- * already drives {@link ServerLimits.maxConcurrentTasks} tasks, because each
3212
- * one holds its own working set until it finishes. */
3213
- async start(type, indexName, op, progress, backupName) {
3214
- if (this.maxConcurrent > 0 && this.live.size >= this.maxConcurrent) {
3215
- throw new NarsilError(
3216
- ServerErrorCodes.TOO_MANY_REQUESTS,
3217
- `This instance already drives ${this.maxConcurrent} tasks; poll one to completion before starting another`,
3218
- { running: this.live.size, limit: this.maxConcurrent }
3219
- );
3335
+ async function readBans(ctx) {
3336
+ try {
3337
+ respondJson(ctx, { bans: await suggestions().bans(ctx.params[0]) });
3338
+ } catch (err) {
3339
+ respondError(ctx, err);
3220
3340
  }
3221
- const now = Date.now();
3222
- const record = {
3223
- id: randomUUID(),
3224
- type,
3225
- indexName,
3226
- ...backupName === void 0 ? {} : { backupName },
3227
- owner: this.instanceId,
3228
- status: "running",
3229
- createdAt: now,
3230
- startedAt: now,
3231
- ...progress === void 0 ? {} : { progress }
3232
- };
3233
- const task = {
3234
- record,
3235
- controller: new AbortController(),
3236
- writes: Promise.resolve(),
3237
- lastProgressWriteAt: now,
3238
- settled: false
3239
- };
3240
- this.live.set(record.id, task);
3241
- await this.enqueueWrite(task, record, RUNNING_TASK_TTL_MS);
3242
- void this.drive(task, op);
3243
- return record;
3244
3341
  }
3245
- get(id) {
3246
- return this.store.get(id);
3342
+ async function replaceBans(ctx) {
3343
+ const body = readObject(ctx);
3344
+ if (body === void 0) return;
3345
+ if (!isStringList(body.bans)) {
3346
+ badRequest(ctx.res, 'The body must carry the ban list as an array of strings in "bans"');
3347
+ return;
3348
+ }
3349
+ try {
3350
+ respondJson(ctx, { bans: await suggestions().updateBans(ctx.params[0], body.bans) });
3351
+ } catch (err) {
3352
+ respondError(ctx, err);
3353
+ }
3247
3354
  }
3248
- /** Reads one page of task records, newest first, keeping only what the query
3249
- * matches. Filtering happens here rather than in the store, so every custom
3250
- * {@link TaskStore} works unchanged. */
3251
- async list(query = {}) {
3252
- const from = Math.max(0, Math.trunc(query.from ?? 0));
3253
- const limit = Math.max(1, Math.trunc(query.limit ?? DEFAULT_TASK_PAGE_SIZE));
3254
- const matched = (await this.store.list()).filter((record) => matchesQuery(record, query)).sort(byNewestFirst);
3255
- const page = matched.slice(from, from + limit);
3256
- const consumed = from + page.length;
3257
- return { tasks: page, total: matched.length, from, limit, next: consumed < matched.length ? consumed : null };
3355
+ async function remove(ctx) {
3356
+ const body = readObject(ctx);
3357
+ if (body === void 0) return;
3358
+ const query = readQuery(ctx, body);
3359
+ if (query === void 0) return;
3360
+ try {
3361
+ const held = await suggestions().removeQuery(ctx.params[0], query);
3362
+ respondJson(ctx, { suggestionIndexes: held });
3363
+ } catch (err) {
3364
+ respondError(ctx, err);
3365
+ }
3258
3366
  }
3259
- /** Asks a running task to stop. The work stops between units, so the task
3260
- * reaches `cancelled` only once it actually stopped, and a request that
3261
- * arrives too late leaves it `succeeded`. */
3262
- async cancel(id) {
3263
- const stored = await this.store.get(id);
3264
- if (!stored) return { outcome: "not-found", record: null };
3265
- if (isTerminalTaskStatus(stored.status)) {
3266
- return { outcome: "already-finished", record: stored };
3367
+ async function manual(ctx) {
3368
+ const body = readObject(ctx);
3369
+ if (body === void 0) return;
3370
+ const query = readQuery(ctx, body);
3371
+ if (query === void 0) return;
3372
+ const { group, count } = body;
3373
+ if (typeof group !== "string" || count !== void 0 && typeof count !== "number") {
3374
+ badRequest(ctx.res, 'The body must carry the group as a string in "group", and any count as a number in "count"');
3375
+ return;
3267
3376
  }
3268
- const task = this.live.get(id);
3269
- if (!task) {
3270
- const outcome = stored.owner === this.instanceId ? "already-finished" : "owned-by-another-instance";
3271
- return { outcome, record: stored };
3377
+ try {
3378
+ const name = await suggestions().addSuggestion(ctx.params[0], {
3379
+ group,
3380
+ query,
3381
+ ...count === void 0 ? {} : { count }
3382
+ });
3383
+ respondJson(ctx, { indexName: name });
3384
+ } catch (err) {
3385
+ respondError(ctx, err);
3386
+ }
3387
+ }
3388
+ return { update, status, record, build, preview, readBans, replaceBans, remove, manual };
3389
+ }
3390
+
3391
+ // src/server/handlers/search.ts
3392
+ function createSearchHandlers(deps) {
3393
+ const { engine, limits, allowRegex } = deps;
3394
+ async function search(ctx) {
3395
+ const params = readQueryBody(ctx, limits, allowRegex, "read");
3396
+ if (!params) return;
3397
+ try {
3398
+ const query = withServerQueryLimits(params, limits);
3399
+ const result = await searchWithSignal(ctx, query);
3400
+ await respondAnswer(ctx, encodeQueryResult(result), "hits", `a search on index '${ctx.params[0]}'`);
3401
+ } catch (err) {
3402
+ respondError(ctx, err);
3272
3403
  }
3273
- task.controller.abort(new Error("The task was cancelled"));
3274
- const record = { ...task.record, cancelRequestedAt: Date.now() };
3275
- task.record = record;
3276
- await this.enqueueWrite(task, record, RUNNING_TASK_TTL_MS);
3277
- return { outcome: "cancelling", record };
3278
3404
  }
3279
- /** Fails this instance's previously-running tasks after a restart. A task can
3280
- * only be advanced by the process that owns it, so its own running tasks from
3281
- * a prior life are dead and must not keep showing a stale running status. This
3282
- * is effective only with a stable instanceId; with the default random id it
3283
- * finds none of its own prior tasks, which is also correct for the in-memory
3284
- * default (which starts empty). */
3285
- async reconcile() {
3286
- let records;
3405
+ async function searchWithSignal(ctx, params) {
3406
+ bindQuerySignal(params, requestSignal(ctx.abort));
3287
3407
  try {
3288
- records = await this.store.list();
3289
- } catch {
3290
- return;
3408
+ return await engine.query(ctx.params[0], params);
3409
+ } finally {
3410
+ clearQuerySignal(params);
3291
3411
  }
3292
- for (const record of records) {
3293
- if (record.owner !== this.instanceId) continue;
3294
- if (isTerminalTaskStatus(record.status)) continue;
3295
- const failed = {
3296
- ...record,
3297
- status: "failed",
3298
- completedAt: Date.now(),
3299
- error: { code: ServerErrorCodes.TASK_INTERRUPTED, message: "The server restarted while this task was running" }
3300
- };
3412
+ }
3413
+ async function preflight(ctx) {
3414
+ const params = readQueryBody(ctx, limits, allowRegex, "read");
3415
+ if (!params) return;
3416
+ try {
3417
+ const query = withServerQueryLimits(params, limits);
3418
+ bindQuerySignal(query, requestSignal(ctx.abort));
3301
3419
  try {
3302
- await this.store.set(failed, TERMINAL_TASK_TTL_MS);
3303
- } catch {
3420
+ respondJson(ctx, await engine.preflight(ctx.params[0], query));
3421
+ } finally {
3422
+ clearQuerySignal(query);
3304
3423
  }
3424
+ } catch (err) {
3425
+ respondError(ctx, err);
3305
3426
  }
3306
3427
  }
3307
- enqueueWrite(task, record, ttlMs) {
3308
- task.writes = task.writes.then(
3309
- () => this.store.set(record, ttlMs),
3310
- () => this.store.set(record, ttlMs)
3311
- );
3312
- return task.writes.catch(() => {
3313
- });
3428
+ async function suggest(ctx) {
3429
+ const params = parseJsonObject(ctx, "read");
3430
+ if (!params) return;
3431
+ const failure = validateSuggest(params);
3432
+ if (failure) {
3433
+ rejectInvalid(ctx, failure);
3434
+ return;
3435
+ }
3436
+ try {
3437
+ respondJson(ctx, await engine.suggest(ctx.params[0], params));
3438
+ } catch (err) {
3439
+ respondError(ctx, err);
3440
+ }
3314
3441
  }
3315
- reportProgress(task, change) {
3316
- if (task.settled) return;
3317
- task.record = { ...task.record, ...change };
3318
- const now = Date.now();
3319
- if (now - task.lastProgressWriteAt < TASK_PROGRESS_WRITE_INTERVAL_MS) return;
3320
- task.lastProgressWriteAt = now;
3321
- void this.enqueueWrite(task, task.record, RUNNING_TASK_TTL_MS);
3442
+ return { search, preflight, suggest };
3443
+ }
3444
+
3445
+ // src/server/sync/setting-handlers.ts
3446
+ function createSyncSettingHandlers(engine) {
3447
+ async function getSync(ctx) {
3448
+ try {
3449
+ respondJson(ctx, await engine.getSync(ctx.params[0]));
3450
+ } catch (err) {
3451
+ respondError(ctx, err);
3452
+ }
3322
3453
  }
3323
- async drive(task, op) {
3324
- const context = {
3325
- signal: task.controller.signal,
3326
- reportProgress: (progress) => this.reportProgress(task, { progress }),
3327
- reportSteps: (steps) => this.reportProgress(task, { steps }),
3328
- reportResult: (result) => {
3329
- task.record = { ...task.record, result };
3330
- },
3331
- reportBuild: (build) => {
3332
- task.record = { ...task.record, build };
3333
- }
3334
- };
3335
- let settled;
3454
+ async function updateSync(ctx) {
3455
+ const body = parseJson(ctx);
3456
+ if (body === void 0) return;
3336
3457
  try {
3337
- await op(context);
3338
- settled = { ...task.record, status: "succeeded" };
3458
+ await engine.updateSync(ctx.params[0], body);
3459
+ respondJson(ctx, { ok: true });
3339
3460
  } catch (err) {
3340
- settled = task.controller.signal.aborted ? { ...task.record, status: "cancelled" } : {
3341
- ...task.record,
3342
- status: "failed",
3343
- error: err instanceof NarsilError ? serializeNarsilError(err) : { code: "INTERNAL_ERROR", message: "The operation failed" }
3344
- };
3461
+ respondError(ctx, err);
3345
3462
  }
3346
- task.settled = true;
3347
- task.record = { ...settled, completedAt: Date.now() };
3348
- await this.enqueueWrite(task, task.record, TERMINAL_TASK_TTL_MS);
3349
- this.live.delete(task.record.id);
3350
3463
  }
3351
- };
3352
-
3353
- // src/server/create-server.ts
3354
- function bodyBytesInFlightOf(fraction) {
3355
- const processLimitBytes = readMemoryFigures()?.processLimitBytes;
3356
- if (fraction === 0 || processLimitBytes === void 0) return Number.POSITIVE_INFINITY;
3357
- return Math.floor(fraction * processLimitBytes);
3358
- }
3359
- function resolveLimits(limits, engineRerankWindow) {
3360
- return {
3361
- maxBodyBytesInFlight: bodyBytesInFlightOf(
3362
- limits?.maxBodyBytesInFlightFraction ?? DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION
3363
- ),
3364
- maxBodyBytes: limits?.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES,
3365
- maxImportBytes: limits?.maxImportBytes ?? DEFAULT_MAX_IMPORT_BYTES,
3366
- maxRestoreBytes: limits?.maxRestoreBytes ?? DEFAULT_MAX_RESTORE_BYTES,
3367
- maxLineBytes: limits?.maxLineBytes ?? DEFAULT_MAX_LINE_BYTES,
3368
- importBatchSize: limits?.importBatchSize ?? DEFAULT_IMPORT_BATCH_SIZE,
3369
- maxConcurrentRequests: limits?.maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS,
3370
- maxResultWindow: limits?.maxResultWindow ?? DEFAULT_MAX_RESULT_WINDOW,
3371
- maxRerankWindow: limits?.maxRerankWindow ?? Math.min(DEFAULT_MAX_RERANK_WINDOW, engineRerankWindow ?? DEFAULT_MAX_RERANK_WINDOW),
3372
- maxFetchDocuments: limits?.maxFetchDocuments ?? DEFAULT_MAX_FETCH_DOCUMENTS,
3373
- maxImportErrors: limits?.maxImportErrors ?? DEFAULT_MAX_IMPORT_ERRORS,
3374
- maxTaskPageSize: limits?.maxTaskPageSize ?? DEFAULT_MAX_TASK_PAGE_SIZE,
3375
- maxConcurrentTasks: limits?.maxConcurrentTasks ?? DEFAULT_MAX_CONCURRENT_TASKS
3376
- };
3377
- }
3378
- function resolveBuild(build) {
3379
- return {
3380
- version: build?.version ?? publishedPackageVersion(),
3381
- gitSha: build?.gitSha ?? null,
3382
- dirty: build?.dirty ?? false
3383
- };
3384
- }
3385
- function isLoopback(host) {
3386
- return host === "127.0.0.1" || host === "::1" || host === "localhost" || host.startsWith("127.");
3387
- }
3388
- function assertSecureBinding(host, options) {
3389
- if (isLoopback(host) || options.onRequest || options.allowInsecure) return;
3390
- throw new NarsilError(
3391
- ErrorCodes.CONFIG_INVALID,
3392
- `Refusing to start: bound to a non-loopback address ("${host}") with no onRequest auth hook. The server exposes destructive admin endpoints (restore, drop, clear, rebalance, optimize). Resolve one of: (1) set options.onRequest to authenticate requests, (2) bind to 127.0.0.1, or (3) set options.allowInsecure: true if this address is on a trusted private network.`,
3393
- { host }
3394
- );
3464
+ return { getSync, updateSync };
3395
3465
  }
3466
+
3467
+ // src/server/create-server/handlers.ts
3396
3468
  function mainThreadHandlers(deps, syncServing) {
3397
3469
  const idx = createIndexHandlers(deps);
3398
3470
  const doc = createDocumentHandlers(deps);
@@ -3479,6 +3551,39 @@ function mainThreadHandlers(deps, syncServing) {
3479
3551
  addManualSuggestion: suggestions.manual
3480
3552
  };
3481
3553
  }
3554
+
3555
+ // src/server/create-server/server.ts
3556
+ function bodyBytesInFlightOf(fraction) {
3557
+ const processLimitBytes = readMemoryFigures()?.processLimitBytes;
3558
+ if (fraction === 0 || processLimitBytes === void 0) return Number.POSITIVE_INFINITY;
3559
+ return Math.floor(fraction * processLimitBytes);
3560
+ }
3561
+ function resolveLimits(limits, engineRerankWindow) {
3562
+ return {
3563
+ maxBodyBytesInFlight: bodyBytesInFlightOf(
3564
+ limits?.maxBodyBytesInFlightFraction ?? DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION
3565
+ ),
3566
+ maxBodyBytes: limits?.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES,
3567
+ maxImportBytes: limits?.maxImportBytes ?? DEFAULT_MAX_IMPORT_BYTES,
3568
+ maxRestoreBytes: limits?.maxRestoreBytes ?? DEFAULT_MAX_RESTORE_BYTES,
3569
+ maxLineBytes: limits?.maxLineBytes ?? DEFAULT_MAX_LINE_BYTES,
3570
+ importBatchSize: limits?.importBatchSize ?? DEFAULT_IMPORT_BATCH_SIZE,
3571
+ maxConcurrentRequests: limits?.maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS,
3572
+ maxResultWindow: limits?.maxResultWindow ?? DEFAULT_MAX_RESULT_WINDOW,
3573
+ maxRerankWindow: limits?.maxRerankWindow ?? Math.min(DEFAULT_MAX_RERANK_WINDOW, engineRerankWindow ?? DEFAULT_MAX_RERANK_WINDOW),
3574
+ maxFetchDocuments: limits?.maxFetchDocuments ?? DEFAULT_MAX_FETCH_DOCUMENTS,
3575
+ maxImportErrors: limits?.maxImportErrors ?? DEFAULT_MAX_IMPORT_ERRORS,
3576
+ maxTaskPageSize: limits?.maxTaskPageSize ?? DEFAULT_MAX_TASK_PAGE_SIZE,
3577
+ maxConcurrentTasks: limits?.maxConcurrentTasks ?? DEFAULT_MAX_CONCURRENT_TASKS
3578
+ };
3579
+ }
3580
+ function resolveBuild(build) {
3581
+ return {
3582
+ version: build?.version ?? publishedPackageVersion(),
3583
+ gitSha: build?.gitSha ?? null,
3584
+ dirty: build?.dirty ?? false
3585
+ };
3586
+ }
3482
3587
  var NarsilHttpServer = class {
3483
3588
  host;
3484
3589
  port;
@@ -3486,6 +3591,8 @@ var NarsilHttpServer = class {
3486
3591
  engine;
3487
3592
  deps;
3488
3593
  uws = null;
3594
+ app = null;
3595
+ closingRequests = null;
3489
3596
  listenSocket = null;
3490
3597
  ready = false;
3491
3598
  threads = null;
@@ -3494,6 +3601,9 @@ var NarsilHttpServer = class {
3494
3601
  syncServing = null;
3495
3602
  feeds = null;
3496
3603
  stopIdleHeapCleanup = () => void 0;
3604
+ listening = null;
3605
+ closing = null;
3606
+ cancelListen = null;
3497
3607
  constructor(engine, options = {}) {
3498
3608
  assertServerEngine(engine);
3499
3609
  const engineRerankWindow = rerankWindowCapOf(engine);
@@ -3528,14 +3638,36 @@ var NarsilHttpServer = class {
3528
3638
  querySuggestions: options.querySuggestions
3529
3639
  };
3530
3640
  }
3531
- async listen() {
3641
+ listen() {
3642
+ if (this.listening !== null) return this.listening;
3643
+ const closing = this.closing;
3644
+ this.listening = (async () => {
3645
+ if (closing !== null) await closing;
3646
+ if (this.listenSocket !== null) return;
3647
+ let cancelled = false;
3648
+ const cancellation = new Promise((resolve) => {
3649
+ this.cancelListen = () => {
3650
+ cancelled = true;
3651
+ resolve();
3652
+ };
3653
+ });
3654
+ await this.startListening(cancellation, () => cancelled);
3655
+ })().finally(() => {
3656
+ this.listening = null;
3657
+ this.cancelListen = null;
3658
+ });
3659
+ return this.listening;
3660
+ }
3661
+ async startListening(cancellation, cancelled) {
3532
3662
  assertFeedHasAuthentication("sync", this.sync, this.options);
3533
3663
  assertFeedHasAuthentication("changes", this.changes, this.options);
3534
3664
  assertSecureBinding(this.host, this.options);
3535
3665
  const uws = await loadUWebSockets();
3536
3666
  this.uws = uws;
3537
3667
  const app = uws.App();
3668
+ this.app = app;
3538
3669
  await this.registerRoutes(app);
3670
+ if (cancelled()) return;
3539
3671
  await new Promise((resolve, reject) => {
3540
3672
  app.listen(this.host, this.port, (socket) => {
3541
3673
  if (socket) {
@@ -3546,11 +3678,24 @@ var NarsilHttpServer = class {
3546
3678
  }
3547
3679
  });
3548
3680
  });
3549
- await this.deps.tasks.reconcile();
3681
+ await Promise.race([this.deps.tasks.reconcile(cancelled), cancellation]);
3682
+ if (cancelled()) return;
3550
3683
  this.stopIdleHeapCleanup = startIdleHeapCleanup();
3551
3684
  this.ready = true;
3552
3685
  }
3553
- async close() {
3686
+ close() {
3687
+ if (this.closing !== null) return this.closing;
3688
+ this.cancelListen?.();
3689
+ const listening = this.listening;
3690
+ this.closing = (async () => {
3691
+ if (listening !== null) await listening.catch(() => void 0);
3692
+ await this.stopListening();
3693
+ })().then(() => {
3694
+ this.closing = null;
3695
+ });
3696
+ return this.closing;
3697
+ }
3698
+ async stopListening() {
3554
3699
  this.ready = false;
3555
3700
  this.stopIdleHeapCleanup();
3556
3701
  this.stopIdleHeapCleanup = () => void 0;
@@ -3560,15 +3705,23 @@ var NarsilHttpServer = class {
3560
3705
  }
3561
3706
  this.feeds?.close();
3562
3707
  this.feeds = null;
3563
- this.syncServing = null;
3564
- if (this.threads !== null) {
3565
- const threads = this.threads;
3566
- this.threads = null;
3567
- await threads.stop();
3708
+ const app = this.app;
3709
+ const closingRequests = this.closingRequests;
3710
+ const threads = this.threads;
3711
+ this.app = null;
3712
+ this.closingRequests = null;
3713
+ this.threads = null;
3714
+ try {
3715
+ await Promise.all([closingRequests?.close(SERVER_CLOSE_DRAIN_DEADLINE_MS), threads?.stop()]);
3716
+ } finally {
3717
+ await new Promise((resolve) => setImmediate(resolve));
3718
+ app?.close();
3719
+ this.syncServing = null;
3568
3720
  }
3569
3721
  }
3570
3722
  async shutdown() {
3571
3723
  await this.close();
3724
+ await this.deps.tasks.shutdown();
3572
3725
  await this.engine.shutdown();
3573
3726
  }
3574
3727
  get listeningPort() {
@@ -3589,8 +3742,9 @@ var NarsilHttpServer = class {
3589
3742
  this.syncServing = started.syncServing;
3590
3743
  this.feeds = started.host;
3591
3744
  const { sync, changes } = started.host.gateways;
3592
- if (sync !== void 0) registerFeedRoute(app, sync, SYNC_FEED_ROUTE);
3593
- if (changes !== void 0) registerFeedRoute(app, changes, CHANGES_FEED_ROUTE);
3745
+ const accept = this.closingRequests?.accept;
3746
+ if (sync !== void 0) registerFeedRoute(app, sync, SYNC_FEED_ROUTE, accept);
3747
+ if (changes !== void 0) registerFeedRoute(app, changes, CHANGES_FEED_ROUTE, accept);
3594
3748
  }
3595
3749
  async registerRoutes(app) {
3596
3750
  const cors = resolveCors(this.options.cors);
@@ -3601,14 +3755,19 @@ var NarsilHttpServer = class {
3601
3755
  const bodyBudget = createBodyBudgetBuffer(gateSlotOfWorker(threadCount));
3602
3756
  const { limits, build, allowRegex } = this.deps;
3603
3757
  const handlers = mainThreadHandlers(this.deps, () => this.syncServing);
3758
+ const closingRequests = watchClosingRequests();
3759
+ this.closingRequests = closingRequests;
3604
3760
  this.startFeeds(app, authorize, core);
3605
3761
  const run = createRouteRunner({
3762
+ accept: closingRequests.accept,
3606
3763
  authorize,
3607
3764
  gate: createSharedRequestGate(gate, limits.maxConcurrentRequests, MAIN_THREAD_GATE_SLOT),
3608
3765
  bodyBudget: createSharedBodyBudget(bodyBudget, limits.maxBodyBytesInFlight, MAIN_THREAD_GATE_SLOT),
3609
- corsHeaders: cors ? corsHeaderReader(cors) : void 0
3766
+ corsHeaders: cors ? corsHeaderReader(cors) : void 0,
3767
+ respondThrough: closingRequests.respondThrough,
3768
+ track: closingRequests.track
3610
3769
  });
3611
- registerServerRoutes(app, run, handlers, limits, cors);
3770
+ registerServerRoutes(app, run, handlers, limits, cors, closingRequests.accept);
3612
3771
  if (core === void 0 || threadCount === 0 || !detectRuntime().supportsWorkerThreads) return;
3613
3772
  try {
3614
3773
  this.threads = await startRequestThreads({
@@ -3633,4 +3792,170 @@ function createServer(engine, options) {
3633
3792
  return new NarsilHttpServer(engine, options);
3634
3793
  }
3635
3794
 
3636
- export { InMemoryTaskStore, createServer };
3795
+ // src/server/persistence-task-store.ts
3796
+ var RECORD_KEY_SUFFIX = ".json";
3797
+ var encoder = new TextEncoder();
3798
+ var decoder = new TextDecoder("utf-8", { fatal: true });
3799
+ function keyOf(id) {
3800
+ return `${encodeURIComponent(id)}${RECORD_KEY_SUFFIX}`;
3801
+ }
3802
+ function idOfKey(key) {
3803
+ if (!key.endsWith(RECORD_KEY_SUFFIX)) return null;
3804
+ try {
3805
+ const id = decodeURIComponent(key.slice(0, -RECORD_KEY_SUFFIX.length));
3806
+ return keyOf(id) === key ? id : null;
3807
+ } catch {
3808
+ return null;
3809
+ }
3810
+ }
3811
+ function isStoredRecord(value, id) {
3812
+ if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
3813
+ const record = value;
3814
+ return record.id === id && isTaskType(record.type) && isTaskStatus(record.status) && typeof record.indexName === "string" && typeof record.owner === "string" && typeof record.createdAt === "number" && Number.isFinite(record.createdAt);
3815
+ }
3816
+ function byOldestFirst(left, right) {
3817
+ if (left.createdAt !== right.createdAt) return left.createdAt - right.createdAt;
3818
+ return left.id < right.id ? -1 : left.id > right.id ? 1 : 0;
3819
+ }
3820
+ var PersistenceTaskStore = class {
3821
+ /**
3822
+ * Builds a store over an adapter that it owns alone.
3823
+ *
3824
+ * @param adapter - The adapter that holds the records, such as one from
3825
+ * `createFilesystemPersistence` scoped to a folder that holds nothing else.
3826
+ * The store deletes every key of the adapter that holds no task record.
3827
+ * @param maxRetained - Records to keep before the oldest finished ones are
3828
+ * dropped. Defaults to 1000.
3829
+ */
3830
+ constructor(adapter, maxRetained = DEFAULT_MAX_RETAINED_TASKS) {
3831
+ this.adapter = adapter;
3832
+ this.maxRetained = maxRetained;
3833
+ }
3834
+ records = /* @__PURE__ */ new Map();
3835
+ pendingWrites = /* @__PURE__ */ new Map();
3836
+ loading = null;
3837
+ /**
3838
+ * Stores a record and writes it through the adapter, replacing whatever its
3839
+ * id already held.
3840
+ *
3841
+ * @param record - The record to store. A copy is kept, so changing the
3842
+ * object afterwards leaves the store untouched.
3843
+ * @throws The adapter's error where the write fails. The store still
3844
+ * answers reads with the new record, and the next write of that id tries
3845
+ * again.
3846
+ */
3847
+ async set(record) {
3848
+ const id = record.id;
3849
+ const json = JSON.stringify(record);
3850
+ await this.ensureLoaded();
3851
+ this.records.set(id, JSON.parse(json));
3852
+ const written = this.persist(id, json);
3853
+ this.prune();
3854
+ await written;
3855
+ }
3856
+ /**
3857
+ * Reads one record back.
3858
+ *
3859
+ * @param id - The task to read.
3860
+ * @returns A copy of the record, or `null` when the store holds no such id.
3861
+ */
3862
+ async get(id) {
3863
+ await this.ensureLoaded();
3864
+ const record = this.records.get(id);
3865
+ return record ? structuredClone(record) : null;
3866
+ }
3867
+ /**
3868
+ * Lists every record the store holds.
3869
+ *
3870
+ * @returns A copy of each record, oldest first.
3871
+ */
3872
+ async list() {
3873
+ await this.ensureLoaded();
3874
+ return [...this.records.values()].map((record) => structuredClone(record));
3875
+ }
3876
+ /**
3877
+ * Removes one record and deletes its key. An id the store never held is not
3878
+ * an error.
3879
+ *
3880
+ * @param id - The task to remove.
3881
+ */
3882
+ async delete(id) {
3883
+ await this.ensureLoaded();
3884
+ this.records.delete(id);
3885
+ await this.persist(id, null);
3886
+ }
3887
+ /** Resolves once every write that the store has started reaches the adapter, failed or not. */
3888
+ async shutdown() {
3889
+ await this.loading;
3890
+ while (this.pendingWrites.size > 0) {
3891
+ await Promise.allSettled([...this.pendingWrites.values()]);
3892
+ }
3893
+ }
3894
+ ensureLoaded() {
3895
+ if (this.loading === null) {
3896
+ this.loading = this.load().catch((err) => {
3897
+ this.loading = null;
3898
+ throw err;
3899
+ });
3900
+ }
3901
+ return this.loading;
3902
+ }
3903
+ async load() {
3904
+ const keys2 = await this.adapter.list("");
3905
+ const loaded = [];
3906
+ const malformed = [];
3907
+ let next = 0;
3908
+ const readNext = async () => {
3909
+ while (next < keys2.length) {
3910
+ const key = keys2[next];
3911
+ next += 1;
3912
+ const outcome = await this.loadKey(key);
3913
+ if (outcome.kind === "record") loaded.push(outcome.record);
3914
+ else if (outcome.kind === "malformed") malformed.push(key);
3915
+ }
3916
+ };
3917
+ await Promise.all(Array.from({ length: Math.min(TASK_RECORD_LOAD_CONCURRENCY, keys2.length) }, readNext));
3918
+ loaded.sort(byOldestFirst);
3919
+ for (const record of loaded) this.records.set(record.id, record);
3920
+ await Promise.allSettled(malformed.map((key) => this.adapter.delete(key)));
3921
+ this.prune();
3922
+ }
3923
+ async loadKey(key) {
3924
+ const id = idOfKey(key);
3925
+ if (id === null) return { kind: "malformed" };
3926
+ let bytes;
3927
+ try {
3928
+ bytes = await this.adapter.load(key);
3929
+ } catch {
3930
+ return { kind: "unreadable" };
3931
+ }
3932
+ if (bytes === null) return { kind: "unreadable" };
3933
+ try {
3934
+ const parsed = JSON.parse(decoder.decode(bytes));
3935
+ return isStoredRecord(parsed, id) ? { kind: "record", record: parsed } : { kind: "malformed" };
3936
+ } catch {
3937
+ return { kind: "malformed" };
3938
+ }
3939
+ }
3940
+ persist(id, json) {
3941
+ const key = keyOf(id);
3942
+ const previous = this.pendingWrites.get(id) ?? Promise.resolve();
3943
+ const write = previous.catch(() => void 0).then(() => json === null ? this.adapter.delete(key) : this.adapter.save(key, encoder.encode(json)));
3944
+ this.pendingWrites.set(id, write);
3945
+ write.finally(() => {
3946
+ if (this.pendingWrites.get(id) === write) this.pendingWrites.delete(id);
3947
+ }).catch(() => void 0);
3948
+ return write;
3949
+ }
3950
+ prune() {
3951
+ if (this.records.size <= this.maxRetained) return;
3952
+ for (const [id, record] of this.records) {
3953
+ if (this.records.size <= this.maxRetained) break;
3954
+ if (!isTerminalTaskStatus(record.status)) continue;
3955
+ this.records.delete(id);
3956
+ this.persist(id, null).catch(() => void 0);
3957
+ }
3958
+ }
3959
+ };
3960
+
3961
+ export { InMemoryTaskStore, PersistenceTaskStore, createServer };