@delali/narsil 0.4.1-next.215 → 0.4.2-next.271
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +9 -3
- package/dist/adapters/gcs.d.ts +1 -1
- package/dist/adapters/indexeddb.d.ts +6 -1
- package/dist/adapters/indexeddb.mjs +20 -26
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/s3.d.ts +1 -1
- package/dist/{adapters-pljAqmrC.d.ts → adapters-8LcdNB77.d.ts} +18 -1
- package/dist/browser/{chunk-GBJSEHVJ.mjs → chunk-DPRCAVC2.mjs} +14 -8
- package/dist/browser/{chunk-YINZFPLT.mjs → chunk-WRRXR57N.mjs} +1 -1
- package/dist/browser/{chunk-QAOVAYAG.mjs → chunk-XBWABEL3.mjs} +1 -1
- package/dist/browser/{chunk-4E5APHTM.mjs → chunk-Y5CSZDOH.mjs} +1 -1
- package/dist/browser/{durability-config-75NSFCQN.mjs → durability-config-UP3OTUMB.mjs} +34 -2
- package/dist/browser/{msgpack-leaf-44XNWSSY.mjs → msgpack-leaf-FEPCLXXY.mjs} +7 -1
- package/dist/browser/{optional-reads-6LN3HMUH.mjs → optional-reads-NQO5DKM4.mjs} +18 -16
- package/dist/browser/{optional-writes-76DWMRQH.mjs → optional-writes-HVPPDHSY.mjs} +259 -104
- package/dist/browser/{storage-leaf-5UTR5K7J.mjs → storage-leaf-42IZEOJJ.mjs} +1473 -305
- package/dist/{capabilities-BOFFMx84.d.ts → capabilities-BoWiLNv7.d.ts} +76 -61
- package/dist/{chunk-DBC4EJWB.mjs → chunk-33ZMFOAR.mjs} +1 -1
- package/dist/{chunk-NCIGNQJE.mjs → chunk-4GCHJ3XD.mjs} +930 -387
- package/dist/{chunk-7P5YWDRQ.mjs → chunk-4WQNWAQE.mjs} +1 -1
- package/dist/{chunk-357KIU5E.mjs → chunk-5E3TJ4P3.mjs} +5 -5
- package/dist/{chunk-GXCDWLQI.mjs → chunk-5GYGHPCH.mjs} +7 -1
- package/dist/{chunk-SMQPZHPS.mjs → chunk-ATQ6JR3I.mjs} +1 -1
- package/dist/{chunk-2K24MEYV.mjs → chunk-BBIAUHAB.mjs} +3 -3
- package/dist/{react-native/chunks/chunk-MMPBO4WR.mjs → chunk-DA32WXBT.mjs} +2 -2
- package/dist/{chunk-MSJD74GN.mjs → chunk-EALKCTWH.mjs} +5 -5
- package/dist/{chunk-755NM4PL.mjs → chunk-FOGTBAYN.mjs} +183 -21
- package/dist/{chunk-6GHX35ZM.mjs → chunk-GTBJSNN2.mjs} +1 -1
- package/dist/{chunk-DDUXEJXW.mjs → chunk-H6LWGMPQ.mjs} +45 -22
- package/dist/{chunk-UNEEFTEU.mjs → chunk-HZ3OAUTL.mjs} +2 -1
- package/dist/{chunk-M3B7SGJB.mjs → chunk-KL4ORYJW.mjs} +6 -6
- package/dist/{chunk-IL5DRCNU.mjs → chunk-KQWTS7AM.mjs} +1 -1
- package/dist/{chunk-DKIMHIBC.mjs → chunk-KX3OP6VP.mjs} +2 -2
- package/dist/{chunk-D456ZFPQ.mjs → chunk-MIIJ5O3O.mjs} +2 -2
- package/dist/{chunk-PP3W5PQQ.mjs → chunk-MS746XNY.mjs} +285 -118
- package/dist/{chunk-WM5KJSQV.mjs → chunk-NFA3WHJU.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-ZUJCUPVS.mjs → chunk-OETLM4EH.mjs} +1 -1
- package/dist/{chunk-MYN4K5IS.mjs → chunk-OLXFIUIZ.mjs} +1 -1
- package/dist/{chunk-P4F4UWLC.mjs → chunk-PSO7V7VW.mjs} +17 -3
- package/dist/{chunk-X3EQCQLP.mjs → chunk-Q4NLL66U.mjs} +2 -2
- package/dist/{chunk-WC5KAWXQ.mjs → chunk-QJFKL4NK.mjs} +4 -4
- package/dist/{chunk-MSIN4NM3.mjs → chunk-S3JHYPH7.mjs} +6 -1
- package/dist/{chunk-3IE3A74H.mjs → chunk-SSITKQU6.mjs} +3 -3
- package/dist/{chunk-B3MO37YF.mjs → chunk-TANKI7TG.mjs} +1 -1
- package/dist/{chunk-Z7XSHKOF.mjs → chunk-TI4G3SIG.mjs} +63 -144
- package/dist/{chunk-OGYSPNL2.mjs → chunk-TT3UDDL3.mjs} +7 -7
- package/dist/{react-native/chunks/chunk-4FUCRLSA.mjs → chunk-VHIOGDVD.mjs} +1 -1
- package/dist/{chunk-AXDB6KUQ.mjs → chunk-W3JH2IK6.mjs} +4 -4
- package/dist/{chunk-CERPUGRK.mjs → chunk-WIT6B5OB.mjs} +2 -2
- package/dist/{chunk-MI6UZGUR.mjs → chunk-XGVSG6ZU.mjs} +3 -3
- package/dist/{chunk-6I6OO6B2.mjs → chunk-XWPNRHTG.mjs} +4 -4
- package/dist/{chunk-WTMCGZ74.mjs → chunk-YHZSSPG3.mjs} +2 -2
- package/dist/chunk-ZYEEOZTT.mjs +52 -0
- package/dist/{client-BhNxHtcZ.d.ts → client-B0NPZSKY.d.ts} +4 -4
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -2
- package/dist/{config-P9zW3Zz7.d.ts → config-B3-sw887.d.ts} +3 -3
- package/dist/{config-BJSL7piq.d.ts → config-DL_nkguq.d.ts} +4 -4
- package/dist/{direct-executor-GWJXDFLD.mjs → direct-executor-PCBB6FCO.mjs} +4 -4
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +33 -3
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +12 -7
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +167 -91
- package/dist/{durability-config-VWS2K24E.mjs → durability-config-HWKWP35M.mjs} +34 -2
- package/dist/embeddings/openai.d.ts +7 -3
- package/dist/embeddings/openai.mjs +37 -31
- package/dist/{engine-DoFnkRoF.d.ts → engine-DGG_yX0p.d.ts} +3 -3
- package/dist/{index-B_vmxpgW.d.ts → index-fX_FbuRT.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +4 -4
- package/dist/index.react-native.mjs +4 -4
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-GNTDYDOZ.mjs → long-query-thread-4ALYIIC4.mjs} +14 -14
- package/dist/msgpack-leaf-6ZBZWFRD.mjs +9 -0
- package/dist/{optional-reads-UCS23AWP.mjs → optional-reads-ESDVS225.mjs} +18 -16
- package/dist/{optional-writes-2AHJCGTY.mjs → optional-writes-2OK4TOLY.mjs} +9 -8
- package/dist/{paged-partition-archive-GBAHDT55.mjs → paged-partition-archive-Y5OAKNRB.mjs} +13 -13
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +4 -4
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -14
- package/dist/{query-actions-IDCUA7UV.mjs → query-actions-UQVQZFIJ.mjs} +4 -4
- package/dist/query-suggestions/merge-worker.mjs +7 -7
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +3 -3
- package/dist/query-suggestions.mjs +5 -5
- package/dist/{chunk-F26DOOUH.mjs → react-native/chunks/chunk-4OOUY2K7.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-LPX26NNL.mjs → chunk-5MYHYBET.mjs} +286 -118
- package/dist/react-native/chunks/{chunk-MRH36RN3.mjs → chunk-AVCMTWET.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-HTBY4GON.mjs → chunk-GS6LGXY3.mjs} +1 -1
- package/dist/{chunk-UHLFCGZ3.mjs → react-native/chunks/chunk-LE4DTMDL.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-TG56N5JC.mjs → chunk-VIO5IPF7.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-UO44MA5M.mjs → chunk-VRI5XH2M.mjs} +7 -1
- package/dist/{chunk-R2PJKSNL.mjs → react-native/chunks/chunk-XGS2UYVI.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-AJAH6X7W.mjs → chunk-XW6GZQWP.mjs} +4 -4
- package/dist/react-native/chunks/{chunk-SCGBU47K.mjs → chunk-Z44S6WW5.mjs} +2 -2
- package/dist/react-native/chunks/{durability-config-FCSD4MXY.mjs → durability-config-HJETGZVB.mjs} +34 -2
- package/dist/react-native/chunks/msgpack-leaf-4MIGXWZT.mjs +7 -0
- package/dist/react-native/chunks/{optional-reads-4XCJ34QG.mjs → optional-reads-TN3PT6M4.mjs} +18 -16
- package/dist/react-native/chunks/{optional-writes-XYZVTBFB.mjs → optional-writes-LZDHB7FX.mjs} +8 -7
- package/dist/react-native/chunks/{restore-queue-NOKQZJ2Z.mjs → restore-queue-J7SB263E.mjs} +7 -7
- package/dist/react-native/chunks/{segment-codec-leaf-3ZXBJVKH.mjs → segment-codec-leaf-PEKX7TBL.mjs} +5 -5
- package/dist/react-native/chunks/{segment-merge-leaf-3Z7MMPWE.mjs → segment-merge-leaf-I2EC7H6R.mjs} +6 -6
- package/dist/react-native/chunks/{storage-leaf-KJOX3MMW.mjs → storage-leaf-RQXM7HBO.mjs} +1481 -314
- package/dist/react-native/chunks/{store-leaf-QGLVSO5X.mjs → store-leaf-EZGCLSLM.mjs} +6 -6
- package/dist/react-native/chunks/{suggestions-leaf-6GI5ZSW6.mjs → suggestions-leaf-JNGH2JYC.mjs} +6 -6
- package/dist/react-native/chunks/{worker-copies-ALOD7TSP.mjs → worker-copies-2OWLLBCV.mjs} +5 -5
- package/dist/react-native/partial-words.mjs +4 -4
- package/dist/react-native/pattern.mjs +2 -2
- package/dist/react-native/query-suggestions.mjs +4 -4
- package/dist/react-native/sync.mjs +4 -4
- package/dist/react-native/synonyms.mjs +4 -4
- package/dist/react.d.ts +10 -10
- package/dist/react.mjs +0 -2
- package/dist/{rerank-HB6DU6GS.mjs → rerank-TQ4SU76L.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +19 -15
- package/dist/{restore-queue-VUQHGCXE.mjs → restore-queue-W7U2X6V3.mjs} +7 -7
- package/dist/{schema-CzG9OUw5.d.ts → schema-CFnQ-WI3.d.ts} +1 -1
- package/dist/{segment-codec-leaf-TW2FPAAT.mjs → segment-codec-leaf-WGP4FBXF.mjs} +5 -5
- package/dist/{segment-merge-leaf-BS3QV5PZ.mjs → segment-merge-leaf-FYWJQSPL.mjs} +5 -5
- package/dist/server.d.ts +86 -12
- package/dist/server.mjs +2243 -1918
- package/dist/{storage-leaf-5IG2HPNY.mjs → storage-leaf-GRG44RQA.mjs} +1193 -219
- package/dist/{store-leaf-ZWSOK2DV.mjs → store-leaf-YDJVH73G.mjs} +6 -6
- package/dist/{suggestions-leaf-MUZNKQVF.mjs → suggestions-leaf-5UFJV7SC.mjs} +8 -8
- package/dist/sync/filesystem.d.ts +4 -4
- package/dist/sync/filesystem.mjs +12 -12
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +4 -4
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +4 -4
- package/dist/sync.d.ts +5 -5
- package/dist/sync.mjs +4 -4
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +4 -4
- package/dist/{tasks-DrLiolTG.d.ts → tasks-B2MDjPL0.d.ts} +4 -4
- package/dist/{thread-6ZBCENAP.mjs → thread-6YLVL2NC.mjs} +13 -13
- package/dist/{types-pS-mAJmX.d.ts → types-DGjk684l.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/{worker-copies-32HWFUH7.mjs → worker-copies-IP4L7KEW.mjs} +7 -7
- package/dist/workers/entry.mjs +16 -16
- package/package.json +12 -10
- package/dist/chunk-HLPRHOSO.mjs +0 -17
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +0 -9
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +0 -7
package/dist/server.mjs
CHANGED
|
@@ -1,43 +1,43 @@
|
|
|
1
|
-
import { answerRequest } from './chunk-
|
|
1
|
+
import { answerRequest } from './chunk-W3JH2IK6.mjs';
|
|
2
2
|
import { resolveVectorText } from './chunk-NJJN4FMM.mjs';
|
|
3
|
-
import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, registerFeedRoute, SYNC_FEED_ROUTE, CHANGES_FEED_ROUTE, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, createSyncSnapshotHandler, acceptorOf, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, PROMETHEUS_TEXT_CONTENT_TYPE, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, createCloseAtExitWords, createDepartureWord, departureConfirmed, refusedRegex, toHttpError,
|
|
4
|
-
export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, CHANGE_ANALYSIS_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-
|
|
5
|
-
import { startSyncRetentionSweep, publishedSyncSnapshot, publishedArchiveLayout } from './chunk-
|
|
3
|
+
import { DEFAULT_MAX_RETAINED_TASKS, isTerminalTaskStatus, TASK_RECORD_LOAD_CONCURRENCY, loadUWebSockets, SERVER_CLOSE_DRAIN_DEADLINE_MS, registerFeedRoute, SYNC_FEED_ROUTE, CHANGES_FEED_ROUTE, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, isTaskType, isTaskStatus, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, createSyncSnapshotHandler, acceptorOf, createServerCloseSignal, signalServerClose, LeavingResponse, refuseAndClose, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, PROMETHEUS_TEXT_CONTENT_TYPE, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, createCloseAtExitWords, createDepartureWord, departureConfirmed, threadCloseAcknowledgedWithin, SERVER_CLOSE_ACKNOWLEDGE_GRACE_MS, threadCloseAcknowledged, refusedRegex, toHttpError, parseJsonObject, validateCreateIndex, rejectInvalid, isRemoveOnlyBatch, validateBatch, serializeBatchResult, parseWriteJsonObject, validateDocumentBody, validateSuggest, readQueryBody, withServerQueryLimits, requestSignal, respondAnswer, encodeQueryResult, parseTaskListQuery, parseJsonOptional, validateChangeAnalysis, validateRebalance, parseJson, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, respondPreparingError, parseRefusalOf, IMPORT_FLUSHES_IN_FLIGHT, isThreadFeedNote, clearGateSlot, clearBodyBudgetSlot, confirmDeparture, SERVER_CAPABILITIES, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-4GCHJ3XD.mjs';
|
|
4
|
+
export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, CHANGE_ANALYSIS_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-4GCHJ3XD.mjs';
|
|
5
|
+
import { startSyncRetentionSweep, publishedSyncSnapshot, publishedArchiveLayout } from './chunk-TT3UDDL3.mjs';
|
|
6
6
|
import { SYNC_CLOSE_RESYNC, SYNC_CLOSE_CREDENTIAL_EXPIRED, SYNC_CLOSE_POLICY, MAX_DEVICE_MESSAGE_BYTES, SYNC_CLOSE_TOO_LARGE, deviceMessageOf, SYNC_CLOSE_SERVER_ERROR, positionList, positionsByPartition, CHANGES_CLOSE_RESYNC, CHANGES_CLOSE_POLICY, MAX_SUBSCRIBER_MESSAGE_BYTES, CHANGES_CLOSE_TOO_LARGE, closeReasonOf, CHANGES_CLOSE_CREDENTIAL_EXPIRED, CHANGES_CLOSE_SETTING, CHANGES_CLOSE_SERVER_ERROR, CHANGES_CLOSE_GOING_AWAY, SYNC_CLOSE_GOING_AWAY } from './chunk-TACP6VK6.mjs';
|
|
7
|
-
import { syncServingSourceOf, changesServingSourceOf, writeBlockLookupOf, indexReadsOf, validateBackupName, readRestorePlacement } from './chunk-
|
|
8
|
-
import { isRecord, validateIndexName } from './chunk-
|
|
7
|
+
import { syncServingSourceOf, changesServingSourceOf, writeBlockLookupOf, indexReadsOf, validateBackupName, readRestorePlacement } from './chunk-XGVSG6ZU.mjs';
|
|
8
|
+
import { isRecord, validateIndexName } from './chunk-BBIAUHAB.mjs';
|
|
9
9
|
import { executeUntilAnswered } from './chunk-4H4RG2FR.mjs';
|
|
10
|
-
import './chunk-
|
|
11
|
-
import { engineChangeSource, createSemaphore, createByteBudget, changesNotEnabled, subscriberMessageOf, ChangeCursor, changePositionList, seqNosByPartition } from './chunk-
|
|
10
|
+
import './chunk-NFA3WHJU.mjs';
|
|
11
|
+
import { engineChangeSource, createSemaphore, createByteBudget, changesNotEnabled, subscriberMessageOf, ChangeCursor, changePositionList, seqNosByPartition } from './chunk-SSITKQU6.mjs';
|
|
12
12
|
import './chunk-M2Z5WVWW.mjs';
|
|
13
|
-
import { DEFAULT_MAX_FEED_BUFFERED_BYTES, DEFAULT_MAX_FEED_CATCH_UPS, DEFAULT_MAX_FEED_CONNECTIONS, DEFAULT_FEED_WINDOW_BYTES, estimatedArchiveBytes, TIMER_DELAY_LIMIT_MS, FEED_HEAD_RETRY_MS, ENTRIES_MESSAGE_LIMIT_BYTES } from './chunk-
|
|
13
|
+
import { DEFAULT_MAX_FEED_BUFFERED_BYTES, DEFAULT_MAX_FEED_CATCH_UPS, DEFAULT_MAX_FEED_CONNECTIONS, DEFAULT_FEED_WINDOW_BYTES, estimatedArchiveBytes, TIMER_DELAY_LIMIT_MS, FEED_HEAD_RETRY_MS, ENTRIES_MESSAGE_LIMIT_BYTES } from './chunk-XWPNRHTG.mjs';
|
|
14
14
|
import './chunk-JDDXV6UF.mjs';
|
|
15
|
-
import { manifestFileRefs } from './chunk-
|
|
16
|
-
import './chunk-MSIN4NM3.mjs';
|
|
15
|
+
import { manifestFileRefs } from './chunk-TANKI7TG.mjs';
|
|
17
16
|
import { encode as encode$1 } from './chunk-S5QMWECL.mjs';
|
|
18
17
|
import './chunk-LNGE7SLY.mjs';
|
|
19
|
-
import './chunk-
|
|
20
|
-
import './chunk-
|
|
21
|
-
import './chunk-
|
|
18
|
+
import './chunk-OETLM4EH.mjs';
|
|
19
|
+
import './chunk-VHIOGDVD.mjs';
|
|
20
|
+
import './chunk-DA32WXBT.mjs';
|
|
22
21
|
import './chunk-R724DJSG.mjs';
|
|
23
22
|
import './chunk-V6EUBAIT.mjs';
|
|
24
|
-
import './chunk-
|
|
25
|
-
import { querySuggestions } from './chunk-
|
|
23
|
+
import './chunk-WIT6B5OB.mjs';
|
|
24
|
+
import { querySuggestions } from './chunk-33ZMFOAR.mjs';
|
|
26
25
|
import './chunk-S2POFTLN.mjs';
|
|
27
|
-
import { rerankWindowCapOf, engineCoreOf, readMemoryFigures, listenForSyncChanges, MAX_RERANK_WINDOW, bindQuerySignal, clearQuerySignal, providedDocId, loadRerankRuntime, rulesOfStoredSynonyms } from './chunk-
|
|
26
|
+
import { rerankWindowCapOf, engineCoreOf, readMemoryFigures, listenForSyncChanges, MAX_RERANK_WINDOW, bindQuerySignal, clearQuerySignal, providedDocId, loadRerankRuntime, rulesOfStoredSynonyms } from './chunk-MS746XNY.mjs';
|
|
28
27
|
import './chunk-2CKDCC65.mjs';
|
|
29
28
|
import './chunk-2AFZE7Y3.mjs';
|
|
30
|
-
import {
|
|
29
|
+
import { decodeBoundedMessage } from './chunk-5GYGHPCH.mjs';
|
|
31
30
|
import './chunk-BUMREPLU.mjs';
|
|
32
31
|
import './chunk-7T4LAYT4.mjs';
|
|
33
|
-
import './chunk-
|
|
34
|
-
import './chunk-
|
|
32
|
+
import './chunk-KQWTS7AM.mjs';
|
|
33
|
+
import './chunk-4WQNWAQE.mjs';
|
|
35
34
|
import './chunk-U2BKYEC7.mjs';
|
|
36
35
|
import { detectRuntime } from './chunk-FJXJZUWF.mjs';
|
|
37
36
|
import { startIdleHeapCleanup } from './chunk-5EI5CXO6.mjs';
|
|
38
37
|
import './chunk-3ECIRFS5.mjs';
|
|
39
38
|
import './chunk-ASZLNVWW.mjs';
|
|
40
39
|
import './chunk-ZXX5GZEX.mjs';
|
|
40
|
+
import './chunk-S3JHYPH7.mjs';
|
|
41
41
|
import './chunk-GR5GMFJJ.mjs';
|
|
42
42
|
import './chunk-4PY4JYMY.mjs';
|
|
43
43
|
import { ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-ZKOQR7EA.mjs';
|
|
@@ -45,29 +45,83 @@ export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilEr
|
|
|
45
45
|
import { compareCodePoints } from './chunk-TGPKS2UM.mjs';
|
|
46
46
|
import { randomUUID } from 'crypto';
|
|
47
47
|
import { encode } from '@msgpack/msgpack';
|
|
48
|
-
import { createReadStream } from 'fs';
|
|
49
48
|
import { MessageChannel } from 'worker_threads';
|
|
49
|
+
import { createReadStream } from 'fs';
|
|
50
50
|
|
|
51
|
-
// src/server/
|
|
52
|
-
function
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
51
|
+
// src/server/wall-clock-wait.ts
|
|
52
|
+
function waitUntilWallClock(deadline) {
|
|
53
|
+
let timer;
|
|
54
|
+
let cancelled = false;
|
|
55
|
+
const reached = new Promise((resolve) => {
|
|
56
|
+
const check = () => {
|
|
57
|
+
if (cancelled) return;
|
|
58
|
+
const remaining = deadline - Date.now();
|
|
59
|
+
if (remaining <= 0) {
|
|
60
|
+
resolve();
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
timer = setTimeout(check, remaining);
|
|
64
|
+
};
|
|
65
|
+
check();
|
|
66
|
+
});
|
|
67
|
+
return {
|
|
68
|
+
reached,
|
|
69
|
+
cancel() {
|
|
70
|
+
cancelled = true;
|
|
71
|
+
clearTimeout(timer);
|
|
58
72
|
}
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// src/server/closing-requests.ts
|
|
77
|
+
var CLOSING_REFUSAL = {
|
|
78
|
+
error: {
|
|
79
|
+
code: ErrorCodes.WORKER_CRASHED,
|
|
80
|
+
message: "The server is closing, so retry the request on a new connection"
|
|
59
81
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
}
|
|
82
|
+
};
|
|
83
|
+
function watchClosingRequests() {
|
|
84
|
+
let closing = false;
|
|
85
|
+
const running = /* @__PURE__ */ new Set();
|
|
86
|
+
const waitingForIdle = [];
|
|
87
|
+
function settle(request) {
|
|
88
|
+
if (!running.delete(request) || running.size > 0) return;
|
|
89
|
+
for (const resume of waitingForIdle.splice(0)) resume();
|
|
69
90
|
}
|
|
70
|
-
return {
|
|
91
|
+
return {
|
|
92
|
+
accept(res) {
|
|
93
|
+
if (!closing) return true;
|
|
94
|
+
refuseAndClose(res, 503, CLOSING_REFUSAL);
|
|
95
|
+
return false;
|
|
96
|
+
},
|
|
97
|
+
respondThrough(res) {
|
|
98
|
+
return new LeavingResponse(res, () => closing);
|
|
99
|
+
},
|
|
100
|
+
track(res, abort) {
|
|
101
|
+
const request = {};
|
|
102
|
+
running.add(request);
|
|
103
|
+
let handled = false;
|
|
104
|
+
const finish = () => {
|
|
105
|
+
if (abort.aborted || handled && (!(res instanceof LeavingResponse) || res.isFinished)) settle(request);
|
|
106
|
+
};
|
|
107
|
+
if (res instanceof LeavingResponse) res.whenFinished(finish);
|
|
108
|
+
abort.onAbort(finish);
|
|
109
|
+
return () => {
|
|
110
|
+
handled = true;
|
|
111
|
+
finish();
|
|
112
|
+
};
|
|
113
|
+
},
|
|
114
|
+
async close(deadlineMs) {
|
|
115
|
+
closing = true;
|
|
116
|
+
if (running.size === 0) return;
|
|
117
|
+
const deadline = waitUntilWallClock(Date.now() + deadlineMs);
|
|
118
|
+
try {
|
|
119
|
+
await Promise.race([new Promise((resolve) => waitingForIdle.push(resolve)), deadline.reached]);
|
|
120
|
+
} finally {
|
|
121
|
+
deadline.cancel();
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
};
|
|
71
125
|
}
|
|
72
126
|
|
|
73
127
|
// src/changes/encode.ts
|
|
@@ -181,7 +235,7 @@ var ChangesSession = class {
|
|
|
181
235
|
}
|
|
182
236
|
let message;
|
|
183
237
|
try {
|
|
184
|
-
message = subscriberMessageOf(
|
|
238
|
+
message = subscriberMessageOf(decodeBoundedMessage(bytes));
|
|
185
239
|
} catch {
|
|
186
240
|
message = null;
|
|
187
241
|
}
|
|
@@ -630,7 +684,7 @@ var FeedSession = class {
|
|
|
630
684
|
}
|
|
631
685
|
let message;
|
|
632
686
|
try {
|
|
633
|
-
message = deviceMessageOf(
|
|
687
|
+
message = deviceMessageOf(decodeBoundedMessage(bytes));
|
|
634
688
|
} catch {
|
|
635
689
|
message = null;
|
|
636
690
|
}
|
|
@@ -1254,2145 +1308,2163 @@ function startFeeds(engine, core, settings) {
|
|
|
1254
1308
|
if (hosted.sync === void 0 && hosted.changes === void 0) return null;
|
|
1255
1309
|
return { host: createFeedHost(hosted, settings.authorize), syncServing };
|
|
1256
1310
|
}
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1311
|
+
|
|
1312
|
+
// src/server/options-validation.ts
|
|
1313
|
+
function isLoopback(host) {
|
|
1314
|
+
return host === "127.0.0.1" || host === "::1" || host === "localhost" || host.startsWith("127.");
|
|
1260
1315
|
}
|
|
1261
|
-
function
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1316
|
+
function assertSecureBinding(host, options) {
|
|
1317
|
+
if (isLoopback(host) || options.onRequest || options.allowInsecure) return;
|
|
1318
|
+
throw new NarsilError(
|
|
1319
|
+
ErrorCodes.CONFIG_INVALID,
|
|
1320
|
+
`Refusing to start: bound to a non-loopback address ("${host}") with no onRequest auth hook. The server exposes destructive admin endpoints (restore, drop, clear, rebalance, optimize). Resolve one of: (1) set options.onRequest to authenticate requests, (2) bind to 127.0.0.1, or (3) set options.allowInsecure: true if this address is on a trusted private network.`,
|
|
1321
|
+
{ host }
|
|
1322
|
+
);
|
|
1323
|
+
}
|
|
1324
|
+
var ENGINE_METHOD_SAMPLE = ["createIndex", "listIndexes", "insert", "query", "get", "shutdown"];
|
|
1325
|
+
var LIMITS_UNBOUNDED_AT_ZERO = [
|
|
1326
|
+
"maxBodyBytes",
|
|
1327
|
+
"maxImportBytes",
|
|
1328
|
+
"maxRestoreBytes",
|
|
1329
|
+
"maxLineBytes",
|
|
1330
|
+
"maxImportErrors",
|
|
1331
|
+
"maxConcurrentRequests",
|
|
1332
|
+
"maxConcurrentTasks"
|
|
1333
|
+
];
|
|
1334
|
+
var COUNTED_LIMITS = [
|
|
1335
|
+
"importBatchSize",
|
|
1336
|
+
"maxResultWindow",
|
|
1337
|
+
"maxRerankWindow",
|
|
1338
|
+
"maxFetchDocuments",
|
|
1339
|
+
"maxTaskPageSize"
|
|
1340
|
+
];
|
|
1341
|
+
function assertServerEngine(engine) {
|
|
1342
|
+
const candidate = engine;
|
|
1343
|
+
if (typeof candidate !== "object" || candidate === null) {
|
|
1344
|
+
throw new NarsilError(
|
|
1345
|
+
ErrorCodes.CONFIG_INVALID,
|
|
1346
|
+
"The first argument to createServer must be the engine that createNarsil returns, with the options second",
|
|
1347
|
+
{ received: candidate === null ? "null" : typeof candidate }
|
|
1348
|
+
);
|
|
1289
1349
|
}
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1350
|
+
const members = candidate;
|
|
1351
|
+
const missing = ENGINE_METHOD_SAMPLE.filter((method) => typeof members[method] !== "function");
|
|
1352
|
+
if (missing.length === 0) return;
|
|
1353
|
+
throw new NarsilError(
|
|
1354
|
+
ErrorCodes.CONFIG_INVALID,
|
|
1355
|
+
`The first argument to createServer offers no ${missing.join(", ")}, so it is some value other than the engine that createNarsil returns`,
|
|
1356
|
+
{ missing }
|
|
1357
|
+
);
|
|
1358
|
+
}
|
|
1359
|
+
function assertLimit(field, value, minimum) {
|
|
1360
|
+
if (value === void 0) return;
|
|
1361
|
+
if (Number.isInteger(value) && value >= minimum) return;
|
|
1362
|
+
throw new NarsilError(
|
|
1363
|
+
ErrorCodes.CONFIG_INVALID,
|
|
1364
|
+
`limits.${field} must be a whole number of at least ${minimum}; however, this configuration sets it to ${String(value)}`,
|
|
1365
|
+
{ field, value, minimum }
|
|
1366
|
+
);
|
|
1367
|
+
}
|
|
1368
|
+
function assertFraction(field, value) {
|
|
1369
|
+
if (value === void 0) return;
|
|
1370
|
+
if (Number.isFinite(value) && value >= 0 && value <= MAX_BODY_BYTES_IN_FLIGHT_FRACTION) return;
|
|
1371
|
+
throw new NarsilError(
|
|
1372
|
+
ErrorCodes.CONFIG_INVALID,
|
|
1373
|
+
`limits.${field} must be a fraction from 0 to ${MAX_BODY_BYTES_IN_FLIGHT_FRACTION}; however, this configuration sets it to ${String(value)}`,
|
|
1374
|
+
{ field, value, maximum: MAX_BODY_BYTES_IN_FLIGHT_FRACTION }
|
|
1375
|
+
);
|
|
1376
|
+
}
|
|
1377
|
+
function assertRerankWindowCap(value, engineCap) {
|
|
1378
|
+
if (value === void 0) return;
|
|
1379
|
+
const cap = Math.min(MAX_RERANK_WINDOW, engineCap ?? MAX_RERANK_WINDOW);
|
|
1380
|
+
if (value <= cap) return;
|
|
1381
|
+
throw new NarsilError(
|
|
1382
|
+
ErrorCodes.CONFIG_INVALID,
|
|
1383
|
+
`limits.maxRerankWindow must be at most ${cap}, the largest window that the engine accepts; however, this configuration sets it to ${value}`,
|
|
1384
|
+
{ field: "maxRerankWindow", value, maximum: cap }
|
|
1385
|
+
);
|
|
1386
|
+
}
|
|
1387
|
+
function assertServerLimits(limits, engineRerankWindow) {
|
|
1388
|
+
if (limits === void 0) return;
|
|
1389
|
+
for (const field of LIMITS_UNBOUNDED_AT_ZERO) assertLimit(field, limits[field], MIN_UNBOUNDED_SERVER_LIMIT);
|
|
1390
|
+
for (const field of COUNTED_LIMITS) assertLimit(field, limits[field], MIN_COUNTED_SERVER_LIMIT);
|
|
1391
|
+
assertFraction("maxBodyBytesInFlightFraction", limits.maxBodyBytesInFlightFraction);
|
|
1392
|
+
assertRerankWindowCap(limits.maxRerankWindow, engineRerankWindow);
|
|
1393
|
+
}
|
|
1394
|
+
function assertQuerySuggestionsOptions(options) {
|
|
1395
|
+
if (options === void 0) return;
|
|
1396
|
+
const identify = typeof options === "object" && options !== null ? Reflect.get(options, "identify") : void 0;
|
|
1397
|
+
if (typeof identify === "function") return;
|
|
1398
|
+
throw new NarsilError(
|
|
1399
|
+
ErrorCodes.CONFIG_INVALID,
|
|
1400
|
+
"querySuggestions must carry an identify function that returns the person and the group of a request",
|
|
1401
|
+
{ field: "querySuggestions.identify" }
|
|
1402
|
+
);
|
|
1403
|
+
}
|
|
1404
|
+
function assertAllowRegex(allowRegex) {
|
|
1405
|
+
if (allowRegex === void 0 || typeof allowRegex === "boolean") return;
|
|
1406
|
+
throw new NarsilError(
|
|
1407
|
+
ErrorCodes.CONFIG_INVALID,
|
|
1408
|
+
`allowRegex must be true or false; however, this configuration sets it to ${String(allowRegex)}`,
|
|
1409
|
+
{ field: "allowRegex", value: String(allowRegex) }
|
|
1410
|
+
);
|
|
1411
|
+
}
|
|
1412
|
+
|
|
1413
|
+
// package.json
|
|
1414
|
+
var version = "0.4.2-next.271";
|
|
1415
|
+
|
|
1416
|
+
// src/server/package-version.ts
|
|
1417
|
+
function publishedPackageVersion() {
|
|
1418
|
+
return version;
|
|
1419
|
+
}
|
|
1420
|
+
|
|
1421
|
+
// src/server/request-threads/close-deadline.ts
|
|
1422
|
+
async function closeThreadsByDeadline(words, closed, deadline) {
|
|
1423
|
+
const drained = waitUntilWallClock(deadline);
|
|
1424
|
+
try {
|
|
1425
|
+
await Promise.race([closed, drained.reached]);
|
|
1426
|
+
closeEveryThreadAtExit(words, 0);
|
|
1427
|
+
await Promise.all([...words].map((word) => threadCloseAcknowledgedWithin(word, SERVER_CLOSE_ACKNOWLEDGE_GRACE_MS)));
|
|
1428
|
+
const unacknowledged = [...words].filter((word) => !threadCloseAcknowledged(word)).length;
|
|
1429
|
+
if (unacknowledged > 0) {
|
|
1430
|
+
throw new NarsilError(
|
|
1431
|
+
ErrorCodes.WORKER_TIMEOUT,
|
|
1432
|
+
`Request threads that did not acknowledge closing their connections: ${unacknowledged}; server close deadline: ${SERVER_CLOSE_DRAIN_DEADLINE_MS}ms`,
|
|
1433
|
+
{ threads: unacknowledged, deadlineMs: SERVER_CLOSE_DRAIN_DEADLINE_MS }
|
|
1434
|
+
);
|
|
1295
1435
|
}
|
|
1436
|
+
} finally {
|
|
1437
|
+
drained.cancel();
|
|
1296
1438
|
}
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
} catch (err) {
|
|
1304
|
-
respondError(ctx, err);
|
|
1305
|
-
}
|
|
1439
|
+
}
|
|
1440
|
+
|
|
1441
|
+
// src/server/request-threads/relay-answers.ts
|
|
1442
|
+
async function answerAuthorize(authorize, request) {
|
|
1443
|
+
if (authorize === void 0) {
|
|
1444
|
+
return { type: "authorized", requestId: request.requestId, denial: null, hookError: false, expiresAt: null };
|
|
1306
1445
|
}
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
}
|
|
1446
|
+
const authorization = await authorize(request.context);
|
|
1447
|
+
if (authorization.allowed) {
|
|
1448
|
+
return {
|
|
1449
|
+
type: "authorized",
|
|
1450
|
+
requestId: request.requestId,
|
|
1451
|
+
denial: null,
|
|
1452
|
+
hookError: false,
|
|
1453
|
+
expiresAt: authorization.expiresAt
|
|
1454
|
+
};
|
|
1316
1455
|
}
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1456
|
+
return {
|
|
1457
|
+
type: "authorized",
|
|
1458
|
+
requestId: request.requestId,
|
|
1459
|
+
denial: authorization.denial,
|
|
1460
|
+
hookError: authorization.denial === null,
|
|
1461
|
+
expiresAt: null
|
|
1462
|
+
};
|
|
1463
|
+
}
|
|
1464
|
+
async function answerEmbed(core, request, signal) {
|
|
1465
|
+
try {
|
|
1466
|
+
const entry = core.requireIndex(request.indexName);
|
|
1467
|
+
const resolved = await resolveVectorText(
|
|
1468
|
+
{ vector: request.vector },
|
|
1469
|
+
entry.embeddingAdapter,
|
|
1470
|
+
signal === void 0 ? core.abortController.signal : AbortSignal.any([core.abortController.signal, signal]),
|
|
1471
|
+
entry.embeddingAdapterName
|
|
1472
|
+
);
|
|
1473
|
+
const value = resolved.vector?.value;
|
|
1474
|
+
return {
|
|
1475
|
+
type: "embedded",
|
|
1476
|
+
requestId: request.requestId,
|
|
1477
|
+
value: value === void 0 ? null : Array.from(value),
|
|
1478
|
+
error: null
|
|
1479
|
+
};
|
|
1480
|
+
} catch (err) {
|
|
1481
|
+
if (!isNarsilError(err)) console.warn("Embedding a query for a request thread failed:", err);
|
|
1482
|
+
return { type: "embedded", requestId: request.requestId, value: null, error: toHttpError(err).body };
|
|
1323
1483
|
}
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
respondJson(ctx, { ok: true });
|
|
1334
|
-
} catch (err) {
|
|
1335
|
-
respondError(ctx, err);
|
|
1336
|
-
}
|
|
1337
|
-
}
|
|
1338
|
-
async function updateBm25(ctx) {
|
|
1339
|
-
const body = parseJson(ctx);
|
|
1340
|
-
if (body === void 0) return;
|
|
1341
|
-
try {
|
|
1342
|
-
await engine.updateBM25(ctx.params[0], body);
|
|
1343
|
-
respondJson(ctx, { ok: true });
|
|
1344
|
-
} catch (err) {
|
|
1345
|
-
respondError(ctx, err);
|
|
1346
|
-
}
|
|
1347
|
-
}
|
|
1348
|
-
async function memory(ctx) {
|
|
1349
|
-
try {
|
|
1350
|
-
const stats = await engine.getMemoryStats();
|
|
1351
|
-
const response = { ...stats, requestThreads: deps.requestThreadCount() };
|
|
1352
|
-
respondJson(ctx, response);
|
|
1353
|
-
} catch (err) {
|
|
1354
|
-
respondError(ctx, err);
|
|
1355
|
-
}
|
|
1356
|
-
}
|
|
1357
|
-
async function requireIndexExists(name) {
|
|
1358
|
-
await indexReadsOf(engine).getStats(name);
|
|
1359
|
-
}
|
|
1360
|
-
async function optimize(ctx) {
|
|
1361
|
-
const body = parseJsonOptional(ctx);
|
|
1362
|
-
if (!body) return;
|
|
1363
|
-
const name = ctx.params[0];
|
|
1364
|
-
try {
|
|
1365
|
-
await requireIndexExists(name);
|
|
1366
|
-
} catch (err) {
|
|
1367
|
-
respondError(ctx, err);
|
|
1368
|
-
return;
|
|
1369
|
-
}
|
|
1370
|
-
const record = await tasks.start("optimizeVectors", name, () => engine.optimizeVectors(name, body.field));
|
|
1371
|
-
respondJson(ctx, record, 202);
|
|
1372
|
-
}
|
|
1373
|
-
async function rebuildAnalysis(ctx) {
|
|
1374
|
-
const name = ctx.params[0];
|
|
1375
|
-
try {
|
|
1376
|
-
await requireIndexExists(name);
|
|
1377
|
-
} catch (err) {
|
|
1378
|
-
respondError(ctx, err);
|
|
1379
|
-
return;
|
|
1380
|
-
}
|
|
1381
|
-
const record = await tasks.start("rebuildAnalysis", name, () => engine.rebuildAnalysis(name));
|
|
1382
|
-
respondJson(ctx, record, 202);
|
|
1383
|
-
}
|
|
1384
|
-
async function changeAnalysis(ctx) {
|
|
1385
|
-
const body = parseJsonOptional(ctx);
|
|
1386
|
-
if (!body) return;
|
|
1387
|
-
const failure = validateChangeAnalysis(body);
|
|
1388
|
-
if (failure) {
|
|
1389
|
-
rejectInvalid(ctx, failure);
|
|
1390
|
-
return;
|
|
1391
|
-
}
|
|
1392
|
-
const name = ctx.params[0];
|
|
1393
|
-
try {
|
|
1394
|
-
await requireIndexExists(name);
|
|
1395
|
-
} catch (err) {
|
|
1396
|
-
respondError(ctx, err);
|
|
1397
|
-
return;
|
|
1398
|
-
}
|
|
1399
|
-
const record = await tasks.start("changeAnalysis", name, () => engine.changeAnalysis(name, body));
|
|
1400
|
-
respondJson(ctx, record, 202);
|
|
1401
|
-
}
|
|
1402
|
-
async function rebalance(ctx) {
|
|
1403
|
-
const body = parseJsonOptional(ctx);
|
|
1404
|
-
if (!body) return;
|
|
1405
|
-
const failure = validateRebalance(body);
|
|
1406
|
-
if (failure) {
|
|
1407
|
-
rejectInvalid(ctx, failure);
|
|
1408
|
-
return;
|
|
1409
|
-
}
|
|
1410
|
-
const target = body.targetPartitionCount;
|
|
1411
|
-
const name = ctx.params[0];
|
|
1412
|
-
try {
|
|
1413
|
-
await requireIndexExists(name);
|
|
1414
|
-
} catch (err) {
|
|
1415
|
-
respondError(ctx, err);
|
|
1416
|
-
return;
|
|
1417
|
-
}
|
|
1418
|
-
const record = await tasks.start("rebalance", name, () => engine.rebalance(name, target));
|
|
1419
|
-
respondJson(ctx, record, 202);
|
|
1420
|
-
}
|
|
1421
|
-
async function restore(ctx) {
|
|
1422
|
-
const spooled = ctx.spooledBody;
|
|
1423
|
-
if (spooled === null || spooled.bytes === 0) {
|
|
1424
|
-
await spooled?.remove();
|
|
1425
|
-
sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
|
|
1426
|
-
return;
|
|
1427
|
-
}
|
|
1428
|
-
const name = ctx.params[0];
|
|
1429
|
-
try {
|
|
1430
|
-
const record = await tasks.start("restore", name, async () => {
|
|
1431
|
-
try {
|
|
1432
|
-
await engine.restore(name, createReadStream(spooled.path));
|
|
1433
|
-
} finally {
|
|
1434
|
-
await spooled.remove();
|
|
1435
|
-
}
|
|
1436
|
-
});
|
|
1437
|
-
respondJson(ctx, record, 202);
|
|
1438
|
-
} catch (err) {
|
|
1439
|
-
await spooled.remove();
|
|
1440
|
-
respondError(ctx, err);
|
|
1441
|
-
}
|
|
1442
|
-
}
|
|
1443
|
-
async function getTask(ctx) {
|
|
1444
|
-
const record = await tasks.get(ctx.params[0]);
|
|
1445
|
-
if (!record) {
|
|
1446
|
-
sendError(ctx.res, 404, ServerErrorCodes.TASK_NOT_FOUND, `Task "${ctx.params[0]}" not found`);
|
|
1447
|
-
return;
|
|
1448
|
-
}
|
|
1449
|
-
respondJson(ctx, record);
|
|
1450
|
-
}
|
|
1451
|
-
async function listTasks(ctx) {
|
|
1452
|
-
const parsed = parseTaskListQuery(ctx.query, limits.maxTaskPageSize);
|
|
1453
|
-
if ("failure" in parsed) {
|
|
1454
|
-
rejectInvalid(ctx, parsed.failure);
|
|
1455
|
-
return;
|
|
1456
|
-
}
|
|
1457
|
-
respondJson(ctx, await tasks.list(parsed.query));
|
|
1458
|
-
}
|
|
1459
|
-
async function cancelTask(ctx) {
|
|
1460
|
-
const taskId = ctx.params[0];
|
|
1461
|
-
const { outcome, record } = await tasks.cancel(taskId);
|
|
1462
|
-
if (outcome === "not-found") {
|
|
1463
|
-
sendError(ctx.res, 404, ServerErrorCodes.TASK_NOT_FOUND, `Task "${taskId}" not found`);
|
|
1464
|
-
return;
|
|
1465
|
-
}
|
|
1466
|
-
if (outcome === "already-finished") {
|
|
1467
|
-
sendError(
|
|
1468
|
-
ctx.res,
|
|
1469
|
-
409,
|
|
1470
|
-
ServerErrorCodes.TASK_NOT_CANCELLABLE,
|
|
1471
|
-
`Task "${taskId}" has already finished`,
|
|
1472
|
-
record === null ? void 0 : { status: record.status }
|
|
1473
|
-
);
|
|
1474
|
-
return;
|
|
1475
|
-
}
|
|
1476
|
-
if (outcome === "owned-by-another-instance") {
|
|
1477
|
-
sendError(
|
|
1478
|
-
ctx.res,
|
|
1479
|
-
409,
|
|
1480
|
-
ServerErrorCodes.TASK_OWNED_BY_ANOTHER_INSTANCE,
|
|
1481
|
-
`Task "${taskId}" runs on another server instance, which is the only one that can stop it`,
|
|
1482
|
-
record === null ? void 0 : { owner: record.owner }
|
|
1483
|
-
);
|
|
1484
|
-
return;
|
|
1485
|
-
}
|
|
1486
|
-
respondJson(ctx, record, 202);
|
|
1484
|
+
}
|
|
1485
|
+
async function answerRerank(core, request, signal) {
|
|
1486
|
+
try {
|
|
1487
|
+
const { rerankOnHost } = await loadRerankRuntime();
|
|
1488
|
+
const scores = await rerankOnHost(core, request.adapter, request.query, request.documents, signal);
|
|
1489
|
+
return { type: "reranked", requestId: request.requestId, scores, error: null };
|
|
1490
|
+
} catch (err) {
|
|
1491
|
+
if (!isNarsilError(err)) console.warn("Reranking a query for a request thread failed:", err);
|
|
1492
|
+
return { type: "reranked", requestId: request.requestId, scores: null, error: toHttpError(err).body };
|
|
1487
1493
|
}
|
|
1494
|
+
}
|
|
1495
|
+
async function answerAwaitWrites(core, request) {
|
|
1496
|
+
await core.orchestrator.awaitWrites(request.indexName).catch(() => void 0);
|
|
1497
|
+
return { type: "writes-applied", requestId: request.requestId };
|
|
1498
|
+
}
|
|
1499
|
+
async function answerFeedAdmit(feeds, request) {
|
|
1500
|
+
if (feeds !== null) return feeds.admit(request);
|
|
1488
1501
|
return {
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
partitionConfig,
|
|
1494
|
-
readSynonyms,
|
|
1495
|
-
replaceSynonyms,
|
|
1496
|
-
updateBm25,
|
|
1497
|
-
memory,
|
|
1498
|
-
optimize,
|
|
1499
|
-
rebalance,
|
|
1500
|
-
rebuildAnalysis,
|
|
1501
|
-
changeAnalysis,
|
|
1502
|
-
restore,
|
|
1503
|
-
getTask,
|
|
1504
|
-
listTasks,
|
|
1505
|
-
cancelTask
|
|
1502
|
+
type: "feed-admitted",
|
|
1503
|
+
requestId: request.requestId,
|
|
1504
|
+
feedId: null,
|
|
1505
|
+
error: { code: ServerErrorCodes.NOT_FOUND, message: "Route not found" }
|
|
1506
1506
|
};
|
|
1507
1507
|
}
|
|
1508
1508
|
|
|
1509
|
-
// src/server/
|
|
1510
|
-
function
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
}
|
|
1518
|
-
return deps.cluster;
|
|
1509
|
+
// src/server/request-threads/host.ts
|
|
1510
|
+
function threadClosed(port, clearSlots) {
|
|
1511
|
+
return new Promise((resolve) => {
|
|
1512
|
+
port.once("close", () => {
|
|
1513
|
+
clearSlots();
|
|
1514
|
+
resolve();
|
|
1515
|
+
});
|
|
1516
|
+
});
|
|
1519
1517
|
}
|
|
1520
|
-
|
|
1521
|
-
|
|
1518
|
+
var RELAYED_TYPES = /* @__PURE__ */ new Set([
|
|
1519
|
+
"request",
|
|
1520
|
+
"authorize",
|
|
1521
|
+
"embed",
|
|
1522
|
+
"rerank",
|
|
1523
|
+
"await-writes",
|
|
1524
|
+
"stream-credit",
|
|
1525
|
+
"stream-cancel",
|
|
1526
|
+
"feed-admit"
|
|
1527
|
+
]);
|
|
1528
|
+
function isRelayRequest(message) {
|
|
1529
|
+
if (typeof message !== "object" || message === null) return false;
|
|
1530
|
+
const { type, requestId } = message;
|
|
1531
|
+
if (type === "touch" || type === "leaving") return true;
|
|
1532
|
+
return typeof requestId === "number" && typeof type === "string" && RELAYED_TYPES.has(type);
|
|
1522
1533
|
}
|
|
1523
|
-
function
|
|
1524
|
-
|
|
1525
|
-
validateIndexName(indexName);
|
|
1526
|
-
return null;
|
|
1527
|
-
} catch (error) {
|
|
1528
|
-
return error instanceof Error ? error.message : String(error);
|
|
1529
|
-
}
|
|
1534
|
+
function touchIntervalFor(copyIdleTimeoutMs) {
|
|
1535
|
+
return Math.min(MAX_INDEX_TOUCH_INTERVAL_MS, Math.floor(copyIdleTimeoutMs / INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT));
|
|
1530
1536
|
}
|
|
1531
|
-
function
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
}
|
|
1537
|
+
async function startRequestThreads(options) {
|
|
1538
|
+
const found = acceptorOf(options.app);
|
|
1539
|
+
if (found === null) return null;
|
|
1540
|
+
const acceptor = found;
|
|
1541
|
+
const { core, handlers, authorize } = options;
|
|
1542
|
+
const searchHooks = core.pluginRegistry.hasHooks("beforeSearch") || core.pluginRegistry.hasHooks("afterSearch");
|
|
1543
|
+
const touchIntervalMs = touchIntervalFor(core.orchestrator.copyIdleTimeoutMs());
|
|
1544
|
+
const threads = /* @__PURE__ */ new Map();
|
|
1545
|
+
const slotUsers = /* @__PURE__ */ new Map();
|
|
1546
|
+
const longQueriesInFlight = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT);
|
|
1547
|
+
let longQueryThread = null;
|
|
1548
|
+
const closeAtExit = /* @__PURE__ */ new Set();
|
|
1549
|
+
const retiring = /* @__PURE__ */ new Set();
|
|
1550
|
+
const serverClose = createServerCloseSignal();
|
|
1551
|
+
let stopped = false;
|
|
1552
|
+
function closeThreadsAtExit() {
|
|
1553
|
+
closeEveryThreadAtExit(closeAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS);
|
|
1549
1554
|
}
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
badRequest(ctx.res, malformed, { indexName });
|
|
1557
|
-
return;
|
|
1558
|
-
}
|
|
1559
|
-
const table = await cluster.getAllocation(indexName);
|
|
1560
|
-
if (table === null) {
|
|
1561
|
-
respondJson(ctx, {
|
|
1562
|
-
indexName,
|
|
1563
|
-
allocated: false,
|
|
1564
|
-
version: null,
|
|
1565
|
-
partitions: []
|
|
1566
|
-
});
|
|
1567
|
-
return;
|
|
1568
|
-
}
|
|
1569
|
-
const partitions = [];
|
|
1570
|
-
for (const [partitionId, assignment] of table.assignments) {
|
|
1571
|
-
partitions.push({
|
|
1572
|
-
partitionId,
|
|
1573
|
-
state: assignment.state,
|
|
1574
|
-
primary: assignment.primary,
|
|
1575
|
-
primaryTerm: assignment.primaryTerm,
|
|
1576
|
-
commitPoint: assignment.commitPoint,
|
|
1577
|
-
replicas: [...assignment.replicas],
|
|
1578
|
-
inSyncSet: [...assignment.inSyncSet],
|
|
1579
|
-
lastHolders: [...assignment.lastHolders ?? []],
|
|
1580
|
-
unassignedReason: assignment.unassignedReason ?? null
|
|
1581
|
-
});
|
|
1555
|
+
function releaseDescriptor(thread) {
|
|
1556
|
+
thread.feeds?.closeAll();
|
|
1557
|
+
if (thread.descriptor !== null) {
|
|
1558
|
+
try {
|
|
1559
|
+
acceptor.removeChildAppDescriptor(thread.descriptor);
|
|
1560
|
+
} catch {
|
|
1582
1561
|
}
|
|
1583
|
-
|
|
1584
|
-
respondJson(ctx, {
|
|
1585
|
-
indexName,
|
|
1586
|
-
allocated: partitions.length > 0,
|
|
1587
|
-
version: table.version,
|
|
1588
|
-
partitions
|
|
1589
|
-
});
|
|
1590
|
-
} catch (err) {
|
|
1591
|
-
respondError(ctx, err);
|
|
1562
|
+
thread.descriptor = null;
|
|
1592
1563
|
}
|
|
1564
|
+
confirmDeparture(thread.departure);
|
|
1593
1565
|
}
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1566
|
+
function relayFrom(thread) {
|
|
1567
|
+
const { port } = thread;
|
|
1568
|
+
const streams = /* @__PURE__ */ new Map();
|
|
1569
|
+
const calls = /* @__PURE__ */ new Map();
|
|
1570
|
+
port.on("close", () => {
|
|
1571
|
+
thread.feeds?.closeAll();
|
|
1572
|
+
for (const stream of streams.values()) stream.cancel();
|
|
1573
|
+
for (const call of calls.values()) call.abort();
|
|
1574
|
+
calls.clear();
|
|
1575
|
+
});
|
|
1576
|
+
port.on("message", (message) => {
|
|
1577
|
+
if (isThreadFeedNote(message)) {
|
|
1578
|
+
thread.feeds?.handle(message);
|
|
1601
1579
|
return;
|
|
1602
1580
|
}
|
|
1603
|
-
|
|
1604
|
-
if (
|
|
1605
|
-
|
|
1606
|
-
if (typeof factor !== "number") {
|
|
1607
|
-
badRequest(ctx.res, "The body must carry a numeric replicationFactor", { indexName });
|
|
1581
|
+
if (!isRelayRequest(message)) return;
|
|
1582
|
+
if (message.type === "leaving") {
|
|
1583
|
+
releaseDescriptor(thread);
|
|
1608
1584
|
return;
|
|
1609
1585
|
}
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
} catch (err) {
|
|
1613
|
-
respondError(ctx, err);
|
|
1614
|
-
}
|
|
1615
|
-
}
|
|
1616
|
-
async function health(ctx) {
|
|
1617
|
-
try {
|
|
1618
|
-
respondJson(ctx, await requireCluster(deps).getHealth());
|
|
1619
|
-
} catch (err) {
|
|
1620
|
-
respondError(ctx, err);
|
|
1621
|
-
}
|
|
1622
|
-
}
|
|
1623
|
-
return { topology, allocation, replicationFactor, health };
|
|
1624
|
-
}
|
|
1625
|
-
|
|
1626
|
-
// src/server/handlers/backups.ts
|
|
1627
|
-
function stepsOf(context) {
|
|
1628
|
-
return { onProgress: (progress) => context.reportSteps({ done: progress.done, total: progress.total }) };
|
|
1629
|
-
}
|
|
1630
|
-
function restoreOptionsOf(body) {
|
|
1631
|
-
const { indexName } = body;
|
|
1632
|
-
if (typeof indexName !== "string") return 'The body must carry the new index name as a string in "indexName"';
|
|
1633
|
-
const placement = readRestorePlacement(body);
|
|
1634
|
-
return typeof placement === "string" ? placement : { indexName, ...placement };
|
|
1635
|
-
}
|
|
1636
|
-
function readBody(ctx) {
|
|
1637
|
-
const body = parseJson(ctx);
|
|
1638
|
-
if (body === void 0) return void 0;
|
|
1639
|
-
if (!isRecord(body)) {
|
|
1640
|
-
badRequest(ctx.res, "The body must be a JSON object");
|
|
1641
|
-
return void 0;
|
|
1642
|
-
}
|
|
1643
|
-
return body;
|
|
1644
|
-
}
|
|
1645
|
-
function createBackupHandlers(deps) {
|
|
1646
|
-
async function create(ctx) {
|
|
1647
|
-
try {
|
|
1648
|
-
const cluster = requireCluster(deps);
|
|
1649
|
-
const indexName = ctx.params[0] ?? "";
|
|
1650
|
-
const body = readBody(ctx);
|
|
1651
|
-
if (body === void 0) return;
|
|
1652
|
-
if (typeof body.backup !== "string") {
|
|
1653
|
-
badRequest(ctx.res, 'The body must carry the backup name as a string in "backup"', { indexName });
|
|
1586
|
+
if (message.type === "touch") {
|
|
1587
|
+
core.orchestrator.noteAccess(message.indexName);
|
|
1654
1588
|
return;
|
|
1655
1589
|
}
|
|
1656
|
-
if (
|
|
1657
|
-
|
|
1590
|
+
if (message.type === "stream-credit") {
|
|
1591
|
+
streams.get(message.requestId)?.credit();
|
|
1658
1592
|
return;
|
|
1659
1593
|
}
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
indexName,
|
|
1665
|
-
async (context) => {
|
|
1666
|
-
await cluster.createBackup(indexName, backupName, { ...stepsOf(context), queryHistory });
|
|
1667
|
-
},
|
|
1668
|
-
void 0,
|
|
1669
|
-
backupName
|
|
1670
|
-
);
|
|
1671
|
-
respondJson(ctx, record, 202);
|
|
1672
|
-
} catch (err) {
|
|
1673
|
-
respondError(ctx, err);
|
|
1674
|
-
}
|
|
1675
|
-
}
|
|
1676
|
-
async function list(ctx) {
|
|
1677
|
-
try {
|
|
1678
|
-
const cluster = requireCluster(deps);
|
|
1679
|
-
respondJson(ctx, { backups: await cluster.listBackups() });
|
|
1680
|
-
} catch (err) {
|
|
1681
|
-
respondError(ctx, err);
|
|
1682
|
-
}
|
|
1683
|
-
}
|
|
1684
|
-
async function restore(ctx) {
|
|
1685
|
-
try {
|
|
1686
|
-
const cluster = requireCluster(deps);
|
|
1687
|
-
const backupName = validateBackupName(ctx.params[0]);
|
|
1688
|
-
const body = readBody(ctx);
|
|
1689
|
-
if (body === void 0) return;
|
|
1690
|
-
const options = restoreOptionsOf(body);
|
|
1691
|
-
if (typeof options === "string") {
|
|
1692
|
-
badRequest(ctx.res, options, { backupName });
|
|
1594
|
+
if (message.type === "stream-cancel") {
|
|
1595
|
+
streams.get(message.requestId)?.cancel();
|
|
1596
|
+
calls.get(message.requestId)?.abort();
|
|
1597
|
+
calls.delete(message.requestId);
|
|
1693
1598
|
return;
|
|
1694
1599
|
}
|
|
1695
|
-
const
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1600
|
+
const call = message.type === "embed" || message.type === "rerank" ? new AbortController() : void 0;
|
|
1601
|
+
if (call !== void 0) calls.set(message.requestId, call);
|
|
1602
|
+
const reply = message.type === "request" ? answerRequest(handlers[message.route], message, port, streams) : message.type === "authorize" ? answerAuthorize(authorize, message) : message.type === "feed-admit" ? answerFeedAdmit(thread.feeds, message) : message.type === "embed" ? answerEmbed(core, message, call?.signal) : message.type === "await-writes" ? answerAwaitWrites(core, message) : answerRerank(core, message, call?.signal);
|
|
1603
|
+
void reply.then(
|
|
1604
|
+
(answer) => {
|
|
1605
|
+
calls.delete(message.requestId);
|
|
1606
|
+
if (answer !== null) port.postMessage(answer);
|
|
1700
1607
|
},
|
|
1701
|
-
|
|
1702
|
-
|
|
1608
|
+
() => {
|
|
1609
|
+
calls.delete(message.requestId);
|
|
1610
|
+
}
|
|
1703
1611
|
);
|
|
1704
|
-
|
|
1705
|
-
} catch (err) {
|
|
1706
|
-
respondError(ctx, err);
|
|
1707
|
-
}
|
|
1612
|
+
});
|
|
1708
1613
|
}
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
"backupDelete",
|
|
1715
|
-
"",
|
|
1716
|
-
async (context) => {
|
|
1717
|
-
await cluster.deleteBackup(backupName, stepsOf(context));
|
|
1718
|
-
},
|
|
1719
|
-
void 0,
|
|
1720
|
-
backupName
|
|
1721
|
-
);
|
|
1722
|
-
respondJson(ctx, record, 202);
|
|
1723
|
-
} catch (err) {
|
|
1724
|
-
respondError(ctx, err);
|
|
1725
|
-
}
|
|
1614
|
+
function connectToLongQueries(thread) {
|
|
1615
|
+
if (longQueryThread === null) return;
|
|
1616
|
+
const channel = new MessageChannel();
|
|
1617
|
+
thread.port.postMessage({ type: "long-query-port", port: channel.port1 }, [channel.port1]);
|
|
1618
|
+
longQueryThread.port.postMessage({ type: "long-query-port", port: channel.port2 }, [channel.port2]);
|
|
1726
1619
|
}
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
function createCapabilitiesHandler() {
|
|
1732
|
-
function report(ctx) {
|
|
1733
|
-
respondJson(ctx, { capabilities: [...SERVER_CAPABILITIES] });
|
|
1620
|
+
function dropLongQueryThread() {
|
|
1621
|
+
if (longQueryThread === null) return;
|
|
1622
|
+
longQueryThread.port.close();
|
|
1623
|
+
longQueryThread = null;
|
|
1734
1624
|
}
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
function createDocumentExportHandlers(deps) {
|
|
1747
|
-
const { engine, tasks, limits } = deps;
|
|
1748
|
-
async function exportDocuments(ctx) {
|
|
1749
|
-
try {
|
|
1750
|
-
const lines = await engine.exportDocuments(ctx.params[0]);
|
|
1751
|
-
if (ctx.abort.aborted) {
|
|
1752
|
-
await lines.cancel();
|
|
1753
|
-
return;
|
|
1754
|
-
}
|
|
1755
|
-
await sendStream(ctx.res, ctx.abort, lines, [["Content-Type", DOCUMENT_EXPORT_CONTENT_TYPE]]);
|
|
1756
|
-
} catch (err) {
|
|
1757
|
-
respondPreparingError(ctx, err, ErrorCodes.CHANGES_EXPORT_PREPARING);
|
|
1758
|
-
}
|
|
1625
|
+
function retireThread(workerId, thread, refuseNewRequests = false) {
|
|
1626
|
+
releaseDescriptor(thread);
|
|
1627
|
+
thread.port.unref();
|
|
1628
|
+
retiring.add(thread.closed);
|
|
1629
|
+
void thread.closed.then(() => retiring.delete(thread.closed));
|
|
1630
|
+
void executeUntilAnswered(thread.executor, {
|
|
1631
|
+
type: "stopServing",
|
|
1632
|
+
requestId: `stop-serving-${workerId}`,
|
|
1633
|
+
refuseNewRequests
|
|
1634
|
+
}).catch(() => void 0);
|
|
1635
|
+
return thread.closed;
|
|
1759
1636
|
}
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1637
|
+
const listener = {
|
|
1638
|
+
async onLongQueryWorkerReady(workerId, executor) {
|
|
1639
|
+
if (stopped) return;
|
|
1640
|
+
dropLongQueryThread();
|
|
1641
|
+
const channel = new MessageChannel();
|
|
1642
|
+
const relaying = {
|
|
1643
|
+
feeds: null,
|
|
1644
|
+
descriptor: null,
|
|
1645
|
+
port: channel.port2,
|
|
1646
|
+
executor,
|
|
1647
|
+
departure: createDepartureWord(),
|
|
1648
|
+
closeAtExit: createCloseAtExitWords(),
|
|
1649
|
+
closed: new Promise((resolve) => channel.port2.once("close", resolve))
|
|
1650
|
+
};
|
|
1651
|
+
relayFrom(relaying);
|
|
1652
|
+
try {
|
|
1653
|
+
await executor.execute(
|
|
1654
|
+
{
|
|
1655
|
+
type: "serveLongQueries",
|
|
1656
|
+
requestId: `serve-long-queries-${workerId}`,
|
|
1657
|
+
settings: {
|
|
1658
|
+
port: channel.port1,
|
|
1659
|
+
touchIntervalMs,
|
|
1660
|
+
limits: options.limits,
|
|
1661
|
+
patternWorkCap: options.core.patternWorkCap,
|
|
1662
|
+
maxRerankWindow: options.core.rerankSettings.maxWindow,
|
|
1663
|
+
allowRegex: options.allowRegex
|
|
1664
|
+
}
|
|
1665
|
+
},
|
|
1666
|
+
[channel.port1]
|
|
1667
|
+
);
|
|
1668
|
+
} catch (err) {
|
|
1669
|
+
channel.port2.close();
|
|
1670
|
+
console.warn(`Worker ${workerId} could not start answering long queries:`, err);
|
|
1671
|
+
return;
|
|
1672
|
+
}
|
|
1673
|
+
if (stopped) {
|
|
1674
|
+
channel.port2.close();
|
|
1675
|
+
return;
|
|
1676
|
+
}
|
|
1677
|
+
longQueryThread = { workerId, port: channel.port2, executor };
|
|
1678
|
+
for (const thread of threads.values()) connectToLongQueries(thread);
|
|
1679
|
+
},
|
|
1680
|
+
async onWorkerReady(workerId, executor) {
|
|
1681
|
+
if (stopped) return;
|
|
1682
|
+
const channel = new MessageChannel();
|
|
1683
|
+
const slot = gateSlotOfWorker(workerId);
|
|
1684
|
+
slotUsers.set(slot, (slotUsers.get(slot) ?? 0) + 1);
|
|
1685
|
+
const thread = {
|
|
1686
|
+
feeds: options.feeds?.forThread(channel.port2) ?? null,
|
|
1687
|
+
descriptor: null,
|
|
1688
|
+
port: channel.port2,
|
|
1689
|
+
executor,
|
|
1690
|
+
departure: createDepartureWord(),
|
|
1691
|
+
closeAtExit: createCloseAtExitWords(),
|
|
1692
|
+
closed: threadClosed(channel.port2, () => {
|
|
1693
|
+
const remaining = (slotUsers.get(slot) ?? 1) - 1;
|
|
1694
|
+
if (remaining > 0) {
|
|
1695
|
+
slotUsers.set(slot, remaining);
|
|
1696
|
+
return;
|
|
1697
|
+
}
|
|
1698
|
+
slotUsers.delete(slot);
|
|
1699
|
+
clearGateSlot(options.gate, slot);
|
|
1700
|
+
clearBodyBudgetSlot(options.bodyBudget, slot);
|
|
1701
|
+
})
|
|
1702
|
+
};
|
|
1703
|
+
relayFrom(thread);
|
|
1704
|
+
closeAtExit.add(thread.closeAtExit);
|
|
1705
|
+
void thread.closed.then(() => closeAtExit.delete(thread.closeAtExit));
|
|
1706
|
+
const longQueries = longQueryThread;
|
|
1707
|
+
const longQueryChannel = longQueries === null ? null : new MessageChannel();
|
|
1708
|
+
const transfer = longQueryChannel === null ? [channel.port1] : [channel.port1, longQueryChannel.port1];
|
|
1709
|
+
let result;
|
|
1710
|
+
try {
|
|
1711
|
+
result = await executor.execute(
|
|
1712
|
+
{
|
|
1713
|
+
type: "serveRequests",
|
|
1714
|
+
requestId: `serve-${workerId}`,
|
|
1715
|
+
settings: {
|
|
1716
|
+
port: channel.port1,
|
|
1717
|
+
departure: thread.departure.buffer,
|
|
1718
|
+
serverClose: serverClose.buffer,
|
|
1719
|
+
closeAtExit: thread.closeAtExit.buffer,
|
|
1720
|
+
gate: options.gate,
|
|
1721
|
+
gateSlot: gateSlotOfWorker(workerId),
|
|
1722
|
+
maxConcurrentRequests: options.limits.maxConcurrentRequests,
|
|
1723
|
+
bodyBudget: options.bodyBudget,
|
|
1724
|
+
touchIntervalMs,
|
|
1725
|
+
cors: options.cors,
|
|
1726
|
+
limits: options.limits,
|
|
1727
|
+
build: options.build,
|
|
1728
|
+
authorizes: authorize !== void 0,
|
|
1729
|
+
searchHooks,
|
|
1730
|
+
patternWorkCap: options.core.patternWorkCap,
|
|
1731
|
+
maxRerankWindow: options.core.rerankSettings.maxWindow,
|
|
1732
|
+
allowRegex: options.allowRegex,
|
|
1733
|
+
feeds: Object.keys(options.feeds?.gateways ?? {}),
|
|
1734
|
+
longQueriesInFlight,
|
|
1735
|
+
longQueryPort: longQueryChannel?.port1 ?? null
|
|
1736
|
+
}
|
|
1737
|
+
},
|
|
1738
|
+
transfer
|
|
1739
|
+
);
|
|
1740
|
+
} catch (err) {
|
|
1741
|
+
closeAtExit.delete(thread.closeAtExit);
|
|
1742
|
+
channel.port2.close();
|
|
1743
|
+
longQueryChannel?.port2.close();
|
|
1744
|
+
console.warn(`Worker ${workerId} could not start taking requests:`, err);
|
|
1745
|
+
return;
|
|
1746
|
+
}
|
|
1747
|
+
thread.descriptor = result.descriptor;
|
|
1748
|
+
if (stopped || departureConfirmed(thread.departure)) {
|
|
1749
|
+
longQueryChannel?.port2.close();
|
|
1750
|
+
await retireThread(workerId, thread);
|
|
1751
|
+
return;
|
|
1752
|
+
}
|
|
1753
|
+
if (longQueryChannel !== null && longQueries !== null && longQueryThread === longQueries) {
|
|
1754
|
+
longQueries.port.postMessage({ type: "long-query-port", port: longQueryChannel.port2 }, [
|
|
1755
|
+
longQueryChannel.port2
|
|
1756
|
+
]);
|
|
1757
|
+
} else {
|
|
1758
|
+
longQueryChannel?.port2.close();
|
|
1759
|
+
connectToLongQueries(thread);
|
|
1760
|
+
}
|
|
1761
|
+
acceptor.addChildAppDescriptor(result.descriptor);
|
|
1762
|
+
threads.set(workerId, thread);
|
|
1763
|
+
},
|
|
1764
|
+
onWorkerGone(workerId) {
|
|
1765
|
+
if (longQueryThread?.workerId === workerId) {
|
|
1766
|
+
dropLongQueryThread();
|
|
1767
|
+
return;
|
|
1768
|
+
}
|
|
1769
|
+
const thread = threads.get(workerId);
|
|
1770
|
+
if (thread === void 0) return;
|
|
1771
|
+
threads.delete(workerId);
|
|
1772
|
+
void retireThread(workerId, thread);
|
|
1766
1773
|
}
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1774
|
+
};
|
|
1775
|
+
process.on("exit", closeThreadsAtExit);
|
|
1776
|
+
await core.orchestrator.serveRequestsOnWorkers(listener);
|
|
1777
|
+
return {
|
|
1778
|
+
get threadCount() {
|
|
1779
|
+
return threads.size;
|
|
1780
|
+
},
|
|
1781
|
+
async stop() {
|
|
1782
|
+
const deadline = signalServerClose(serverClose, SERVER_CLOSE_DRAIN_DEADLINE_MS);
|
|
1783
|
+
stopped = true;
|
|
1784
|
+
process.off("exit", closeThreadsAtExit);
|
|
1785
|
+
const stopping = [...threads.entries()].map(([workerId, thread]) => {
|
|
1786
|
+
threads.delete(workerId);
|
|
1787
|
+
return retireThread(workerId, thread, true);
|
|
1776
1788
|
});
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1789
|
+
core.orchestrator.stopRequestThreads();
|
|
1790
|
+
const longQueries = longQueryThread;
|
|
1791
|
+
dropLongQueryThread();
|
|
1792
|
+
if (longQueries !== null) {
|
|
1793
|
+
stopping.push(
|
|
1794
|
+
longQueries.executor.execute({ type: "stopServing", requestId: `stop-long-queries-${longQueries.workerId}` }).then(
|
|
1795
|
+
() => void 0,
|
|
1796
|
+
() => void 0
|
|
1797
|
+
)
|
|
1798
|
+
);
|
|
1799
|
+
}
|
|
1800
|
+
await closeThreadsByDeadline(closeAtExit, Promise.all([...stopping, ...retiring]), deadline);
|
|
1781
1801
|
}
|
|
1802
|
+
};
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1805
|
+
// src/server/sync/options.ts
|
|
1806
|
+
var FEED_OPTION_FIELDS = ["windowBytes", "maxConnections", "maxCatchUps", "maxBufferedBytes"];
|
|
1807
|
+
var FEED_REFUSALS = {
|
|
1808
|
+
sync: "Refusing to start: the device sync routes are on with no onRequest hook. A device receives the whole index, so set options.onRequest to admit only the user whose documents the index holds. allowInsecure does not lift this refusal.",
|
|
1809
|
+
changes: "Refusing to start: the change feed is on with no onRequest hook. A subscriber receives every document of the index, so set options.onRequest to admit only the services that may read the whole index. allowInsecure does not lift this refusal."
|
|
1810
|
+
};
|
|
1811
|
+
function assertCount(feed, field, value) {
|
|
1812
|
+
if (value === void 0) return;
|
|
1813
|
+
if (typeof value === "number" && Number.isSafeInteger(value) && value >= 1) return;
|
|
1814
|
+
throw new NarsilError(
|
|
1815
|
+
ErrorCodes.CONFIG_INVALID,
|
|
1816
|
+
`${feed}.${field} must be a whole number of at least 1; however, this configuration sets it to ${String(value)}`,
|
|
1817
|
+
{ field, value: String(value) }
|
|
1818
|
+
);
|
|
1819
|
+
}
|
|
1820
|
+
function resolveFeedOptions(feed, value) {
|
|
1821
|
+
if (value === void 0 || value === false) return null;
|
|
1822
|
+
if (value !== true && (typeof value !== "object" || value === null)) {
|
|
1823
|
+
throw new NarsilError(
|
|
1824
|
+
ErrorCodes.CONFIG_INVALID,
|
|
1825
|
+
`${feed} must be true, false, or an object of settings; however, this configuration sets it to ${String(value)}`,
|
|
1826
|
+
{ field: feed, value: String(value) }
|
|
1827
|
+
);
|
|
1782
1828
|
}
|
|
1783
|
-
|
|
1829
|
+
const settings = value === true ? {} : value;
|
|
1830
|
+
for (const field of FEED_OPTION_FIELDS) assertCount(feed, field, settings[field]);
|
|
1831
|
+
return {
|
|
1832
|
+
windowBytes: settings.windowBytes ?? DEFAULT_FEED_WINDOW_BYTES,
|
|
1833
|
+
maxConnections: settings.maxConnections ?? DEFAULT_MAX_FEED_CONNECTIONS,
|
|
1834
|
+
maxCatchUps: settings.maxCatchUps ?? DEFAULT_MAX_FEED_CATCH_UPS,
|
|
1835
|
+
maxBufferedBytes: settings.maxBufferedBytes ?? DEFAULT_MAX_FEED_BUFFERED_BYTES
|
|
1836
|
+
};
|
|
1837
|
+
}
|
|
1838
|
+
function assertFeedHasAuthentication(feed, resolved, options) {
|
|
1839
|
+
if (resolved === null || options.onRequest !== void 0) return;
|
|
1840
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, FEED_REFUSALS[feed], { field: feed });
|
|
1784
1841
|
}
|
|
1785
1842
|
|
|
1786
|
-
// src/server/
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
rejectInvalid(ctx, failure);
|
|
1797
|
-
return;
|
|
1798
|
-
}
|
|
1799
|
-
try {
|
|
1800
|
-
const id = await engine.insert(ctx.params[0], body.document, body.id, body.options);
|
|
1801
|
-
respondJson(ctx, { id }, 201);
|
|
1802
|
-
} catch (err) {
|
|
1803
|
-
respondError(ctx, err);
|
|
1804
|
-
}
|
|
1843
|
+
// src/server/task-store.ts
|
|
1844
|
+
var InMemoryTaskStore = class {
|
|
1845
|
+
/**
|
|
1846
|
+
* Builds a store that keeps task records in this process alone.
|
|
1847
|
+
*
|
|
1848
|
+
* @param maxRetained - Records to keep before the oldest finished ones are
|
|
1849
|
+
* dropped. Defaults to 1000.
|
|
1850
|
+
*/
|
|
1851
|
+
constructor(maxRetained = DEFAULT_MAX_RETAINED_TASKS) {
|
|
1852
|
+
this.maxRetained = maxRetained;
|
|
1805
1853
|
}
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1854
|
+
records = /* @__PURE__ */ new Map();
|
|
1855
|
+
/**
|
|
1856
|
+
* Stores a record, replacing whatever its id already held.
|
|
1857
|
+
*
|
|
1858
|
+
* @param record - The record to store. A copy is kept, so changing the
|
|
1859
|
+
* object afterwards leaves the store untouched.
|
|
1860
|
+
*/
|
|
1861
|
+
async set(record) {
|
|
1862
|
+
this.records.set(record.id, { ...record });
|
|
1863
|
+
this.prune();
|
|
1864
|
+
}
|
|
1865
|
+
/**
|
|
1866
|
+
* Reads one record back.
|
|
1867
|
+
*
|
|
1868
|
+
* @param id - The task to read.
|
|
1869
|
+
* @returns A copy of the record, or `null` when the store holds no such id.
|
|
1870
|
+
*/
|
|
1871
|
+
async get(id) {
|
|
1872
|
+
const record = this.records.get(id);
|
|
1873
|
+
return record ? { ...record } : null;
|
|
1874
|
+
}
|
|
1875
|
+
/**
|
|
1876
|
+
* Lists every record the store holds.
|
|
1877
|
+
*
|
|
1878
|
+
* @returns A copy of each record, oldest first.
|
|
1879
|
+
*/
|
|
1880
|
+
async list() {
|
|
1881
|
+
return [...this.records.values()].map((record) => ({ ...record }));
|
|
1882
|
+
}
|
|
1883
|
+
/**
|
|
1884
|
+
* Removes one record. An id the store never held is not an error.
|
|
1885
|
+
*
|
|
1886
|
+
* @param id - The task to remove.
|
|
1887
|
+
*/
|
|
1888
|
+
async delete(id) {
|
|
1889
|
+
this.records.delete(id);
|
|
1890
|
+
}
|
|
1891
|
+
prune() {
|
|
1892
|
+
if (this.records.size <= this.maxRetained) return;
|
|
1893
|
+
for (const [id, record] of this.records) {
|
|
1894
|
+
if (this.records.size <= this.maxRetained) break;
|
|
1895
|
+
if (isTerminalTaskStatus(record.status)) this.records.delete(id);
|
|
1813
1896
|
}
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1897
|
+
}
|
|
1898
|
+
};
|
|
1899
|
+
function byNewestFirst(left, right) {
|
|
1900
|
+
if (left.createdAt !== right.createdAt) return right.createdAt - left.createdAt;
|
|
1901
|
+
return left.id.localeCompare(right.id);
|
|
1902
|
+
}
|
|
1903
|
+
function matchesQuery(record, query) {
|
|
1904
|
+
if (query.indexName !== void 0 && record.indexName !== query.indexName) return false;
|
|
1905
|
+
if (query.type !== void 0 && query.type.length > 0 && !query.type.includes(record.type)) return false;
|
|
1906
|
+
if (query.status !== void 0 && query.status.length > 0 && !query.status.includes(record.status)) return false;
|
|
1907
|
+
return true;
|
|
1908
|
+
}
|
|
1909
|
+
var TaskRegistry = class {
|
|
1910
|
+
constructor(store, instanceId, maxConcurrent = 0) {
|
|
1911
|
+
this.store = store;
|
|
1912
|
+
this.instanceId = instanceId;
|
|
1913
|
+
this.maxConcurrent = maxConcurrent;
|
|
1914
|
+
}
|
|
1915
|
+
live = /* @__PURE__ */ new Map();
|
|
1916
|
+
/** Records a task as running and drives `op` to completion in the background.
|
|
1917
|
+
* Returns the record once it is persisted so the caller can respond 202.
|
|
1918
|
+
* @throws A `NarsilError` with `TOO_MANY_REQUESTS` where this instance
|
|
1919
|
+
* already drives {@link ServerLimits.maxConcurrentTasks} tasks, because each
|
|
1920
|
+
* one holds its own working set until it finishes. */
|
|
1921
|
+
async start(type, indexName, op, progress, backupName) {
|
|
1922
|
+
if (this.maxConcurrent > 0 && this.live.size >= this.maxConcurrent) {
|
|
1923
|
+
throw new NarsilError(
|
|
1924
|
+
ServerErrorCodes.TOO_MANY_REQUESTS,
|
|
1925
|
+
`This instance already drives ${this.maxConcurrent} tasks; poll one to completion before starting another`,
|
|
1926
|
+
{ running: this.live.size, limit: this.maxConcurrent }
|
|
1927
|
+
);
|
|
1928
|
+
}
|
|
1929
|
+
const now = Date.now();
|
|
1930
|
+
const record = {
|
|
1931
|
+
id: randomUUID(),
|
|
1932
|
+
type,
|
|
1933
|
+
indexName,
|
|
1934
|
+
...backupName === void 0 ? {} : { backupName },
|
|
1935
|
+
owner: this.instanceId,
|
|
1936
|
+
status: "running",
|
|
1937
|
+
createdAt: now,
|
|
1938
|
+
startedAt: now,
|
|
1939
|
+
...progress === void 0 ? {} : { progress }
|
|
1940
|
+
};
|
|
1941
|
+
const task = {
|
|
1942
|
+
record,
|
|
1943
|
+
controller: new AbortController(),
|
|
1944
|
+
writes: Promise.resolve(),
|
|
1945
|
+
lastProgressWriteAt: now,
|
|
1946
|
+
settled: false,
|
|
1947
|
+
finished: Promise.resolve()
|
|
1948
|
+
};
|
|
1949
|
+
this.live.set(record.id, task);
|
|
1950
|
+
const accepted = Promise.resolve().then(() => this.store.set(record, RUNNING_TASK_TTL_MS));
|
|
1951
|
+
task.writes = accepted;
|
|
1952
|
+
task.finished = accepted.then(
|
|
1953
|
+
() => this.drive(task, op),
|
|
1954
|
+
async () => {
|
|
1955
|
+
this.live.delete(record.id);
|
|
1956
|
+
await this.store.delete(record.id).catch(() => void 0);
|
|
1822
1957
|
}
|
|
1958
|
+
);
|
|
1959
|
+
try {
|
|
1960
|
+
await accepted;
|
|
1823
1961
|
} catch (err) {
|
|
1824
|
-
|
|
1962
|
+
await task.finished;
|
|
1963
|
+
throw err;
|
|
1825
1964
|
}
|
|
1965
|
+
return record;
|
|
1826
1966
|
}
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1967
|
+
get(id) {
|
|
1968
|
+
return this.store.get(id);
|
|
1969
|
+
}
|
|
1970
|
+
async shutdown() {
|
|
1971
|
+
const tasks = [...this.live.values()];
|
|
1972
|
+
for (const task of tasks) task.controller.abort(new Error("The server is shutting down"));
|
|
1973
|
+
await Promise.all(tasks.map((task) => task.finished));
|
|
1974
|
+
}
|
|
1975
|
+
/** Reads one page of task records, newest first, keeping only what the query
|
|
1976
|
+
* matches. Filtering happens here rather than in the store, so every custom
|
|
1977
|
+
* {@link TaskStore} works unchanged. */
|
|
1978
|
+
async list(query = {}) {
|
|
1979
|
+
const from = Math.max(0, Math.trunc(query.from ?? 0));
|
|
1980
|
+
const limit = Math.max(1, Math.trunc(query.limit ?? DEFAULT_TASK_PAGE_SIZE));
|
|
1981
|
+
const matched = (await this.store.list()).filter((record) => matchesQuery(record, query)).sort(byNewestFirst);
|
|
1982
|
+
const page = matched.slice(from, from + limit);
|
|
1983
|
+
const consumed = from + page.length;
|
|
1984
|
+
return { tasks: page, total: matched.length, from, limit, next: consumed < matched.length ? consumed : null };
|
|
1985
|
+
}
|
|
1986
|
+
/** Asks a running task to stop. The work stops between units, so the task
|
|
1987
|
+
* reaches `cancelled` only once it actually stopped, and a request that
|
|
1988
|
+
* arrives too late leaves it `succeeded`. */
|
|
1989
|
+
async cancel(id) {
|
|
1990
|
+
const stored = await this.store.get(id);
|
|
1991
|
+
if (!stored) return { outcome: "not-found", record: null };
|
|
1992
|
+
if (isTerminalTaskStatus(stored.status)) {
|
|
1993
|
+
return { outcome: "already-finished", record: stored };
|
|
1994
|
+
}
|
|
1995
|
+
const task = this.live.get(id);
|
|
1996
|
+
if (!task) {
|
|
1997
|
+
const outcome = stored.owner === this.instanceId ? "already-finished" : "owned-by-another-instance";
|
|
1998
|
+
return { outcome, record: stored };
|
|
1999
|
+
}
|
|
2000
|
+
task.controller.abort(new Error("The task was cancelled"));
|
|
2001
|
+
const record = { ...task.record, cancelRequestedAt: Date.now() };
|
|
2002
|
+
task.record = record;
|
|
2003
|
+
await this.enqueueWrite(task, record, RUNNING_TASK_TTL_MS);
|
|
2004
|
+
return { outcome: "cancelling", record };
|
|
2005
|
+
}
|
|
2006
|
+
/** Fails this instance's previously-running tasks after a restart. A task can
|
|
2007
|
+
* only be advanced by the process that owns it, so its own running tasks from
|
|
2008
|
+
* a prior life are dead and must not keep showing a stale running status. This
|
|
2009
|
+
* is effective only with a stable instanceId; with the default random id it
|
|
2010
|
+
* finds none of its own prior tasks, which is also correct for the in-memory
|
|
2011
|
+
* default (which starts empty). */
|
|
2012
|
+
async reconcile(cancelled = () => false) {
|
|
2013
|
+
let records;
|
|
2014
|
+
try {
|
|
2015
|
+
records = await this.store.list();
|
|
2016
|
+
} catch {
|
|
1833
2017
|
return;
|
|
1834
2018
|
}
|
|
2019
|
+
for (const record of records) {
|
|
2020
|
+
if (cancelled()) return;
|
|
2021
|
+
if (record.owner !== this.instanceId) continue;
|
|
2022
|
+
if (isTerminalTaskStatus(record.status)) continue;
|
|
2023
|
+
const failed = {
|
|
2024
|
+
...record,
|
|
2025
|
+
status: "failed",
|
|
2026
|
+
completedAt: Date.now(),
|
|
2027
|
+
error: { code: ServerErrorCodes.TASK_INTERRUPTED, message: "The server restarted while this task was running" }
|
|
2028
|
+
};
|
|
2029
|
+
try {
|
|
2030
|
+
await this.store.set(failed, TERMINAL_TASK_TTL_MS);
|
|
2031
|
+
} catch {
|
|
2032
|
+
}
|
|
2033
|
+
}
|
|
2034
|
+
}
|
|
2035
|
+
enqueueWrite(task, record, ttlMs) {
|
|
2036
|
+
task.writes = task.writes.then(
|
|
2037
|
+
() => this.store.set(record, ttlMs),
|
|
2038
|
+
() => this.store.set(record, ttlMs)
|
|
2039
|
+
);
|
|
2040
|
+
return task.writes.catch(() => {
|
|
2041
|
+
});
|
|
2042
|
+
}
|
|
2043
|
+
reportProgress(task, change) {
|
|
2044
|
+
if (task.settled) return;
|
|
2045
|
+
task.record = { ...task.record, ...change };
|
|
2046
|
+
const now = Date.now();
|
|
2047
|
+
if (now - task.lastProgressWriteAt < TASK_PROGRESS_WRITE_INTERVAL_MS) return;
|
|
2048
|
+
task.lastProgressWriteAt = now;
|
|
2049
|
+
void this.enqueueWrite(task, task.record, RUNNING_TASK_TTL_MS);
|
|
2050
|
+
}
|
|
2051
|
+
async drive(task, op) {
|
|
2052
|
+
const context = {
|
|
2053
|
+
signal: task.controller.signal,
|
|
2054
|
+
reportProgress: (progress) => this.reportProgress(task, { progress }),
|
|
2055
|
+
reportSteps: (steps) => this.reportProgress(task, { steps }),
|
|
2056
|
+
reportResult: (result) => {
|
|
2057
|
+
task.record = { ...task.record, result };
|
|
2058
|
+
},
|
|
2059
|
+
reportBuild: (build) => {
|
|
2060
|
+
task.record = { ...task.record, build };
|
|
2061
|
+
}
|
|
2062
|
+
};
|
|
2063
|
+
let settled;
|
|
2064
|
+
try {
|
|
2065
|
+
await op(context);
|
|
2066
|
+
settled = { ...task.record, status: "succeeded" };
|
|
2067
|
+
} catch (err) {
|
|
2068
|
+
settled = task.controller.signal.aborted ? { ...task.record, status: "cancelled" } : {
|
|
2069
|
+
...task.record,
|
|
2070
|
+
status: "failed",
|
|
2071
|
+
error: err instanceof NarsilError ? serializeNarsilError(err) : { code: "INTERNAL_ERROR", message: "The operation failed" }
|
|
2072
|
+
};
|
|
2073
|
+
}
|
|
2074
|
+
task.settled = true;
|
|
2075
|
+
task.record = { ...settled, completedAt: Date.now() };
|
|
2076
|
+
await this.enqueueWrite(task, task.record, TERMINAL_TASK_TTL_MS);
|
|
2077
|
+
this.live.delete(task.record.id);
|
|
2078
|
+
}
|
|
2079
|
+
};
|
|
2080
|
+
|
|
2081
|
+
// src/server/changes/setting-handlers.ts
|
|
2082
|
+
function createChangesSettingHandlers(engine) {
|
|
2083
|
+
async function getChanges(ctx) {
|
|
1835
2084
|
try {
|
|
1836
|
-
await engine.
|
|
1837
|
-
respondJson(ctx, { id: ctx.params[1] });
|
|
2085
|
+
respondJson(ctx, await engine.getChanges(ctx.params[0]));
|
|
1838
2086
|
} catch (err) {
|
|
1839
2087
|
respondError(ctx, err);
|
|
1840
2088
|
}
|
|
1841
2089
|
}
|
|
1842
|
-
async function
|
|
2090
|
+
async function updateChanges(ctx) {
|
|
2091
|
+
const body = parseJson(ctx);
|
|
2092
|
+
if (body === void 0) return;
|
|
1843
2093
|
try {
|
|
1844
|
-
await engine.
|
|
1845
|
-
respondJson(ctx, {
|
|
2094
|
+
await engine.updateChanges(ctx.params[0], body);
|
|
2095
|
+
respondJson(ctx, { ok: true });
|
|
1846
2096
|
} catch (err) {
|
|
1847
2097
|
respondError(ctx, err);
|
|
1848
2098
|
}
|
|
1849
2099
|
}
|
|
1850
|
-
|
|
2100
|
+
return { getChanges, updateChanges };
|
|
2101
|
+
}
|
|
2102
|
+
function snapshotHistoryOf(raw) {
|
|
2103
|
+
if (raw === null || raw === "false") return false;
|
|
2104
|
+
return raw === "true" ? true : null;
|
|
2105
|
+
}
|
|
2106
|
+
function createAdminHandlers(deps) {
|
|
2107
|
+
const { engine, tasks, limits } = deps;
|
|
2108
|
+
async function checkpoint(ctx) {
|
|
1851
2109
|
try {
|
|
1852
|
-
await engine.
|
|
2110
|
+
await engine.checkpoint(ctx.params[0]);
|
|
1853
2111
|
respondJson(ctx, { ok: true });
|
|
1854
2112
|
} catch (err) {
|
|
1855
2113
|
respondError(ctx, err);
|
|
1856
2114
|
}
|
|
1857
2115
|
}
|
|
1858
|
-
async function
|
|
1859
|
-
const
|
|
1860
|
-
if (
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
2116
|
+
async function snapshot(ctx) {
|
|
2117
|
+
const queryHistory = snapshotHistoryOf(ctx.query.get("queryHistory"));
|
|
2118
|
+
if (queryHistory === null) {
|
|
2119
|
+
badRequest(ctx.res, 'Query parameter "queryHistory" accepts only true or false', {
|
|
2120
|
+
value: ctx.query.get("queryHistory")
|
|
2121
|
+
});
|
|
1864
2122
|
return;
|
|
1865
2123
|
}
|
|
1866
|
-
const action = body.action ?? "insert";
|
|
1867
2124
|
try {
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
);
|
|
1873
|
-
} else if (action === "delete") {
|
|
1874
|
-
respondJson(ctx, serializeBatchResult(await engine.removeBatch(ctx.params[0], body.docIds ?? [], body.options)));
|
|
1875
|
-
} else {
|
|
1876
|
-
respondJson(
|
|
1877
|
-
ctx,
|
|
1878
|
-
serializeBatchResult(await engine.insertBatch(ctx.params[0], body.documents ?? [], body.options))
|
|
1879
|
-
);
|
|
2125
|
+
const archive = await engine.snapshot(ctx.params[0], queryHistory ? { queryHistory } : void 0);
|
|
2126
|
+
if (ctx.abort.aborted) {
|
|
2127
|
+
await archive.cancel();
|
|
2128
|
+
return;
|
|
1880
2129
|
}
|
|
2130
|
+
await sendStream(ctx.res, ctx.abort, archive, [["Content-Type", SNAPSHOT_ARCHIVE_CONTENT_TYPE]]);
|
|
1881
2131
|
} catch (err) {
|
|
1882
2132
|
respondError(ctx, err);
|
|
1883
2133
|
}
|
|
1884
2134
|
}
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
respondJson(ctx, { status: "ok" });
|
|
2135
|
+
function vectorMaintenance(ctx) {
|
|
2136
|
+
try {
|
|
2137
|
+
respondJson(ctx, { fields: engine.vectorMaintenanceStatus(ctx.params[0]) });
|
|
2138
|
+
} catch (err) {
|
|
2139
|
+
respondError(ctx, err);
|
|
2140
|
+
}
|
|
1892
2141
|
}
|
|
1893
|
-
function
|
|
1894
|
-
const
|
|
1895
|
-
if (
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
return;
|
|
2142
|
+
async function compact(ctx) {
|
|
2143
|
+
const body = parseJsonOptional(ctx);
|
|
2144
|
+
if (!body) return;
|
|
2145
|
+
try {
|
|
2146
|
+
await engine.compactVectors(ctx.params[0], body.field);
|
|
2147
|
+
respondJson(ctx, { ok: true });
|
|
2148
|
+
} catch (err) {
|
|
2149
|
+
respondError(ctx, err);
|
|
1902
2150
|
}
|
|
1903
|
-
const readiness = cluster.getReadiness();
|
|
1904
|
-
const ready = deps.isReady() && readiness === "SERVING";
|
|
1905
|
-
respondJson(
|
|
1906
|
-
ctx,
|
|
1907
|
-
{
|
|
1908
|
-
status: ready ? "ready" : "unavailable",
|
|
1909
|
-
cluster: describeNode(cluster)
|
|
1910
|
-
},
|
|
1911
|
-
ready ? 200 : 503
|
|
1912
|
-
);
|
|
1913
2151
|
}
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
line: lineNumber
|
|
1924
|
-
});
|
|
1925
|
-
this.name = "NdjsonLineTooLongError";
|
|
2152
|
+
async function partitionConfig(ctx) {
|
|
2153
|
+
const body = parseJsonOptional(ctx);
|
|
2154
|
+
if (!body) return;
|
|
2155
|
+
try {
|
|
2156
|
+
await engine.updatePartitionConfig(ctx.params[0], body);
|
|
2157
|
+
respondJson(ctx, { ok: true });
|
|
2158
|
+
} catch (err) {
|
|
2159
|
+
respondError(ctx, err);
|
|
2160
|
+
}
|
|
1926
2161
|
}
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
while (start < length) {
|
|
1933
|
-
let end = buffer.indexOf(NEWLINE, start);
|
|
1934
|
-
if (end === -1) end = length;
|
|
1935
|
-
if (end - start > maxLineBytes) throw new NdjsonLineTooLongError(lineNumber + 1);
|
|
1936
|
-
let contentEnd = end;
|
|
1937
|
-
if (contentEnd > start && buffer[contentEnd - 1] === CARRIAGE_RETURN) contentEnd--;
|
|
1938
|
-
if (contentEnd > start) {
|
|
1939
|
-
lineNumber++;
|
|
1940
|
-
yield { lineNumber, bytes: buffer.subarray(start, contentEnd), bytesConsumed: Math.min(end + 1, length) };
|
|
2162
|
+
async function readSynonyms(ctx) {
|
|
2163
|
+
try {
|
|
2164
|
+
respondJson(ctx, { synonyms: await engine.getSynonyms(ctx.params[0]) });
|
|
2165
|
+
} catch (err) {
|
|
2166
|
+
respondError(ctx, err);
|
|
1941
2167
|
}
|
|
1942
|
-
start = end + 1;
|
|
1943
2168
|
}
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
function isPlainObject(value) {
|
|
1951
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1952
|
-
}
|
|
1953
|
-
async function runImport(engine, options) {
|
|
1954
|
-
const { indexName, body, maxLineBytes, batchSize, maxErrors, signal, onProgress } = options;
|
|
1955
|
-
const bytesTotal = body.length;
|
|
1956
|
-
const writeBlock = writeBlockLookupOf(engine);
|
|
1957
|
-
const errors = [];
|
|
1958
|
-
let indexed = 0;
|
|
1959
|
-
let failed = 0;
|
|
1960
|
-
let bytesProcessed = 0;
|
|
1961
|
-
let pending = [];
|
|
1962
|
-
const recordFailure = (error) => {
|
|
1963
|
-
failed += 1;
|
|
1964
|
-
if (errors.length < maxErrors) errors.push(error);
|
|
1965
|
-
};
|
|
1966
|
-
const flushDocuments = async (documents) => {
|
|
1967
|
-
const result = await engine.insertBatch(indexName, documents, { skipClone: true });
|
|
1968
|
-
indexed += result.succeeded.length;
|
|
1969
|
-
for (const failure of result.failed) {
|
|
1970
|
-
const serialized = failure.error instanceof NarsilError ? serializeNarsilError(failure.error) : { code: ServerErrorCodes.INTERNAL_ERROR, message: String(failure.error) };
|
|
1971
|
-
recordFailure({
|
|
1972
|
-
docId: failure.docId,
|
|
1973
|
-
code: serialized.code,
|
|
1974
|
-
message: serialized.message,
|
|
1975
|
-
...serialized.details === void 0 ? {} : { details: serialized.details }
|
|
1976
|
-
});
|
|
2169
|
+
async function replaceSynonyms(ctx) {
|
|
2170
|
+
const body = parseJson(ctx);
|
|
2171
|
+
if (body === void 0) return;
|
|
2172
|
+
if (body === null || typeof body !== "object" || Array.isArray(body) || !Object.hasOwn(body, "synonyms")) {
|
|
2173
|
+
if (!ctx.abort.aborted) badRequest(ctx.res, 'Send the synonym set as { "synonyms": [ ...rules ] }');
|
|
2174
|
+
return;
|
|
1977
2175
|
}
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
const documents = pending.map((entry) => entry.document);
|
|
1984
|
-
pending = [];
|
|
1985
|
-
const run = flushDocuments(documents).catch((err) => {
|
|
1986
|
-
flushError ??= err;
|
|
1987
|
-
}).finally(() => inFlight.delete(run));
|
|
1988
|
-
inFlight.add(run);
|
|
1989
|
-
};
|
|
1990
|
-
const settleFlushes = async () => {
|
|
1991
|
-
await Promise.all(inFlight);
|
|
1992
|
-
if (flushError !== void 0) throw flushError;
|
|
1993
|
-
};
|
|
1994
|
-
const report = () => {
|
|
1995
|
-
onProgress?.({ indexed, failed, bytesProcessed, bytesTotal });
|
|
1996
|
-
};
|
|
1997
|
-
const seenIds = /* @__PURE__ */ new Set();
|
|
1998
|
-
const repeated = [];
|
|
1999
|
-
try {
|
|
2000
|
-
for (const line of iterateNdjson(body, maxLineBytes)) {
|
|
2001
|
-
bytesProcessed = line.bytesConsumed;
|
|
2002
|
-
const refusal = parseRefusalOf(line.bytes);
|
|
2003
|
-
if (refusal !== null) {
|
|
2004
|
-
const block = refusal.status === 429 && writeBlock !== void 0 ? await writeBlock(indexName).catch(() => null) : null;
|
|
2005
|
-
recordFailure(
|
|
2006
|
-
block === null ? { line: line.lineNumber, code: refusal.code, message: refusal.message, details: refusal.details } : { line: line.lineNumber, ...serializeNarsilError(block) }
|
|
2007
|
-
);
|
|
2008
|
-
continue;
|
|
2009
|
-
}
|
|
2010
|
-
let parsed;
|
|
2011
|
-
try {
|
|
2012
|
-
parsed = JSON.parse(line.bytes.toString("utf8"));
|
|
2013
|
-
} catch {
|
|
2014
|
-
recordFailure({ line: line.lineNumber, code: ServerErrorCodes.INVALID_JSON, message: "Line is not valid JSON" });
|
|
2015
|
-
continue;
|
|
2016
|
-
}
|
|
2017
|
-
if (!isPlainObject(parsed)) {
|
|
2018
|
-
recordFailure({
|
|
2019
|
-
line: line.lineNumber,
|
|
2020
|
-
code: ServerErrorCodes.INVALID_REQUEST,
|
|
2021
|
-
message: "Line is not a JSON object"
|
|
2022
|
-
});
|
|
2023
|
-
continue;
|
|
2024
|
-
}
|
|
2025
|
-
const docId = providedDocId(parsed);
|
|
2026
|
-
if (docId !== void 0) {
|
|
2027
|
-
if (seenIds.has(docId)) {
|
|
2028
|
-
repeated.push({ document: parsed, line: line.lineNumber });
|
|
2029
|
-
continue;
|
|
2030
|
-
}
|
|
2031
|
-
seenIds.add(docId);
|
|
2032
|
-
}
|
|
2033
|
-
pending.push({ document: parsed, line: line.lineNumber });
|
|
2034
|
-
if (pending.length >= batchSize) {
|
|
2035
|
-
signal?.throwIfAborted();
|
|
2036
|
-
startFlush();
|
|
2037
|
-
if (inFlight.size >= IMPORT_FLUSHES_IN_FLIGHT) await Promise.race(inFlight);
|
|
2038
|
-
if (flushError !== void 0) await settleFlushes();
|
|
2039
|
-
report();
|
|
2040
|
-
await yieldToEventLoop();
|
|
2041
|
-
signal?.throwIfAborted();
|
|
2042
|
-
}
|
|
2176
|
+
try {
|
|
2177
|
+
await engine.updateSynonyms(ctx.params[0], Reflect.get(body, "synonyms"));
|
|
2178
|
+
respondJson(ctx, { ok: true });
|
|
2179
|
+
} catch (err) {
|
|
2180
|
+
respondError(ctx, err);
|
|
2043
2181
|
}
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2182
|
+
}
|
|
2183
|
+
async function updateBm25(ctx) {
|
|
2184
|
+
const body = parseJson(ctx);
|
|
2185
|
+
if (body === void 0) return;
|
|
2186
|
+
try {
|
|
2187
|
+
await engine.updateBM25(ctx.params[0], body);
|
|
2188
|
+
respondJson(ctx, { ok: true });
|
|
2189
|
+
} catch (err) {
|
|
2190
|
+
respondError(ctx, err);
|
|
2049
2191
|
}
|
|
2050
|
-
} catch (err) {
|
|
2051
|
-
await Promise.allSettled(inFlight);
|
|
2052
|
-
report();
|
|
2053
|
-
throw err;
|
|
2054
2192
|
}
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
}
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2193
|
+
async function memory(ctx) {
|
|
2194
|
+
try {
|
|
2195
|
+
const stats = await engine.getMemoryStats();
|
|
2196
|
+
const response = { ...stats, requestThreads: deps.requestThreadCount() };
|
|
2197
|
+
respondJson(ctx, response);
|
|
2198
|
+
} catch (err) {
|
|
2199
|
+
respondError(ctx, err);
|
|
2200
|
+
}
|
|
2201
|
+
}
|
|
2202
|
+
async function requireIndexExists(name) {
|
|
2203
|
+
await indexReadsOf(engine).getStats(name);
|
|
2204
|
+
}
|
|
2205
|
+
async function optimize(ctx) {
|
|
2206
|
+
const body = parseJsonOptional(ctx);
|
|
2207
|
+
if (!body) return;
|
|
2208
|
+
const name = ctx.params[0];
|
|
2209
|
+
try {
|
|
2210
|
+
await requireIndexExists(name);
|
|
2211
|
+
} catch (err) {
|
|
2212
|
+
respondError(ctx, err);
|
|
2065
2213
|
return;
|
|
2066
2214
|
}
|
|
2067
|
-
const
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
batchSize: limits.importBatchSize,
|
|
2073
|
-
maxErrors: limits.maxImportErrors
|
|
2074
|
-
};
|
|
2075
|
-
if (ctx.query.get("async") === "true") {
|
|
2076
|
-
try {
|
|
2077
|
-
await engine.countDocuments(indexName);
|
|
2078
|
-
} catch (err) {
|
|
2079
|
-
respondError(ctx, err);
|
|
2080
|
-
return;
|
|
2081
|
-
}
|
|
2082
|
-
const record = await tasks.start(
|
|
2083
|
-
"import",
|
|
2084
|
-
indexName,
|
|
2085
|
-
async (context) => {
|
|
2086
|
-
const result = await runImport(engine, {
|
|
2087
|
-
...runOptions,
|
|
2088
|
-
signal: context.signal,
|
|
2089
|
-
onProgress: context.reportProgress
|
|
2090
|
-
});
|
|
2091
|
-
context.reportResult(result);
|
|
2092
|
-
},
|
|
2093
|
-
{ indexed: 0, failed: 0, bytesProcessed: 0, bytesTotal: raw.length }
|
|
2094
|
-
);
|
|
2095
|
-
respondJson(ctx, record, 202);
|
|
2096
|
-
return;
|
|
2097
|
-
}
|
|
2098
|
-
const controller = new AbortController();
|
|
2099
|
-
ctx.abort.onAbort(() => controller.abort());
|
|
2215
|
+
const record = await tasks.start("optimizeVectors", name, () => engine.optimizeVectors(name, body.field));
|
|
2216
|
+
respondJson(ctx, record, 202);
|
|
2217
|
+
}
|
|
2218
|
+
async function rebuildAnalysis(ctx) {
|
|
2219
|
+
const name = ctx.params[0];
|
|
2100
2220
|
try {
|
|
2101
|
-
|
|
2221
|
+
await requireIndexExists(name);
|
|
2102
2222
|
} catch (err) {
|
|
2103
|
-
if (ctx.abort.aborted) return;
|
|
2104
2223
|
respondError(ctx, err);
|
|
2224
|
+
return;
|
|
2105
2225
|
}
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
// src/server/config-mapping.ts
|
|
2110
|
-
function mapHttpIndexConfig(http) {
|
|
2111
|
-
if (typeof http !== "object" || http === null) {
|
|
2112
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index config must be an object");
|
|
2113
|
-
}
|
|
2114
|
-
if (typeof http.schema !== "object" || http.schema === null || Array.isArray(http.schema)) {
|
|
2115
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Index config requires a "schema" object');
|
|
2116
|
-
}
|
|
2117
|
-
const config = { schema: http.schema };
|
|
2118
|
-
if (http.language !== void 0) config.language = http.language;
|
|
2119
|
-
if (http.partitions !== void 0) config.partitions = http.partitions;
|
|
2120
|
-
if (http.defaultScoring !== void 0) config.defaultScoring = http.defaultScoring;
|
|
2121
|
-
if (http.bm25 !== void 0) config.bm25 = http.bm25;
|
|
2122
|
-
if (http.trackPositions !== void 0) config.trackPositions = http.trackPositions;
|
|
2123
|
-
if (http.textStorage !== void 0) config.textStorage = http.textStorage;
|
|
2124
|
-
if (http.surfaceForms !== void 0) config.surfaceForms = http.surfaceForms === true;
|
|
2125
|
-
if (http.vectorPromotion !== void 0) config.vectorPromotion = http.vectorPromotion;
|
|
2126
|
-
if (http.strict !== void 0) config.strict = http.strict;
|
|
2127
|
-
if (http.required !== void 0) config.required = http.required;
|
|
2128
|
-
if (http.patternValueLimit !== void 0) config.patternValueLimit = http.patternValueLimit;
|
|
2129
|
-
if (http.synonyms !== void 0) config.synonyms = http.synonyms;
|
|
2130
|
-
if (http.tokenizer !== void 0) {
|
|
2131
|
-
if (typeof http.tokenizer !== "string") {
|
|
2132
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Field "tokenizer" must be a registered name');
|
|
2133
|
-
}
|
|
2134
|
-
config.tokenizer = http.tokenizer;
|
|
2135
|
-
}
|
|
2136
|
-
if (typeof http.stopWords === "string") {
|
|
2137
|
-
config.stopWords = http.stopWords;
|
|
2138
|
-
} else if (http.stopWords !== void 0) {
|
|
2139
|
-
if (!Array.isArray(http.stopWords) || http.stopWords.some((w) => typeof w !== "string")) {
|
|
2140
|
-
throw new NarsilError(
|
|
2141
|
-
ErrorCodes.CONFIG_INVALID,
|
|
2142
|
-
'Field "stopWords" must be a registered name or an array of strings'
|
|
2143
|
-
);
|
|
2144
|
-
}
|
|
2145
|
-
config.stopWords = new Set(http.stopWords);
|
|
2146
|
-
}
|
|
2147
|
-
if (http.embedding !== void 0) {
|
|
2148
|
-
config.embedding = mapEmbedding(http.embedding);
|
|
2149
|
-
}
|
|
2150
|
-
return config;
|
|
2151
|
-
}
|
|
2152
|
-
function mapEmbedding(embedding) {
|
|
2153
|
-
if (typeof embedding !== "object" || embedding === null) {
|
|
2154
|
-
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, 'Field "embedding" must be an object');
|
|
2155
|
-
}
|
|
2156
|
-
if (typeof embedding.fields !== "object" || embedding.fields === null) {
|
|
2157
|
-
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, 'Field "embedding.fields" is required');
|
|
2158
|
-
}
|
|
2159
|
-
const mapped = { fields: embedding.fields };
|
|
2160
|
-
if (embedding.adapter !== void 0) {
|
|
2161
|
-
if (typeof embedding.adapter !== "string" || embedding.adapter.length === 0) {
|
|
2162
|
-
throw new NarsilError(
|
|
2163
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
2164
|
-
'Field "embedding.adapter" must be a non-empty adapter name'
|
|
2165
|
-
);
|
|
2166
|
-
}
|
|
2167
|
-
mapped.adapter = embedding.adapter;
|
|
2226
|
+
const record = await tasks.start("rebuildAnalysis", name, () => engine.rebuildAnalysis(name));
|
|
2227
|
+
respondJson(ctx, record, 202);
|
|
2168
2228
|
}
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
// src/server/handlers/indexes.ts
|
|
2173
|
-
function createIndexHandlers(deps) {
|
|
2174
|
-
const { engine } = deps;
|
|
2175
|
-
async function create(ctx) {
|
|
2176
|
-
const body = parseJson(ctx);
|
|
2229
|
+
async function changeAnalysis(ctx) {
|
|
2230
|
+
const body = parseJsonOptional(ctx);
|
|
2177
2231
|
if (!body) return;
|
|
2178
|
-
const failure =
|
|
2232
|
+
const failure = validateChangeAnalysis(body);
|
|
2179
2233
|
if (failure) {
|
|
2180
2234
|
rejectInvalid(ctx, failure);
|
|
2181
2235
|
return;
|
|
2182
2236
|
}
|
|
2237
|
+
const name = ctx.params[0];
|
|
2183
2238
|
try {
|
|
2184
|
-
|
|
2185
|
-
await engine.createIndex(body.name, config);
|
|
2186
|
-
respondJson(ctx, { name: body.name }, 201);
|
|
2239
|
+
await requireIndexExists(name);
|
|
2187
2240
|
} catch (err) {
|
|
2188
2241
|
respondError(ctx, err);
|
|
2242
|
+
return;
|
|
2189
2243
|
}
|
|
2244
|
+
const record = await tasks.start("changeAnalysis", name, () => engine.changeAnalysis(name, body));
|
|
2245
|
+
respondJson(ctx, record, 202);
|
|
2190
2246
|
}
|
|
2191
|
-
async function
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2247
|
+
async function rebalance(ctx) {
|
|
2248
|
+
const body = parseJsonOptional(ctx);
|
|
2249
|
+
if (!body) return;
|
|
2250
|
+
const failure = validateRebalance(body);
|
|
2251
|
+
if (failure) {
|
|
2252
|
+
rejectInvalid(ctx, failure);
|
|
2253
|
+
return;
|
|
2196
2254
|
}
|
|
2197
|
-
|
|
2198
|
-
|
|
2255
|
+
const target = body.targetPartitionCount;
|
|
2256
|
+
const name = ctx.params[0];
|
|
2199
2257
|
try {
|
|
2200
|
-
await
|
|
2201
|
-
respondJson(ctx, { name: ctx.params[0], dropped: true });
|
|
2258
|
+
await requireIndexExists(name);
|
|
2202
2259
|
} catch (err) {
|
|
2203
2260
|
respondError(ctx, err);
|
|
2261
|
+
return;
|
|
2204
2262
|
}
|
|
2263
|
+
const record = await tasks.start("rebalance", name, () => engine.rebalance(name, target));
|
|
2264
|
+
respondJson(ctx, record, 202);
|
|
2205
2265
|
}
|
|
2206
|
-
async function
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2266
|
+
async function restore(ctx) {
|
|
2267
|
+
const spooled = ctx.spooledBody;
|
|
2268
|
+
if (spooled === null || spooled.bytes === 0) {
|
|
2269
|
+
await spooled?.remove();
|
|
2270
|
+
sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
|
|
2271
|
+
return;
|
|
2211
2272
|
}
|
|
2212
|
-
|
|
2213
|
-
async function partitions(ctx) {
|
|
2273
|
+
const name = ctx.params[0];
|
|
2214
2274
|
try {
|
|
2215
|
-
|
|
2216
|
-
|
|
2275
|
+
const record = await tasks.start("restore", name, async () => {
|
|
2276
|
+
try {
|
|
2277
|
+
await engine.restore(name, createReadStream(spooled.path));
|
|
2278
|
+
} finally {
|
|
2279
|
+
await spooled.remove();
|
|
2280
|
+
}
|
|
2217
2281
|
});
|
|
2282
|
+
respondJson(ctx, record, 202);
|
|
2218
2283
|
} catch (err) {
|
|
2284
|
+
await spooled.remove();
|
|
2219
2285
|
respondError(ctx, err);
|
|
2220
2286
|
}
|
|
2221
2287
|
}
|
|
2222
|
-
async function
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
respondError(ctx, err);
|
|
2288
|
+
async function getTask(ctx) {
|
|
2289
|
+
const record = await tasks.get(ctx.params[0]);
|
|
2290
|
+
if (!record) {
|
|
2291
|
+
sendError(ctx.res, 404, ServerErrorCodes.TASK_NOT_FOUND, `Task "${ctx.params[0]}" not found`);
|
|
2292
|
+
return;
|
|
2228
2293
|
}
|
|
2294
|
+
respondJson(ctx, record);
|
|
2229
2295
|
}
|
|
2230
|
-
async function
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
respondError(ctx, err);
|
|
2296
|
+
async function listTasks(ctx) {
|
|
2297
|
+
const parsed = parseTaskListQuery(ctx.query, limits.maxTaskPageSize);
|
|
2298
|
+
if ("failure" in parsed) {
|
|
2299
|
+
rejectInvalid(ctx, parsed.failure);
|
|
2300
|
+
return;
|
|
2236
2301
|
}
|
|
2302
|
+
respondJson(ctx, await tasks.list(parsed.query));
|
|
2237
2303
|
}
|
|
2238
|
-
async function
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2304
|
+
async function cancelTask(ctx) {
|
|
2305
|
+
const taskId = ctx.params[0];
|
|
2306
|
+
const { outcome, record } = await tasks.cancel(taskId);
|
|
2307
|
+
if (outcome === "not-found") {
|
|
2308
|
+
sendError(ctx.res, 404, ServerErrorCodes.TASK_NOT_FOUND, `Task "${taskId}" not found`);
|
|
2309
|
+
return;
|
|
2310
|
+
}
|
|
2311
|
+
if (outcome === "already-finished") {
|
|
2312
|
+
sendError(
|
|
2313
|
+
ctx.res,
|
|
2314
|
+
409,
|
|
2315
|
+
ServerErrorCodes.TASK_NOT_CANCELLABLE,
|
|
2316
|
+
`Task "${taskId}" has already finished`,
|
|
2317
|
+
record === null ? void 0 : { status: record.status }
|
|
2318
|
+
);
|
|
2319
|
+
return;
|
|
2320
|
+
}
|
|
2321
|
+
if (outcome === "owned-by-another-instance") {
|
|
2322
|
+
sendError(
|
|
2323
|
+
ctx.res,
|
|
2324
|
+
409,
|
|
2325
|
+
ServerErrorCodes.TASK_OWNED_BY_ANOTHER_INSTANCE,
|
|
2326
|
+
`Task "${taskId}" runs on another server instance, which is the only one that can stop it`,
|
|
2327
|
+
record === null ? void 0 : { owner: record.owner }
|
|
2328
|
+
);
|
|
2329
|
+
return;
|
|
2244
2330
|
}
|
|
2331
|
+
respondJson(ctx, record, 202);
|
|
2245
2332
|
}
|
|
2246
|
-
return {
|
|
2333
|
+
return {
|
|
2334
|
+
checkpoint,
|
|
2335
|
+
snapshot,
|
|
2336
|
+
vectorMaintenance,
|
|
2337
|
+
compact,
|
|
2338
|
+
partitionConfig,
|
|
2339
|
+
readSynonyms,
|
|
2340
|
+
replaceSynonyms,
|
|
2341
|
+
updateBm25,
|
|
2342
|
+
memory,
|
|
2343
|
+
optimize,
|
|
2344
|
+
rebalance,
|
|
2345
|
+
rebuildAnalysis,
|
|
2346
|
+
changeAnalysis,
|
|
2347
|
+
restore,
|
|
2348
|
+
getTask,
|
|
2349
|
+
listTasks,
|
|
2350
|
+
cancelTask
|
|
2351
|
+
};
|
|
2247
2352
|
}
|
|
2248
2353
|
|
|
2249
|
-
// src/server/handlers/
|
|
2250
|
-
function
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
}
|
|
2256
|
-
|
|
2257
|
-
return [`# HELP ${name} ${help}`, `# TYPE ${name} ${type}`, ...samples];
|
|
2258
|
-
}
|
|
2259
|
-
function prometheusText(metrics) {
|
|
2260
|
-
const lines = [
|
|
2261
|
-
...family(
|
|
2262
|
-
"narsil_replication_lag",
|
|
2263
|
-
"gauge",
|
|
2264
|
-
"Entries that the replica has yet to acknowledge, as the primary's last sequence number minus the replica's.",
|
|
2265
|
-
metrics.replicationLag.map(
|
|
2266
|
-
(figure) => `narsil_replication_lag${labels([
|
|
2267
|
-
["index", figure.indexName],
|
|
2268
|
-
["partition", String(figure.partitionId)],
|
|
2269
|
-
["replica", figure.replica]
|
|
2270
|
-
])} ${figure.lag}`
|
|
2271
|
-
)
|
|
2272
|
-
),
|
|
2273
|
-
...family(
|
|
2274
|
-
"narsil_commit_point",
|
|
2275
|
-
"gauge",
|
|
2276
|
-
"The highest sequence number that the leader of the partition has acknowledged to a client.",
|
|
2277
|
-
metrics.commitPoints.map(
|
|
2278
|
-
(figure) => `narsil_commit_point${labels([
|
|
2279
|
-
["index", figure.indexName],
|
|
2280
|
-
["partition", String(figure.partitionId)]
|
|
2281
|
-
])} ${figure.commitPoint}`
|
|
2282
|
-
)
|
|
2283
|
-
),
|
|
2284
|
-
...family(
|
|
2285
|
-
"narsil_commit_point_floor",
|
|
2286
|
-
"gauge",
|
|
2287
|
-
"The floor on the commit point that the allocation table stores, raised when the controller admits a replica.",
|
|
2288
|
-
metrics.partitions.map(
|
|
2289
|
-
(figure) => `narsil_commit_point_floor${labels([
|
|
2290
|
-
["index", figure.indexName],
|
|
2291
|
-
["partition", String(figure.partitionId)]
|
|
2292
|
-
])} ${figure.commitPointFloor}`
|
|
2293
|
-
)
|
|
2294
|
-
),
|
|
2295
|
-
...family(
|
|
2296
|
-
"narsil_in_sync_replicas",
|
|
2297
|
-
"gauge",
|
|
2298
|
-
"The replicas besides the primary that are in the in-sync set of the partition.",
|
|
2299
|
-
metrics.partitions.map(
|
|
2300
|
-
(figure) => `narsil_in_sync_replicas${labels([
|
|
2301
|
-
["index", figure.indexName],
|
|
2302
|
-
["partition", String(figure.partitionId)]
|
|
2303
|
-
])} ${figure.inSyncReplicas}`
|
|
2304
|
-
)
|
|
2305
|
-
),
|
|
2306
|
-
...family(
|
|
2307
|
-
"narsil_vector_rebuild_waiting_for_memory",
|
|
2308
|
-
"gauge",
|
|
2309
|
-
"A vector field whose rebuild waits for the node to have memory for a second graph, reported as 1.",
|
|
2310
|
-
metrics.vectorRebuildsWaitingForMemory.map(
|
|
2311
|
-
(figure) => `narsil_vector_rebuild_waiting_for_memory${labels([
|
|
2312
|
-
["index", figure.indexName],
|
|
2313
|
-
["field", figure.fieldName]
|
|
2314
|
-
])} 1`
|
|
2315
|
-
)
|
|
2316
|
-
),
|
|
2317
|
-
...family(
|
|
2318
|
-
"narsil_coordinator_request_seconds",
|
|
2319
|
-
"histogram",
|
|
2320
|
-
"How long the calls from this node to the cluster coordinator took.",
|
|
2321
|
-
metrics.coordinatorRequests.flatMap((figure) => {
|
|
2322
|
-
const operation = ["operation", figure.operation];
|
|
2323
|
-
return [
|
|
2324
|
-
...figure.bucketBoundsSeconds.map(
|
|
2325
|
-
(bound, position) => `narsil_coordinator_request_seconds_bucket${labels([operation, ["le", String(bound)]])} ${figure.bucketCounts[position] ?? 0}`
|
|
2326
|
-
),
|
|
2327
|
-
`narsil_coordinator_request_seconds_bucket${labels([operation, ["le", "+Inf"]])} ${figure.count}`,
|
|
2328
|
-
`narsil_coordinator_request_seconds_sum${labels([operation])} ${figure.sumSeconds}`,
|
|
2329
|
-
`narsil_coordinator_request_seconds_count${labels([operation])} ${figure.count}`
|
|
2330
|
-
];
|
|
2331
|
-
})
|
|
2332
|
-
)
|
|
2333
|
-
];
|
|
2334
|
-
return `${lines.join("\n")}
|
|
2335
|
-
`;
|
|
2336
|
-
}
|
|
2337
|
-
function createMetricsHandler(deps) {
|
|
2338
|
-
return async function metrics(ctx) {
|
|
2339
|
-
try {
|
|
2340
|
-
if (deps.cluster === void 0) {
|
|
2341
|
-
throw new NarsilError(
|
|
2342
|
-
ErrorCodes.CLUSTER_OPERATION_UNSUPPORTED,
|
|
2343
|
-
"This server fronts a single engine, so it reports no cluster metrics",
|
|
2344
|
-
{}
|
|
2345
|
-
);
|
|
2346
|
-
}
|
|
2347
|
-
const text = prometheusText(await deps.cluster.getMetrics());
|
|
2348
|
-
if (ctx.abort.aborted) return;
|
|
2349
|
-
sendText(ctx.res, text, PROMETHEUS_TEXT_CONTENT_TYPE, ctx.abort);
|
|
2350
|
-
} catch (err) {
|
|
2351
|
-
respondError(ctx, err);
|
|
2352
|
-
}
|
|
2353
|
-
};
|
|
2354
|
-
}
|
|
2355
|
-
|
|
2356
|
-
// src/server/handlers/query-suggestions.ts
|
|
2357
|
-
var PREVIEW_LIMIT_PATTERN = /^\d{1,7}$/;
|
|
2358
|
-
function readObject(ctx) {
|
|
2359
|
-
const body = parseJson(ctx);
|
|
2360
|
-
if (body === void 0) return void 0;
|
|
2361
|
-
if (!isRecord(body)) {
|
|
2362
|
-
badRequest(ctx.res, "The body must be a JSON object");
|
|
2363
|
-
return void 0;
|
|
2354
|
+
// src/server/handlers/cluster.ts
|
|
2355
|
+
function requireCluster(deps) {
|
|
2356
|
+
if (deps.cluster === void 0) {
|
|
2357
|
+
throw new NarsilError(
|
|
2358
|
+
ErrorCodes.CLUSTER_OPERATION_UNSUPPORTED,
|
|
2359
|
+
"This server fronts a single engine, so it reports no cluster",
|
|
2360
|
+
{}
|
|
2361
|
+
);
|
|
2364
2362
|
}
|
|
2365
|
-
return
|
|
2366
|
-
}
|
|
2367
|
-
function readQuery(ctx, body) {
|
|
2368
|
-
if (typeof body.query === "string") return body.query;
|
|
2369
|
-
badRequest(ctx.res, 'The body must carry the phrase as a string in "query"');
|
|
2370
|
-
return void 0;
|
|
2371
|
-
}
|
|
2372
|
-
function isStringList(value) {
|
|
2373
|
-
return Array.isArray(value) && value.every((entry) => typeof entry === "string");
|
|
2374
|
-
}
|
|
2375
|
-
function isIdentity(value) {
|
|
2376
|
-
return isRecord(value) && typeof value.person === "string" && typeof value.group === "string";
|
|
2363
|
+
return deps.cluster;
|
|
2377
2364
|
}
|
|
2378
|
-
function
|
|
2379
|
-
|
|
2380
|
-
return PREVIEW_LIMIT_PATTERN.test(raw) ? Number(raw) : null;
|
|
2365
|
+
function describeNode(cluster) {
|
|
2366
|
+
return { ...cluster.getNodeInfo(), readiness: cluster.getReadiness(), isController: cluster.isControllerActive() };
|
|
2381
2367
|
}
|
|
2382
|
-
|
|
2383
|
-
if (ctx.hookContext === null) return void 0;
|
|
2368
|
+
function malformedIndexName(indexName) {
|
|
2384
2369
|
try {
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
return void 0;
|
|
2370
|
+
validateIndexName(indexName);
|
|
2371
|
+
return null;
|
|
2372
|
+
} catch (error) {
|
|
2373
|
+
return error instanceof Error ? error.message : String(error);
|
|
2390
2374
|
}
|
|
2391
2375
|
}
|
|
2392
|
-
function
|
|
2393
|
-
|
|
2394
|
-
ErrorCodes.QUERY_SUGGESTIONS_NOT_ENABLED,
|
|
2395
|
-
"This server has no identify function for popular query suggestions, so it counts no search",
|
|
2396
|
-
{ indexName }
|
|
2397
|
-
);
|
|
2398
|
-
}
|
|
2399
|
-
function createQuerySuggestionHandlers(deps, options) {
|
|
2400
|
-
const { engine, tasks } = deps;
|
|
2401
|
-
const suggestions = () => querySuggestions(engine);
|
|
2402
|
-
async function update(ctx) {
|
|
2403
|
-
const body = parseJson(ctx);
|
|
2404
|
-
if (body === void 0) return;
|
|
2376
|
+
function createClusterHandlers(deps) {
|
|
2377
|
+
async function topology(ctx) {
|
|
2405
2378
|
try {
|
|
2406
|
-
|
|
2407
|
-
|
|
2379
|
+
const cluster = requireCluster(deps);
|
|
2380
|
+
const [nodes, controllerNodeId] = await Promise.all([cluster.listNodes(), cluster.getControllerNodeId()]);
|
|
2381
|
+
respondJson(ctx, {
|
|
2382
|
+
node: describeNode(cluster),
|
|
2383
|
+
controllerNodeId,
|
|
2384
|
+
nodes: nodes.map((node) => ({
|
|
2385
|
+
nodeId: node.nodeId,
|
|
2386
|
+
address: node.address,
|
|
2387
|
+
roles: [...node.roles],
|
|
2388
|
+
memory: node.memory ?? null
|
|
2389
|
+
}))
|
|
2390
|
+
});
|
|
2408
2391
|
} catch (err) {
|
|
2409
2392
|
respondError(ctx, err);
|
|
2410
2393
|
}
|
|
2411
2394
|
}
|
|
2412
|
-
async function
|
|
2395
|
+
async function allocation(ctx) {
|
|
2413
2396
|
try {
|
|
2414
|
-
|
|
2397
|
+
const cluster = requireCluster(deps);
|
|
2398
|
+
const indexName = ctx.params[0] ?? "";
|
|
2399
|
+
const malformed = malformedIndexName(indexName);
|
|
2400
|
+
if (malformed !== null) {
|
|
2401
|
+
badRequest(ctx.res, malformed, { indexName });
|
|
2402
|
+
return;
|
|
2403
|
+
}
|
|
2404
|
+
const table = await cluster.getAllocation(indexName);
|
|
2405
|
+
if (table === null) {
|
|
2406
|
+
respondJson(ctx, {
|
|
2407
|
+
indexName,
|
|
2408
|
+
allocated: false,
|
|
2409
|
+
version: null,
|
|
2410
|
+
partitions: []
|
|
2411
|
+
});
|
|
2412
|
+
return;
|
|
2413
|
+
}
|
|
2414
|
+
const partitions = [];
|
|
2415
|
+
for (const [partitionId, assignment] of table.assignments) {
|
|
2416
|
+
partitions.push({
|
|
2417
|
+
partitionId,
|
|
2418
|
+
state: assignment.state,
|
|
2419
|
+
primary: assignment.primary,
|
|
2420
|
+
primaryTerm: assignment.primaryTerm,
|
|
2421
|
+
commitPoint: assignment.commitPoint,
|
|
2422
|
+
replicas: [...assignment.replicas],
|
|
2423
|
+
inSyncSet: [...assignment.inSyncSet],
|
|
2424
|
+
lastHolders: [...assignment.lastHolders ?? []],
|
|
2425
|
+
unassignedReason: assignment.unassignedReason ?? null
|
|
2426
|
+
});
|
|
2427
|
+
}
|
|
2428
|
+
partitions.sort((left, right) => left.partitionId - right.partitionId);
|
|
2429
|
+
respondJson(ctx, {
|
|
2430
|
+
indexName,
|
|
2431
|
+
allocated: partitions.length > 0,
|
|
2432
|
+
version: table.version,
|
|
2433
|
+
partitions
|
|
2434
|
+
});
|
|
2415
2435
|
} catch (err) {
|
|
2416
2436
|
respondError(ctx, err);
|
|
2417
2437
|
}
|
|
2418
2438
|
}
|
|
2419
|
-
async function
|
|
2420
|
-
const body = readObject(ctx);
|
|
2421
|
-
if (body === void 0) return;
|
|
2422
|
-
const extra = Object.keys(body).filter((key) => key !== "query");
|
|
2423
|
-
if (extra.length > 0) {
|
|
2424
|
-
badRequest(ctx.res, 'The body carries the search in "query" alone, because the server identifies the person', {
|
|
2425
|
-
fields: extra
|
|
2426
|
-
});
|
|
2427
|
-
return;
|
|
2428
|
-
}
|
|
2429
|
-
const query = readQuery(ctx, body);
|
|
2430
|
-
if (query === void 0) return;
|
|
2431
|
-
const indexName = ctx.params[0];
|
|
2439
|
+
async function replicationFactor(ctx) {
|
|
2432
2440
|
try {
|
|
2433
|
-
|
|
2434
|
-
const
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
}
|
|
2441
|
+
const cluster = requireCluster(deps);
|
|
2442
|
+
const indexName = ctx.params[0] ?? "";
|
|
2443
|
+
const malformed = malformedIndexName(indexName);
|
|
2444
|
+
if (malformed !== null) {
|
|
2445
|
+
badRequest(ctx.res, malformed, { indexName });
|
|
2439
2446
|
return;
|
|
2440
2447
|
}
|
|
2441
|
-
|
|
2442
|
-
|
|
2448
|
+
const body = parseJson(ctx);
|
|
2449
|
+
if (body === void 0) return;
|
|
2450
|
+
const factor = isRecord(body) ? body.replicationFactor : void 0;
|
|
2451
|
+
if (typeof factor !== "number") {
|
|
2452
|
+
badRequest(ctx.res, "The body must carry a numeric replicationFactor", { indexName });
|
|
2443
2453
|
return;
|
|
2444
2454
|
}
|
|
2445
|
-
|
|
2455
|
+
await cluster.setReplicationFactor(indexName, factor);
|
|
2456
|
+
respondJson(ctx, { indexName, replicationFactor: factor });
|
|
2446
2457
|
} catch (err) {
|
|
2447
2458
|
respondError(ctx, err);
|
|
2448
2459
|
}
|
|
2449
2460
|
}
|
|
2450
|
-
async function
|
|
2451
|
-
const indexName = ctx.params[0];
|
|
2461
|
+
async function health(ctx) {
|
|
2452
2462
|
try {
|
|
2453
|
-
|
|
2454
|
-
const current = await operations.status(indexName);
|
|
2455
|
-
if (current.setting?.enabled !== true) {
|
|
2456
|
-
throw new NarsilError(
|
|
2457
|
-
ErrorCodes.QUERY_SUGGESTIONS_NOT_ENABLED,
|
|
2458
|
-
`Index "${indexName}" has popular query suggestions off`,
|
|
2459
|
-
{ indexName }
|
|
2460
|
-
);
|
|
2461
|
-
}
|
|
2462
|
-
const task = await tasks.start("querySuggestionsBuild", indexName, async (context) => {
|
|
2463
|
-
const finished = await operations.build(indexName);
|
|
2464
|
-
context.reportBuild(finished);
|
|
2465
|
-
if (finished.status === "failed") {
|
|
2466
|
-
throw new NarsilError(
|
|
2467
|
-
finished.errorCode ?? ErrorCodes.WORKER_ACTION_FAILED,
|
|
2468
|
-
"The build of popular query suggestions failed",
|
|
2469
|
-
{ indexName, buildId: finished.buildId }
|
|
2470
|
-
);
|
|
2471
|
-
}
|
|
2472
|
-
});
|
|
2473
|
-
respondJson(ctx, task, 202);
|
|
2463
|
+
respondJson(ctx, await requireCluster(deps).getHealth());
|
|
2474
2464
|
} catch (err) {
|
|
2475
2465
|
respondError(ctx, err);
|
|
2476
2466
|
}
|
|
2477
2467
|
}
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2468
|
+
return { topology, allocation, replicationFactor, health };
|
|
2469
|
+
}
|
|
2470
|
+
|
|
2471
|
+
// src/server/handlers/backups.ts
|
|
2472
|
+
function stepsOf(context) {
|
|
2473
|
+
return { onProgress: (progress) => context.reportSteps({ done: progress.done, total: progress.total }) };
|
|
2474
|
+
}
|
|
2475
|
+
function restoreOptionsOf(body) {
|
|
2476
|
+
const { indexName } = body;
|
|
2477
|
+
if (typeof indexName !== "string") return 'The body must carry the new index name as a string in "indexName"';
|
|
2478
|
+
const placement = readRestorePlacement(body);
|
|
2479
|
+
return typeof placement === "string" ? placement : { indexName, ...placement };
|
|
2480
|
+
}
|
|
2481
|
+
function readBody(ctx) {
|
|
2482
|
+
const body = parseJson(ctx);
|
|
2483
|
+
if (body === void 0) return void 0;
|
|
2484
|
+
if (!isRecord(body)) {
|
|
2485
|
+
badRequest(ctx.res, "The body must be a JSON object");
|
|
2486
|
+
return void 0;
|
|
2487
|
+
}
|
|
2488
|
+
return body;
|
|
2489
|
+
}
|
|
2490
|
+
function createBackupHandlers(deps) {
|
|
2491
|
+
async function create(ctx) {
|
|
2484
2492
|
try {
|
|
2485
|
-
|
|
2493
|
+
const cluster = requireCluster(deps);
|
|
2494
|
+
const indexName = ctx.params[0] ?? "";
|
|
2495
|
+
const body = readBody(ctx);
|
|
2496
|
+
if (body === void 0) return;
|
|
2497
|
+
if (typeof body.backup !== "string") {
|
|
2498
|
+
badRequest(ctx.res, 'The body must carry the backup name as a string in "backup"', { indexName });
|
|
2499
|
+
return;
|
|
2500
|
+
}
|
|
2501
|
+
if (body.queryHistory !== void 0 && typeof body.queryHistory !== "boolean") {
|
|
2502
|
+
badRequest(ctx.res, 'The body accepts only true or false in "queryHistory"', { indexName });
|
|
2503
|
+
return;
|
|
2504
|
+
}
|
|
2505
|
+
const backupName = validateBackupName(body.backup);
|
|
2506
|
+
const queryHistory = body.queryHistory === true;
|
|
2507
|
+
const record = await deps.tasks.start(
|
|
2508
|
+
"backup",
|
|
2509
|
+
indexName,
|
|
2510
|
+
async (context) => {
|
|
2511
|
+
await cluster.createBackup(indexName, backupName, { ...stepsOf(context), queryHistory });
|
|
2512
|
+
},
|
|
2513
|
+
void 0,
|
|
2514
|
+
backupName
|
|
2515
|
+
);
|
|
2516
|
+
respondJson(ctx, record, 202);
|
|
2486
2517
|
} catch (err) {
|
|
2487
2518
|
respondError(ctx, err);
|
|
2488
2519
|
}
|
|
2489
2520
|
}
|
|
2490
|
-
async function
|
|
2521
|
+
async function list(ctx) {
|
|
2491
2522
|
try {
|
|
2492
|
-
|
|
2523
|
+
const cluster = requireCluster(deps);
|
|
2524
|
+
respondJson(ctx, { backups: await cluster.listBackups() });
|
|
2493
2525
|
} catch (err) {
|
|
2494
2526
|
respondError(ctx, err);
|
|
2495
2527
|
}
|
|
2496
2528
|
}
|
|
2497
|
-
async function
|
|
2498
|
-
const body = readObject(ctx);
|
|
2499
|
-
if (body === void 0) return;
|
|
2500
|
-
if (!isStringList(body.bans)) {
|
|
2501
|
-
badRequest(ctx.res, 'The body must carry the ban list as an array of strings in "bans"');
|
|
2502
|
-
return;
|
|
2503
|
-
}
|
|
2529
|
+
async function restore(ctx) {
|
|
2504
2530
|
try {
|
|
2505
|
-
|
|
2531
|
+
const cluster = requireCluster(deps);
|
|
2532
|
+
const backupName = validateBackupName(ctx.params[0]);
|
|
2533
|
+
const body = readBody(ctx);
|
|
2534
|
+
if (body === void 0) return;
|
|
2535
|
+
const options = restoreOptionsOf(body);
|
|
2536
|
+
if (typeof options === "string") {
|
|
2537
|
+
badRequest(ctx.res, options, { backupName });
|
|
2538
|
+
return;
|
|
2539
|
+
}
|
|
2540
|
+
const record = await deps.tasks.start(
|
|
2541
|
+
"backupRestore",
|
|
2542
|
+
options.indexName,
|
|
2543
|
+
async (context) => {
|
|
2544
|
+
await cluster.restoreBackup(backupName, { ...options, ...stepsOf(context) });
|
|
2545
|
+
},
|
|
2546
|
+
void 0,
|
|
2547
|
+
backupName
|
|
2548
|
+
);
|
|
2549
|
+
respondJson(ctx, record, 202);
|
|
2506
2550
|
} catch (err) {
|
|
2507
2551
|
respondError(ctx, err);
|
|
2508
2552
|
}
|
|
2509
2553
|
}
|
|
2510
2554
|
async function remove(ctx) {
|
|
2511
|
-
const body = readObject(ctx);
|
|
2512
|
-
if (body === void 0) return;
|
|
2513
|
-
const query = readQuery(ctx, body);
|
|
2514
|
-
if (query === void 0) return;
|
|
2515
2555
|
try {
|
|
2516
|
-
const
|
|
2517
|
-
|
|
2556
|
+
const cluster = requireCluster(deps);
|
|
2557
|
+
const backupName = validateBackupName(ctx.params[0]);
|
|
2558
|
+
const record = await deps.tasks.start(
|
|
2559
|
+
"backupDelete",
|
|
2560
|
+
"",
|
|
2561
|
+
async (context) => {
|
|
2562
|
+
await cluster.deleteBackup(backupName, stepsOf(context));
|
|
2563
|
+
},
|
|
2564
|
+
void 0,
|
|
2565
|
+
backupName
|
|
2566
|
+
);
|
|
2567
|
+
respondJson(ctx, record, 202);
|
|
2518
2568
|
} catch (err) {
|
|
2519
2569
|
respondError(ctx, err);
|
|
2520
2570
|
}
|
|
2521
2571
|
}
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2572
|
+
return { create, list, restore, remove };
|
|
2573
|
+
}
|
|
2574
|
+
|
|
2575
|
+
// src/server/handlers/capabilities.ts
|
|
2576
|
+
function createCapabilitiesHandler() {
|
|
2577
|
+
function report(ctx) {
|
|
2578
|
+
respondJson(ctx, { capabilities: [...SERVER_CAPABILITIES] });
|
|
2579
|
+
}
|
|
2580
|
+
return { report };
|
|
2581
|
+
}
|
|
2582
|
+
function importResultOf(result, maxErrors) {
|
|
2583
|
+
const errors = result.refused.slice(0, maxErrors);
|
|
2584
|
+
return {
|
|
2585
|
+
indexed: result.imported,
|
|
2586
|
+
failed: result.refusedCount,
|
|
2587
|
+
errors,
|
|
2588
|
+
errorsTruncated: result.refusedCount > errors.length
|
|
2589
|
+
};
|
|
2590
|
+
}
|
|
2591
|
+
function createDocumentExportHandlers(deps) {
|
|
2592
|
+
const { engine, tasks, limits } = deps;
|
|
2593
|
+
async function exportDocuments(ctx) {
|
|
2594
|
+
try {
|
|
2595
|
+
const lines = await engine.exportDocuments(ctx.params[0]);
|
|
2596
|
+
if (ctx.abort.aborted) {
|
|
2597
|
+
await lines.cancel();
|
|
2598
|
+
return;
|
|
2599
|
+
}
|
|
2600
|
+
await sendStream(ctx.res, ctx.abort, lines, [["Content-Type", DOCUMENT_EXPORT_CONTENT_TYPE]]);
|
|
2601
|
+
} catch (err) {
|
|
2602
|
+
respondPreparingError(ctx, err, ErrorCodes.CHANGES_EXPORT_PREPARING);
|
|
2603
|
+
}
|
|
2604
|
+
}
|
|
2605
|
+
async function importExport(ctx) {
|
|
2606
|
+
const spooled = ctx.spooledBody;
|
|
2607
|
+
if (spooled === null || spooled.bytes === 0) {
|
|
2608
|
+
await spooled?.remove();
|
|
2609
|
+
sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
|
|
2530
2610
|
return;
|
|
2531
2611
|
}
|
|
2612
|
+
const name = ctx.params[0];
|
|
2532
2613
|
try {
|
|
2533
|
-
const
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2614
|
+
const record = await tasks.start("importExport", name, async (context) => {
|
|
2615
|
+
try {
|
|
2616
|
+
const result = await engine.importDocumentExport(name, createReadStream(spooled.path));
|
|
2617
|
+
context.reportResult(importResultOf(result, limits.maxImportErrors));
|
|
2618
|
+
} finally {
|
|
2619
|
+
await spooled.remove();
|
|
2620
|
+
}
|
|
2537
2621
|
});
|
|
2538
|
-
respondJson(ctx,
|
|
2622
|
+
respondJson(ctx, record, 202);
|
|
2539
2623
|
} catch (err) {
|
|
2624
|
+
await spooled.remove();
|
|
2540
2625
|
respondError(ctx, err);
|
|
2541
2626
|
}
|
|
2542
2627
|
}
|
|
2543
|
-
return {
|
|
2628
|
+
return { exportDocuments, importExport };
|
|
2544
2629
|
}
|
|
2545
2630
|
|
|
2546
|
-
// src/server/handlers/
|
|
2547
|
-
function
|
|
2548
|
-
const { engine
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
const
|
|
2631
|
+
// src/server/handlers/documents.ts
|
|
2632
|
+
function createDocumentHandlers(deps) {
|
|
2633
|
+
const { engine } = deps;
|
|
2634
|
+
const writeBlock = writeBlockLookupOf(engine);
|
|
2635
|
+
const reads = createDocumentReadHandlers(deps);
|
|
2636
|
+
async function insert(ctx) {
|
|
2637
|
+
const body = await parseWriteJsonObject(ctx, writeBlock);
|
|
2638
|
+
if (!body) return;
|
|
2639
|
+
const failure = validateDocumentBody(body);
|
|
2553
2640
|
if (failure) {
|
|
2554
2641
|
rejectInvalid(ctx, failure);
|
|
2555
2642
|
return;
|
|
2556
2643
|
}
|
|
2557
|
-
if (refuseRegex(ctx, params.filters, allowRegex)) return;
|
|
2558
2644
|
try {
|
|
2559
|
-
const
|
|
2560
|
-
|
|
2561
|
-
await respondAnswer(ctx, encodeQueryResult(result), "hits", `a search on index '${ctx.params[0]}'`);
|
|
2645
|
+
const id = await engine.insert(ctx.params[0], body.document, body.id, body.options);
|
|
2646
|
+
respondJson(ctx, { id }, 201);
|
|
2562
2647
|
} catch (err) {
|
|
2563
2648
|
respondError(ctx, err);
|
|
2564
2649
|
}
|
|
2565
2650
|
}
|
|
2566
|
-
async function
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
} finally {
|
|
2571
|
-
clearQuerySignal(params);
|
|
2572
|
-
}
|
|
2573
|
-
}
|
|
2574
|
-
async function preflight(ctx) {
|
|
2575
|
-
const params = parseJson(ctx, "read");
|
|
2576
|
-
if (!params) return;
|
|
2577
|
-
const failure = validateQuery(params, limits);
|
|
2651
|
+
async function put(ctx) {
|
|
2652
|
+
const body = await parseWriteJsonObject(ctx, writeBlock);
|
|
2653
|
+
if (!body) return;
|
|
2654
|
+
const failure = validateDocumentBody(body);
|
|
2578
2655
|
if (failure) {
|
|
2579
2656
|
rejectInvalid(ctx, failure);
|
|
2580
2657
|
return;
|
|
2581
2658
|
}
|
|
2582
|
-
|
|
2659
|
+
const [name, id] = ctx.params;
|
|
2583
2660
|
try {
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
respondJson(ctx,
|
|
2587
|
-
}
|
|
2588
|
-
|
|
2661
|
+
if (await engine.has(name, id)) {
|
|
2662
|
+
await engine.update(name, id, body.document, body.options);
|
|
2663
|
+
respondJson(ctx, { id, created: false });
|
|
2664
|
+
} else {
|
|
2665
|
+
const created = await engine.insert(name, body.document, id, body.options);
|
|
2666
|
+
respondJson(ctx, { id: created, created: true }, 201);
|
|
2589
2667
|
}
|
|
2590
2668
|
} catch (err) {
|
|
2591
2669
|
respondError(ctx, err);
|
|
2592
2670
|
}
|
|
2593
2671
|
}
|
|
2594
|
-
async function
|
|
2595
|
-
const
|
|
2596
|
-
if (!
|
|
2597
|
-
const failure =
|
|
2672
|
+
async function patch(ctx) {
|
|
2673
|
+
const body = await parseWriteJsonObject(ctx, writeBlock);
|
|
2674
|
+
if (!body) return;
|
|
2675
|
+
const failure = validateDocumentBody(body);
|
|
2598
2676
|
if (failure) {
|
|
2599
2677
|
rejectInvalid(ctx, failure);
|
|
2600
2678
|
return;
|
|
2601
2679
|
}
|
|
2602
2680
|
try {
|
|
2603
|
-
|
|
2681
|
+
await engine.update(ctx.params[0], ctx.params[1], body.document, body.options);
|
|
2682
|
+
respondJson(ctx, { id: ctx.params[1] });
|
|
2604
2683
|
} catch (err) {
|
|
2605
2684
|
respondError(ctx, err);
|
|
2606
2685
|
}
|
|
2607
2686
|
}
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
"maxImportBytes",
|
|
2616
|
-
"maxRestoreBytes",
|
|
2617
|
-
"maxLineBytes",
|
|
2618
|
-
"maxImportErrors",
|
|
2619
|
-
"maxConcurrentRequests",
|
|
2620
|
-
"maxConcurrentTasks"
|
|
2621
|
-
];
|
|
2622
|
-
var COUNTED_LIMITS = [
|
|
2623
|
-
"importBatchSize",
|
|
2624
|
-
"maxResultWindow",
|
|
2625
|
-
"maxRerankWindow",
|
|
2626
|
-
"maxFetchDocuments",
|
|
2627
|
-
"maxTaskPageSize"
|
|
2628
|
-
];
|
|
2629
|
-
function assertServerEngine(engine) {
|
|
2630
|
-
const candidate = engine;
|
|
2631
|
-
if (typeof candidate !== "object" || candidate === null) {
|
|
2632
|
-
throw new NarsilError(
|
|
2633
|
-
ErrorCodes.CONFIG_INVALID,
|
|
2634
|
-
"The first argument to createServer must be the engine that createNarsil returns, with the options second",
|
|
2635
|
-
{ received: candidate === null ? "null" : typeof candidate }
|
|
2636
|
-
);
|
|
2687
|
+
async function remove(ctx) {
|
|
2688
|
+
try {
|
|
2689
|
+
await engine.remove(ctx.params[0], ctx.params[1], { wait: ctx.query.get("wait") === "true" });
|
|
2690
|
+
respondJson(ctx, { id: ctx.params[1], removed: true });
|
|
2691
|
+
} catch (err) {
|
|
2692
|
+
respondError(ctx, err);
|
|
2693
|
+
}
|
|
2637
2694
|
}
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
);
|
|
2646
|
-
}
|
|
2647
|
-
function assertLimit(field, value, minimum) {
|
|
2648
|
-
if (value === void 0) return;
|
|
2649
|
-
if (Number.isInteger(value) && value >= minimum) return;
|
|
2650
|
-
throw new NarsilError(
|
|
2651
|
-
ErrorCodes.CONFIG_INVALID,
|
|
2652
|
-
`limits.${field} must be a whole number of at least ${minimum}; however, this configuration sets it to ${String(value)}`,
|
|
2653
|
-
{ field, value, minimum }
|
|
2654
|
-
);
|
|
2655
|
-
}
|
|
2656
|
-
function assertFraction(field, value) {
|
|
2657
|
-
if (value === void 0) return;
|
|
2658
|
-
if (Number.isFinite(value) && value >= 0 && value <= MAX_BODY_BYTES_IN_FLIGHT_FRACTION) return;
|
|
2659
|
-
throw new NarsilError(
|
|
2660
|
-
ErrorCodes.CONFIG_INVALID,
|
|
2661
|
-
`limits.${field} must be a fraction from 0 to ${MAX_BODY_BYTES_IN_FLIGHT_FRACTION}; however, this configuration sets it to ${String(value)}`,
|
|
2662
|
-
{ field, value, maximum: MAX_BODY_BYTES_IN_FLIGHT_FRACTION }
|
|
2663
|
-
);
|
|
2664
|
-
}
|
|
2665
|
-
function assertRerankWindowCap(value, engineCap) {
|
|
2666
|
-
if (value === void 0) return;
|
|
2667
|
-
const cap = Math.min(MAX_RERANK_WINDOW, engineCap ?? MAX_RERANK_WINDOW);
|
|
2668
|
-
if (value <= cap) return;
|
|
2669
|
-
throw new NarsilError(
|
|
2670
|
-
ErrorCodes.CONFIG_INVALID,
|
|
2671
|
-
`limits.maxRerankWindow must be at most ${cap}, the largest window that the engine accepts; however, this configuration sets it to ${value}`,
|
|
2672
|
-
{ field: "maxRerankWindow", value, maximum: cap }
|
|
2673
|
-
);
|
|
2674
|
-
}
|
|
2675
|
-
function assertServerLimits(limits, engineRerankWindow) {
|
|
2676
|
-
if (limits === void 0) return;
|
|
2677
|
-
for (const field of LIMITS_UNBOUNDED_AT_ZERO) assertLimit(field, limits[field], MIN_UNBOUNDED_SERVER_LIMIT);
|
|
2678
|
-
for (const field of COUNTED_LIMITS) assertLimit(field, limits[field], MIN_COUNTED_SERVER_LIMIT);
|
|
2679
|
-
assertFraction("maxBodyBytesInFlightFraction", limits.maxBodyBytesInFlightFraction);
|
|
2680
|
-
assertRerankWindowCap(limits.maxRerankWindow, engineRerankWindow);
|
|
2681
|
-
}
|
|
2682
|
-
function assertQuerySuggestionsOptions(options) {
|
|
2683
|
-
if (options === void 0) return;
|
|
2684
|
-
const identify = typeof options === "object" && options !== null ? Reflect.get(options, "identify") : void 0;
|
|
2685
|
-
if (typeof identify === "function") return;
|
|
2686
|
-
throw new NarsilError(
|
|
2687
|
-
ErrorCodes.CONFIG_INVALID,
|
|
2688
|
-
"querySuggestions must carry an identify function that returns the person and the group of a request",
|
|
2689
|
-
{ field: "querySuggestions.identify" }
|
|
2690
|
-
);
|
|
2691
|
-
}
|
|
2692
|
-
function assertAllowRegex(allowRegex) {
|
|
2693
|
-
if (allowRegex === void 0 || typeof allowRegex === "boolean") return;
|
|
2694
|
-
throw new NarsilError(
|
|
2695
|
-
ErrorCodes.CONFIG_INVALID,
|
|
2696
|
-
`allowRegex must be true or false; however, this configuration sets it to ${String(allowRegex)}`,
|
|
2697
|
-
{ field: "allowRegex", value: String(allowRegex) }
|
|
2698
|
-
);
|
|
2699
|
-
}
|
|
2700
|
-
|
|
2701
|
-
// package.json
|
|
2702
|
-
var version = "0.4.1-next.215";
|
|
2703
|
-
|
|
2704
|
-
// src/server/package-version.ts
|
|
2705
|
-
function publishedPackageVersion() {
|
|
2706
|
-
return version;
|
|
2707
|
-
}
|
|
2708
|
-
|
|
2709
|
-
// src/server/request-threads/relay-answers.ts
|
|
2710
|
-
async function answerAuthorize(authorize, request) {
|
|
2711
|
-
if (authorize === void 0) {
|
|
2712
|
-
return { type: "authorized", requestId: request.requestId, denial: null, hookError: false, expiresAt: null };
|
|
2695
|
+
async function waitForWrites(ctx) {
|
|
2696
|
+
try {
|
|
2697
|
+
await engine.waitForWrites(ctx.params[0]);
|
|
2698
|
+
respondJson(ctx, { ok: true });
|
|
2699
|
+
} catch (err) {
|
|
2700
|
+
respondError(ctx, err);
|
|
2701
|
+
}
|
|
2713
2702
|
}
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
return
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2703
|
+
async function batch(ctx) {
|
|
2704
|
+
const body = parseJsonObject(ctx, ctx.rawBody && isRemoveOnlyBatch(ctx.rawBody) ? "read" : "write");
|
|
2705
|
+
if (!body) return;
|
|
2706
|
+
const failure = validateBatch(body);
|
|
2707
|
+
if (failure) {
|
|
2708
|
+
rejectInvalid(ctx, failure);
|
|
2709
|
+
return;
|
|
2710
|
+
}
|
|
2711
|
+
const action = body.action ?? "insert";
|
|
2712
|
+
try {
|
|
2713
|
+
if (action === "update") {
|
|
2714
|
+
respondJson(
|
|
2715
|
+
ctx,
|
|
2716
|
+
serializeBatchResult(await engine.updateBatch(ctx.params[0], body.updates ?? [], body.options))
|
|
2717
|
+
);
|
|
2718
|
+
} else if (action === "delete") {
|
|
2719
|
+
respondJson(ctx, serializeBatchResult(await engine.removeBatch(ctx.params[0], body.docIds ?? [], body.options)));
|
|
2720
|
+
} else {
|
|
2721
|
+
respondJson(
|
|
2722
|
+
ctx,
|
|
2723
|
+
serializeBatchResult(await engine.insertBatch(ctx.params[0], body.documents ?? [], body.options))
|
|
2724
|
+
);
|
|
2725
|
+
}
|
|
2726
|
+
} catch (err) {
|
|
2727
|
+
respondError(ctx, err);
|
|
2728
|
+
}
|
|
2723
2729
|
}
|
|
2724
|
-
return {
|
|
2725
|
-
type: "authorized",
|
|
2726
|
-
requestId: request.requestId,
|
|
2727
|
-
denial: authorization.denial,
|
|
2728
|
-
hookError: authorization.denial === null,
|
|
2729
|
-
expiresAt: null
|
|
2730
|
-
};
|
|
2730
|
+
return { ...reads, insert, put, patch, remove, batch, waitForWrites };
|
|
2731
2731
|
}
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
entry.embeddingAdapter,
|
|
2738
|
-
signal === void 0 ? core.abortController.signal : AbortSignal.any([core.abortController.signal, signal]),
|
|
2739
|
-
entry.embeddingAdapterName
|
|
2740
|
-
);
|
|
2741
|
-
const value = resolved.vector?.value;
|
|
2742
|
-
return {
|
|
2743
|
-
type: "embedded",
|
|
2744
|
-
requestId: request.requestId,
|
|
2745
|
-
value: value === void 0 ? null : Array.from(value),
|
|
2746
|
-
error: null
|
|
2747
|
-
};
|
|
2748
|
-
} catch (err) {
|
|
2749
|
-
if (!isNarsilError(err)) console.warn("Embedding a query for a request thread failed:", err);
|
|
2750
|
-
return { type: "embedded", requestId: request.requestId, value: null, error: toHttpError(err).body };
|
|
2732
|
+
|
|
2733
|
+
// src/server/handlers/health.ts
|
|
2734
|
+
function createHealthHandlers(deps) {
|
|
2735
|
+
function livez(ctx) {
|
|
2736
|
+
respondJson(ctx, { status: "ok" });
|
|
2751
2737
|
}
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
|
|
2738
|
+
function readyz(ctx) {
|
|
2739
|
+
const cluster = deps.cluster;
|
|
2740
|
+
if (cluster === void 0) {
|
|
2741
|
+
if (deps.isReady()) {
|
|
2742
|
+
respondJson(ctx, { status: "ready" });
|
|
2743
|
+
} else {
|
|
2744
|
+
respondJson(ctx, { status: "unavailable" }, 503);
|
|
2745
|
+
}
|
|
2746
|
+
return;
|
|
2747
|
+
}
|
|
2748
|
+
const readiness = cluster.getReadiness();
|
|
2749
|
+
const ready = deps.isReady() && readiness === "SERVING";
|
|
2750
|
+
respondJson(
|
|
2751
|
+
ctx,
|
|
2752
|
+
{
|
|
2753
|
+
status: ready ? "ready" : "unavailable",
|
|
2754
|
+
cluster: describeNode(cluster)
|
|
2755
|
+
},
|
|
2756
|
+
ready ? 200 : 503
|
|
2757
|
+
);
|
|
2761
2758
|
}
|
|
2762
|
-
}
|
|
2763
|
-
async function answerAwaitWrites(core, request) {
|
|
2764
|
-
await core.orchestrator.awaitWrites(request.indexName).catch(() => void 0);
|
|
2765
|
-
return { type: "writes-applied", requestId: request.requestId };
|
|
2766
|
-
}
|
|
2767
|
-
async function answerFeedAdmit(feeds, request) {
|
|
2768
|
-
if (feeds !== null) return feeds.admit(request);
|
|
2769
|
-
return {
|
|
2770
|
-
type: "feed-admitted",
|
|
2771
|
-
requestId: request.requestId,
|
|
2772
|
-
feedId: null,
|
|
2773
|
-
error: { code: ServerErrorCodes.NOT_FOUND, message: "Route not found" }
|
|
2774
|
-
};
|
|
2759
|
+
return { livez, readyz };
|
|
2775
2760
|
}
|
|
2776
2761
|
|
|
2777
|
-
// src/server/
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
2762
|
+
// src/server/ndjson.ts
|
|
2763
|
+
var NEWLINE = 10;
|
|
2764
|
+
var CARRIAGE_RETURN = 13;
|
|
2765
|
+
var NdjsonLineTooLongError = class extends NarsilError {
|
|
2766
|
+
constructor(lineNumber) {
|
|
2767
|
+
super(ServerErrorCodes.PAYLOAD_TOO_LARGE, `NDJSON line ${lineNumber} exceeds the configured per-line size limit`, {
|
|
2768
|
+
line: lineNumber
|
|
2783
2769
|
});
|
|
2784
|
-
|
|
2785
|
-
}
|
|
2770
|
+
this.name = "NdjsonLineTooLongError";
|
|
2771
|
+
}
|
|
2772
|
+
};
|
|
2773
|
+
function* iterateNdjson(buffer, maxLineBytes) {
|
|
2774
|
+
let start = 0;
|
|
2775
|
+
let lineNumber = 0;
|
|
2776
|
+
const length = buffer.length;
|
|
2777
|
+
while (start < length) {
|
|
2778
|
+
let end = buffer.indexOf(NEWLINE, start);
|
|
2779
|
+
if (end === -1) end = length;
|
|
2780
|
+
if (end - start > maxLineBytes) throw new NdjsonLineTooLongError(lineNumber + 1);
|
|
2781
|
+
let contentEnd = end;
|
|
2782
|
+
if (contentEnd > start && buffer[contentEnd - 1] === CARRIAGE_RETURN) contentEnd--;
|
|
2783
|
+
if (contentEnd > start) {
|
|
2784
|
+
lineNumber++;
|
|
2785
|
+
yield { lineNumber, bytes: buffer.subarray(start, contentEnd), bytesConsumed: Math.min(end + 1, length) };
|
|
2786
|
+
}
|
|
2787
|
+
start = end + 1;
|
|
2788
|
+
}
|
|
2786
2789
|
}
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
"rerank",
|
|
2792
|
-
"await-writes",
|
|
2793
|
-
"stream-credit",
|
|
2794
|
-
"stream-cancel",
|
|
2795
|
-
"feed-admit"
|
|
2796
|
-
]);
|
|
2797
|
-
function isRelayRequest(message) {
|
|
2798
|
-
if (typeof message !== "object" || message === null) return false;
|
|
2799
|
-
const { type, requestId } = message;
|
|
2800
|
-
if (type === "touch" || type === "leaving") return true;
|
|
2801
|
-
return typeof requestId === "number" && typeof type === "string" && RELAYED_TYPES.has(type);
|
|
2790
|
+
|
|
2791
|
+
// src/server/handlers/import.ts
|
|
2792
|
+
function yieldToEventLoop() {
|
|
2793
|
+
return new Promise((resolve) => setImmediate(resolve));
|
|
2802
2794
|
}
|
|
2803
|
-
function
|
|
2804
|
-
return
|
|
2795
|
+
function isPlainObject(value) {
|
|
2796
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2805
2797
|
}
|
|
2806
|
-
async function
|
|
2807
|
-
const
|
|
2808
|
-
|
|
2809
|
-
const
|
|
2810
|
-
const
|
|
2811
|
-
|
|
2812
|
-
|
|
2813
|
-
|
|
2814
|
-
|
|
2815
|
-
const
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
}
|
|
2830
|
-
}
|
|
2831
|
-
thread.descriptor = null;
|
|
2798
|
+
async function runImport(engine, options) {
|
|
2799
|
+
const { indexName, body, maxLineBytes, batchSize, maxErrors, signal, onProgress } = options;
|
|
2800
|
+
const bytesTotal = body.length;
|
|
2801
|
+
const writeBlock = writeBlockLookupOf(engine);
|
|
2802
|
+
const errors = [];
|
|
2803
|
+
let indexed = 0;
|
|
2804
|
+
let failed = 0;
|
|
2805
|
+
let bytesProcessed = 0;
|
|
2806
|
+
let pending = [];
|
|
2807
|
+
const recordFailure = (error) => {
|
|
2808
|
+
failed += 1;
|
|
2809
|
+
if (errors.length < maxErrors) errors.push(error);
|
|
2810
|
+
};
|
|
2811
|
+
const flushDocuments = async (documents) => {
|
|
2812
|
+
const result = await engine.insertBatch(indexName, documents, { skipClone: true });
|
|
2813
|
+
indexed += result.succeeded.length;
|
|
2814
|
+
for (const failure of result.failed) {
|
|
2815
|
+
const serialized = failure.error instanceof NarsilError ? serializeNarsilError(failure.error) : { code: ServerErrorCodes.INTERNAL_ERROR, message: String(failure.error) };
|
|
2816
|
+
recordFailure({
|
|
2817
|
+
docId: failure.docId,
|
|
2818
|
+
code: serialized.code,
|
|
2819
|
+
message: serialized.message,
|
|
2820
|
+
...serialized.details === void 0 ? {} : { details: serialized.details }
|
|
2821
|
+
});
|
|
2832
2822
|
}
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
const
|
|
2839
|
-
|
|
2840
|
-
|
|
2841
|
-
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
|
|
2856
|
-
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
2860
|
-
|
|
2861
|
-
|
|
2862
|
-
}
|
|
2863
|
-
if (message.type === "stream-cancel") {
|
|
2864
|
-
streams.get(message.requestId)?.cancel();
|
|
2865
|
-
calls.get(message.requestId)?.abort();
|
|
2866
|
-
calls.delete(message.requestId);
|
|
2867
|
-
return;
|
|
2868
|
-
}
|
|
2869
|
-
const call = message.type === "embed" || message.type === "rerank" ? new AbortController() : void 0;
|
|
2870
|
-
if (call !== void 0) calls.set(message.requestId, call);
|
|
2871
|
-
const reply = message.type === "request" ? answerRequest(handlers[message.route], message, port, streams) : message.type === "authorize" ? answerAuthorize(authorize, message) : message.type === "feed-admit" ? answerFeedAdmit(thread.feeds, message) : message.type === "embed" ? answerEmbed(core, message, call?.signal) : message.type === "await-writes" ? answerAwaitWrites(core, message) : answerRerank(core, message, call?.signal);
|
|
2872
|
-
void reply.then(
|
|
2873
|
-
(answer) => {
|
|
2874
|
-
calls.delete(message.requestId);
|
|
2875
|
-
if (answer !== null) port.postMessage(answer);
|
|
2876
|
-
},
|
|
2877
|
-
() => {
|
|
2878
|
-
calls.delete(message.requestId);
|
|
2879
|
-
}
|
|
2880
|
-
);
|
|
2881
|
-
});
|
|
2882
|
-
}
|
|
2883
|
-
function connectToLongQueries(thread) {
|
|
2884
|
-
if (longQueryThread === null) return;
|
|
2885
|
-
const channel = new MessageChannel();
|
|
2886
|
-
thread.port.postMessage({ type: "long-query-port", port: channel.port1 }, [channel.port1]);
|
|
2887
|
-
longQueryThread.port.postMessage({ type: "long-query-port", port: channel.port2 }, [channel.port2]);
|
|
2888
|
-
}
|
|
2889
|
-
function dropLongQueryThread() {
|
|
2890
|
-
if (longQueryThread === null) return;
|
|
2891
|
-
longQueryThread.port.close();
|
|
2892
|
-
longQueryThread = null;
|
|
2893
|
-
}
|
|
2894
|
-
function retireThread(workerId, thread) {
|
|
2895
|
-
releaseDescriptor(thread);
|
|
2896
|
-
thread.port.unref();
|
|
2897
|
-
retiring.add(thread.closed);
|
|
2898
|
-
void thread.closed.then(() => retiring.delete(thread.closed));
|
|
2899
|
-
void executeUntilAnswered(thread.executor, { type: "stopServing", requestId: `stop-serving-${workerId}` }).catch(
|
|
2900
|
-
() => void 0
|
|
2901
|
-
);
|
|
2902
|
-
return thread.closed;
|
|
2903
|
-
}
|
|
2904
|
-
const listener = {
|
|
2905
|
-
async onLongQueryWorkerReady(workerId, executor) {
|
|
2906
|
-
if (stopped) return;
|
|
2907
|
-
dropLongQueryThread();
|
|
2908
|
-
const channel = new MessageChannel();
|
|
2909
|
-
const relaying = {
|
|
2910
|
-
feeds: null,
|
|
2911
|
-
descriptor: null,
|
|
2912
|
-
port: channel.port2,
|
|
2913
|
-
executor,
|
|
2914
|
-
departure: createDepartureWord(),
|
|
2915
|
-
closeAtExit: createCloseAtExitWords(),
|
|
2916
|
-
closed: new Promise((resolve) => channel.port2.once("close", resolve))
|
|
2917
|
-
};
|
|
2918
|
-
relayFrom(relaying);
|
|
2919
|
-
try {
|
|
2920
|
-
await executor.execute(
|
|
2921
|
-
{
|
|
2922
|
-
type: "serveLongQueries",
|
|
2923
|
-
requestId: `serve-long-queries-${workerId}`,
|
|
2924
|
-
settings: {
|
|
2925
|
-
port: channel.port1,
|
|
2926
|
-
touchIntervalMs,
|
|
2927
|
-
limits: options.limits,
|
|
2928
|
-
patternWorkCap: options.core.patternWorkCap,
|
|
2929
|
-
maxRerankWindow: options.core.rerankSettings.maxWindow,
|
|
2930
|
-
allowRegex: options.allowRegex
|
|
2931
|
-
}
|
|
2932
|
-
},
|
|
2933
|
-
[channel.port1]
|
|
2934
|
-
);
|
|
2935
|
-
} catch (err) {
|
|
2936
|
-
channel.port2.close();
|
|
2937
|
-
console.warn(`Worker ${workerId} could not start answering long queries:`, err);
|
|
2938
|
-
return;
|
|
2939
|
-
}
|
|
2940
|
-
if (stopped) {
|
|
2941
|
-
channel.port2.close();
|
|
2942
|
-
return;
|
|
2943
|
-
}
|
|
2944
|
-
longQueryThread = { workerId, port: channel.port2, executor };
|
|
2945
|
-
for (const thread of threads.values()) connectToLongQueries(thread);
|
|
2946
|
-
},
|
|
2947
|
-
async onWorkerReady(workerId, executor) {
|
|
2948
|
-
if (stopped) return;
|
|
2949
|
-
const channel = new MessageChannel();
|
|
2950
|
-
const slot = gateSlotOfWorker(workerId);
|
|
2951
|
-
slotUsers.set(slot, (slotUsers.get(slot) ?? 0) + 1);
|
|
2952
|
-
const thread = {
|
|
2953
|
-
feeds: options.feeds?.forThread(channel.port2) ?? null,
|
|
2954
|
-
descriptor: null,
|
|
2955
|
-
port: channel.port2,
|
|
2956
|
-
executor,
|
|
2957
|
-
departure: createDepartureWord(),
|
|
2958
|
-
closeAtExit: createCloseAtExitWords(),
|
|
2959
|
-
closed: threadClosed(channel.port2, executor, () => {
|
|
2960
|
-
const remaining = (slotUsers.get(slot) ?? 1) - 1;
|
|
2961
|
-
if (remaining > 0) {
|
|
2962
|
-
slotUsers.set(slot, remaining);
|
|
2963
|
-
return;
|
|
2964
|
-
}
|
|
2965
|
-
slotUsers.delete(slot);
|
|
2966
|
-
clearGateSlot(options.gate, slot);
|
|
2967
|
-
clearBodyBudgetSlot(options.bodyBudget, slot);
|
|
2968
|
-
})
|
|
2969
|
-
};
|
|
2970
|
-
relayFrom(thread);
|
|
2971
|
-
closeAtExit.add(thread.closeAtExit);
|
|
2972
|
-
const longQueries = longQueryThread;
|
|
2973
|
-
const longQueryChannel = longQueries === null ? null : new MessageChannel();
|
|
2974
|
-
const transfer = longQueryChannel === null ? [channel.port1] : [channel.port1, longQueryChannel.port1];
|
|
2975
|
-
let result;
|
|
2976
|
-
try {
|
|
2977
|
-
result = await executor.execute(
|
|
2978
|
-
{
|
|
2979
|
-
type: "serveRequests",
|
|
2980
|
-
requestId: `serve-${workerId}`,
|
|
2981
|
-
settings: {
|
|
2982
|
-
port: channel.port1,
|
|
2983
|
-
departure: thread.departure.buffer,
|
|
2984
|
-
closeAtExit: thread.closeAtExit.buffer,
|
|
2985
|
-
gate: options.gate,
|
|
2986
|
-
gateSlot: gateSlotOfWorker(workerId),
|
|
2987
|
-
maxConcurrentRequests: options.limits.maxConcurrentRequests,
|
|
2988
|
-
bodyBudget: options.bodyBudget,
|
|
2989
|
-
touchIntervalMs,
|
|
2990
|
-
cors: options.cors,
|
|
2991
|
-
limits: options.limits,
|
|
2992
|
-
build: options.build,
|
|
2993
|
-
authorizes: authorize !== void 0,
|
|
2994
|
-
searchHooks,
|
|
2995
|
-
patternWorkCap: options.core.patternWorkCap,
|
|
2996
|
-
maxRerankWindow: options.core.rerankSettings.maxWindow,
|
|
2997
|
-
allowRegex: options.allowRegex,
|
|
2998
|
-
feeds: Object.keys(options.feeds?.gateways ?? {}),
|
|
2999
|
-
longQueriesInFlight,
|
|
3000
|
-
longQueryPort: longQueryChannel?.port1 ?? null
|
|
3001
|
-
}
|
|
3002
|
-
},
|
|
3003
|
-
transfer
|
|
2823
|
+
};
|
|
2824
|
+
const inFlight = /* @__PURE__ */ new Set();
|
|
2825
|
+
let flushError;
|
|
2826
|
+
const startFlush = () => {
|
|
2827
|
+
if (pending.length === 0) return;
|
|
2828
|
+
const documents = pending.map((entry) => entry.document);
|
|
2829
|
+
pending = [];
|
|
2830
|
+
const run = flushDocuments(documents).catch((err) => {
|
|
2831
|
+
flushError ??= err;
|
|
2832
|
+
}).finally(() => inFlight.delete(run));
|
|
2833
|
+
inFlight.add(run);
|
|
2834
|
+
};
|
|
2835
|
+
const settleFlushes = async () => {
|
|
2836
|
+
await Promise.all(inFlight);
|
|
2837
|
+
if (flushError !== void 0) throw flushError;
|
|
2838
|
+
};
|
|
2839
|
+
const report = () => {
|
|
2840
|
+
onProgress?.({ indexed, failed, bytesProcessed, bytesTotal });
|
|
2841
|
+
};
|
|
2842
|
+
const seenIds = /* @__PURE__ */ new Set();
|
|
2843
|
+
const repeated = [];
|
|
2844
|
+
try {
|
|
2845
|
+
for (const line of iterateNdjson(body, maxLineBytes)) {
|
|
2846
|
+
bytesProcessed = line.bytesConsumed;
|
|
2847
|
+
const refusal = parseRefusalOf(line.bytes);
|
|
2848
|
+
if (refusal !== null) {
|
|
2849
|
+
const block = refusal.status === 429 && writeBlock !== void 0 ? await writeBlock(indexName).catch(() => null) : null;
|
|
2850
|
+
recordFailure(
|
|
2851
|
+
block === null ? { line: line.lineNumber, code: refusal.code, message: refusal.message, details: refusal.details } : { line: line.lineNumber, ...serializeNarsilError(block) }
|
|
3004
2852
|
);
|
|
3005
|
-
|
|
3006
|
-
closeAtExit.delete(thread.closeAtExit);
|
|
3007
|
-
channel.port2.close();
|
|
3008
|
-
longQueryChannel?.port2.close();
|
|
3009
|
-
console.warn(`Worker ${workerId} could not start taking requests:`, err);
|
|
3010
|
-
return;
|
|
2853
|
+
continue;
|
|
3011
2854
|
}
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
2855
|
+
let parsed;
|
|
2856
|
+
try {
|
|
2857
|
+
parsed = JSON.parse(line.bytes.toString("utf8"));
|
|
2858
|
+
} catch {
|
|
2859
|
+
recordFailure({ line: line.lineNumber, code: ServerErrorCodes.INVALID_JSON, message: "Line is not valid JSON" });
|
|
2860
|
+
continue;
|
|
3017
2861
|
}
|
|
3018
|
-
if (
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
2862
|
+
if (!isPlainObject(parsed)) {
|
|
2863
|
+
recordFailure({
|
|
2864
|
+
line: line.lineNumber,
|
|
2865
|
+
code: ServerErrorCodes.INVALID_REQUEST,
|
|
2866
|
+
message: "Line is not a JSON object"
|
|
2867
|
+
});
|
|
2868
|
+
continue;
|
|
3025
2869
|
}
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
2870
|
+
const docId = providedDocId(parsed);
|
|
2871
|
+
if (docId !== void 0) {
|
|
2872
|
+
if (seenIds.has(docId)) {
|
|
2873
|
+
repeated.push({ document: parsed, line: line.lineNumber });
|
|
2874
|
+
continue;
|
|
2875
|
+
}
|
|
2876
|
+
seenIds.add(docId);
|
|
2877
|
+
}
|
|
2878
|
+
pending.push({ document: parsed, line: line.lineNumber });
|
|
2879
|
+
if (pending.length >= batchSize) {
|
|
2880
|
+
signal?.throwIfAborted();
|
|
2881
|
+
startFlush();
|
|
2882
|
+
if (inFlight.size >= IMPORT_FLUSHES_IN_FLIGHT) await Promise.race(inFlight);
|
|
2883
|
+
if (flushError !== void 0) await settleFlushes();
|
|
2884
|
+
report();
|
|
2885
|
+
await yieldToEventLoop();
|
|
2886
|
+
signal?.throwIfAborted();
|
|
2887
|
+
}
|
|
2888
|
+
}
|
|
2889
|
+
startFlush();
|
|
2890
|
+
await settleFlushes();
|
|
2891
|
+
for (let start = 0; start < repeated.length; start += batchSize) {
|
|
2892
|
+
signal?.throwIfAborted();
|
|
2893
|
+
await flushDocuments(repeated.slice(start, start + batchSize).map((entry) => entry.document));
|
|
2894
|
+
}
|
|
2895
|
+
} catch (err) {
|
|
2896
|
+
await Promise.allSettled(inFlight);
|
|
2897
|
+
report();
|
|
2898
|
+
throw err;
|
|
2899
|
+
}
|
|
2900
|
+
bytesProcessed = bytesTotal;
|
|
2901
|
+
report();
|
|
2902
|
+
return { indexed, failed, errors, errorsTruncated: failed > errors.length };
|
|
2903
|
+
}
|
|
2904
|
+
function createImportHandler(deps) {
|
|
2905
|
+
const { engine, limits, tasks } = deps;
|
|
2906
|
+
return async function importNdjson(ctx) {
|
|
2907
|
+
const raw = ctx.rawBody;
|
|
2908
|
+
if (!raw || raw.length === 0) {
|
|
2909
|
+
sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
|
|
2910
|
+
return;
|
|
2911
|
+
}
|
|
2912
|
+
const indexName = ctx.params[0];
|
|
2913
|
+
const runOptions = {
|
|
2914
|
+
indexName,
|
|
2915
|
+
body: raw,
|
|
2916
|
+
maxLineBytes: limits.maxLineBytes,
|
|
2917
|
+
batchSize: limits.importBatchSize,
|
|
2918
|
+
maxErrors: limits.maxImportErrors
|
|
2919
|
+
};
|
|
2920
|
+
if (ctx.query.get("async") === "true") {
|
|
2921
|
+
try {
|
|
2922
|
+
await engine.countDocuments(indexName);
|
|
2923
|
+
} catch (err) {
|
|
2924
|
+
respondError(ctx, err);
|
|
3032
2925
|
return;
|
|
3033
2926
|
}
|
|
3034
|
-
const
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
2927
|
+
const record = await tasks.start(
|
|
2928
|
+
"import",
|
|
2929
|
+
indexName,
|
|
2930
|
+
async (context) => {
|
|
2931
|
+
const result = await runImport(engine, {
|
|
2932
|
+
...runOptions,
|
|
2933
|
+
signal: context.signal,
|
|
2934
|
+
onProgress: context.reportProgress
|
|
2935
|
+
});
|
|
2936
|
+
context.reportResult(result);
|
|
2937
|
+
},
|
|
2938
|
+
{ indexed: 0, failed: 0, bytesProcessed: 0, bytesTotal: raw.length }
|
|
2939
|
+
);
|
|
2940
|
+
respondJson(ctx, record, 202);
|
|
2941
|
+
return;
|
|
2942
|
+
}
|
|
2943
|
+
const controller = new AbortController();
|
|
2944
|
+
ctx.abort.onAbort(() => controller.abort());
|
|
2945
|
+
try {
|
|
2946
|
+
respondJson(ctx, await runImport(engine, { ...runOptions, signal: controller.signal }));
|
|
2947
|
+
} catch (err) {
|
|
2948
|
+
if (ctx.abort.aborted) return;
|
|
2949
|
+
respondError(ctx, err);
|
|
3038
2950
|
}
|
|
3039
2951
|
};
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
2952
|
+
}
|
|
2953
|
+
|
|
2954
|
+
// src/server/config-mapping.ts
|
|
2955
|
+
function mapHttpIndexConfig(http) {
|
|
2956
|
+
if (typeof http !== "object" || http === null) {
|
|
2957
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index config must be an object");
|
|
2958
|
+
}
|
|
2959
|
+
if (typeof http.schema !== "object" || http.schema === null || Array.isArray(http.schema)) {
|
|
2960
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Index config requires a "schema" object');
|
|
2961
|
+
}
|
|
2962
|
+
const config = { schema: http.schema };
|
|
2963
|
+
if (http.language !== void 0) config.language = http.language;
|
|
2964
|
+
if (http.partitions !== void 0) config.partitions = http.partitions;
|
|
2965
|
+
if (http.defaultScoring !== void 0) config.defaultScoring = http.defaultScoring;
|
|
2966
|
+
if (http.bm25 !== void 0) config.bm25 = http.bm25;
|
|
2967
|
+
if (http.trackPositions !== void 0) config.trackPositions = http.trackPositions;
|
|
2968
|
+
if (http.textStorage !== void 0) config.textStorage = http.textStorage;
|
|
2969
|
+
if (http.surfaceForms !== void 0) config.surfaceForms = http.surfaceForms === true;
|
|
2970
|
+
if (http.vectorPromotion !== void 0) config.vectorPromotion = http.vectorPromotion;
|
|
2971
|
+
if (http.strict !== void 0) config.strict = http.strict;
|
|
2972
|
+
if (http.required !== void 0) config.required = http.required;
|
|
2973
|
+
if (http.patternValueLimit !== void 0) config.patternValueLimit = http.patternValueLimit;
|
|
2974
|
+
if (http.synonyms !== void 0) config.synonyms = http.synonyms;
|
|
2975
|
+
if (http.tokenizer !== void 0) {
|
|
2976
|
+
if (typeof http.tokenizer !== "string") {
|
|
2977
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Field "tokenizer" must be a registered name');
|
|
2978
|
+
}
|
|
2979
|
+
config.tokenizer = http.tokenizer;
|
|
2980
|
+
}
|
|
2981
|
+
if (typeof http.stopWords === "string") {
|
|
2982
|
+
config.stopWords = http.stopWords;
|
|
2983
|
+
} else if (http.stopWords !== void 0) {
|
|
2984
|
+
if (!Array.isArray(http.stopWords) || http.stopWords.some((w) => typeof w !== "string")) {
|
|
2985
|
+
throw new NarsilError(
|
|
2986
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2987
|
+
'Field "stopWords" must be a registered name or an array of strings'
|
|
2988
|
+
);
|
|
2989
|
+
}
|
|
2990
|
+
config.stopWords = new Set(http.stopWords);
|
|
2991
|
+
}
|
|
2992
|
+
if (http.embedding !== void 0) {
|
|
2993
|
+
config.embedding = mapEmbedding(http.embedding);
|
|
2994
|
+
}
|
|
2995
|
+
return config;
|
|
2996
|
+
}
|
|
2997
|
+
function mapEmbedding(embedding) {
|
|
2998
|
+
if (typeof embedding !== "object" || embedding === null) {
|
|
2999
|
+
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, 'Field "embedding" must be an object');
|
|
3000
|
+
}
|
|
3001
|
+
if (typeof embedding.fields !== "object" || embedding.fields === null) {
|
|
3002
|
+
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, 'Field "embedding.fields" is required');
|
|
3003
|
+
}
|
|
3004
|
+
const mapped = { fields: embedding.fields };
|
|
3005
|
+
if (embedding.adapter !== void 0) {
|
|
3006
|
+
if (typeof embedding.adapter !== "string" || embedding.adapter.length === 0) {
|
|
3007
|
+
throw new NarsilError(
|
|
3008
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3009
|
+
'Field "embedding.adapter" must be a non-empty adapter name'
|
|
3010
|
+
);
|
|
3011
|
+
}
|
|
3012
|
+
mapped.adapter = embedding.adapter;
|
|
3013
|
+
}
|
|
3014
|
+
return mapped;
|
|
3015
|
+
}
|
|
3016
|
+
|
|
3017
|
+
// src/server/handlers/indexes.ts
|
|
3018
|
+
function createIndexHandlers(deps) {
|
|
3019
|
+
const { engine } = deps;
|
|
3020
|
+
async function create(ctx) {
|
|
3021
|
+
const body = parseJsonObject(ctx);
|
|
3022
|
+
if (!body) return;
|
|
3023
|
+
const failure = validateCreateIndex(body);
|
|
3024
|
+
if (failure) {
|
|
3025
|
+
rejectInvalid(ctx, failure);
|
|
3026
|
+
return;
|
|
3027
|
+
}
|
|
3028
|
+
try {
|
|
3029
|
+
const config = mapHttpIndexConfig(body.config);
|
|
3030
|
+
await engine.createIndex(body.name, config);
|
|
3031
|
+
respondJson(ctx, { name: body.name }, 201);
|
|
3032
|
+
} catch (err) {
|
|
3033
|
+
respondError(ctx, err);
|
|
3034
|
+
}
|
|
3035
|
+
}
|
|
3036
|
+
async function list(ctx) {
|
|
3037
|
+
try {
|
|
3038
|
+
respondJson(ctx, { indexes: await indexReadsOf(engine).listIndexes() });
|
|
3039
|
+
} catch (err) {
|
|
3040
|
+
respondError(ctx, err);
|
|
3041
|
+
}
|
|
3042
|
+
}
|
|
3043
|
+
async function drop(ctx) {
|
|
3044
|
+
try {
|
|
3045
|
+
await engine.dropIndex(ctx.params[0]);
|
|
3046
|
+
respondJson(ctx, { name: ctx.params[0], dropped: true });
|
|
3047
|
+
} catch (err) {
|
|
3048
|
+
respondError(ctx, err);
|
|
3049
|
+
}
|
|
3050
|
+
}
|
|
3051
|
+
async function stats(ctx) {
|
|
3052
|
+
try {
|
|
3053
|
+
respondJson(ctx, await indexReadsOf(engine).getStats(ctx.params[0]));
|
|
3054
|
+
} catch (err) {
|
|
3055
|
+
respondError(ctx, err);
|
|
3056
|
+
}
|
|
3057
|
+
}
|
|
3058
|
+
async function partitions(ctx) {
|
|
3059
|
+
try {
|
|
3060
|
+
respondJson(ctx, {
|
|
3061
|
+
partitions: await indexReadsOf(engine).getPartitionStats(ctx.params[0])
|
|
3052
3062
|
});
|
|
3053
|
-
|
|
3054
|
-
|
|
3055
|
-
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
|
|
3059
|
-
|
|
3060
|
-
|
|
3061
|
-
|
|
3063
|
+
} catch (err) {
|
|
3064
|
+
respondError(ctx, err);
|
|
3065
|
+
}
|
|
3066
|
+
}
|
|
3067
|
+
async function clear(ctx) {
|
|
3068
|
+
try {
|
|
3069
|
+
await engine.clear(ctx.params[0]);
|
|
3070
|
+
respondJson(ctx, { name: ctx.params[0], cleared: true });
|
|
3071
|
+
} catch (err) {
|
|
3072
|
+
respondError(ctx, err);
|
|
3073
|
+
}
|
|
3074
|
+
}
|
|
3075
|
+
async function open(ctx) {
|
|
3076
|
+
try {
|
|
3077
|
+
await engine.open(ctx.params[0]);
|
|
3078
|
+
respondJson(ctx, { name: ctx.params[0], state: "open" });
|
|
3079
|
+
} catch (err) {
|
|
3080
|
+
respondError(ctx, err);
|
|
3081
|
+
}
|
|
3082
|
+
}
|
|
3083
|
+
async function close(ctx) {
|
|
3084
|
+
try {
|
|
3085
|
+
await engine.close(ctx.params[0]);
|
|
3086
|
+
respondJson(ctx, { name: ctx.params[0], state: "closed" });
|
|
3087
|
+
} catch (err) {
|
|
3088
|
+
respondError(ctx, err);
|
|
3089
|
+
}
|
|
3090
|
+
}
|
|
3091
|
+
return { create, list, drop, stats, partitions, clear, open, close };
|
|
3092
|
+
}
|
|
3093
|
+
|
|
3094
|
+
// src/server/handlers/metrics.ts
|
|
3095
|
+
function labelValue(value) {
|
|
3096
|
+
return value.replaceAll("\\", "\\\\").replaceAll('"', '\\"').replaceAll("\n", "\\n");
|
|
3097
|
+
}
|
|
3098
|
+
function labels(pairs) {
|
|
3099
|
+
return `{${pairs.map(([name, value]) => `${name}="${labelValue(value)}"`).join(",")}}`;
|
|
3100
|
+
}
|
|
3101
|
+
function family(name, type, help, samples) {
|
|
3102
|
+
return [`# HELP ${name} ${help}`, `# TYPE ${name} ${type}`, ...samples];
|
|
3103
|
+
}
|
|
3104
|
+
function prometheusText(metrics) {
|
|
3105
|
+
const lines = [
|
|
3106
|
+
...family(
|
|
3107
|
+
"narsil_replication_lag",
|
|
3108
|
+
"gauge",
|
|
3109
|
+
"Entries that the replica has yet to acknowledge, as the primary's last sequence number minus the replica's.",
|
|
3110
|
+
metrics.replicationLag.map(
|
|
3111
|
+
(figure) => `narsil_replication_lag${labels([
|
|
3112
|
+
["index", figure.indexName],
|
|
3113
|
+
["partition", String(figure.partitionId)],
|
|
3114
|
+
["replica", figure.replica]
|
|
3115
|
+
])} ${figure.lag}`
|
|
3116
|
+
)
|
|
3117
|
+
),
|
|
3118
|
+
...family(
|
|
3119
|
+
"narsil_commit_point",
|
|
3120
|
+
"gauge",
|
|
3121
|
+
"The highest sequence number that the leader of the partition has acknowledged to a client.",
|
|
3122
|
+
metrics.commitPoints.map(
|
|
3123
|
+
(figure) => `narsil_commit_point${labels([
|
|
3124
|
+
["index", figure.indexName],
|
|
3125
|
+
["partition", String(figure.partitionId)]
|
|
3126
|
+
])} ${figure.commitPoint}`
|
|
3127
|
+
)
|
|
3128
|
+
),
|
|
3129
|
+
...family(
|
|
3130
|
+
"narsil_commit_point_floor",
|
|
3131
|
+
"gauge",
|
|
3132
|
+
"The floor on the commit point that the allocation table stores, raised when the controller admits a replica.",
|
|
3133
|
+
metrics.partitions.map(
|
|
3134
|
+
(figure) => `narsil_commit_point_floor${labels([
|
|
3135
|
+
["index", figure.indexName],
|
|
3136
|
+
["partition", String(figure.partitionId)]
|
|
3137
|
+
])} ${figure.commitPointFloor}`
|
|
3138
|
+
)
|
|
3139
|
+
),
|
|
3140
|
+
...family(
|
|
3141
|
+
"narsil_in_sync_replicas",
|
|
3142
|
+
"gauge",
|
|
3143
|
+
"The replicas besides the primary that are in the in-sync set of the partition.",
|
|
3144
|
+
metrics.partitions.map(
|
|
3145
|
+
(figure) => `narsil_in_sync_replicas${labels([
|
|
3146
|
+
["index", figure.indexName],
|
|
3147
|
+
["partition", String(figure.partitionId)]
|
|
3148
|
+
])} ${figure.inSyncReplicas}`
|
|
3149
|
+
)
|
|
3150
|
+
),
|
|
3151
|
+
...family(
|
|
3152
|
+
"narsil_vector_rebuild_waiting_for_memory",
|
|
3153
|
+
"gauge",
|
|
3154
|
+
"A vector field whose rebuild waits for the node to have memory for a second graph, reported as 1.",
|
|
3155
|
+
metrics.vectorRebuildsWaitingForMemory.map(
|
|
3156
|
+
(figure) => `narsil_vector_rebuild_waiting_for_memory${labels([
|
|
3157
|
+
["index", figure.indexName],
|
|
3158
|
+
["field", figure.fieldName]
|
|
3159
|
+
])} 1`
|
|
3160
|
+
)
|
|
3161
|
+
),
|
|
3162
|
+
...family(
|
|
3163
|
+
"narsil_coordinator_request_seconds",
|
|
3164
|
+
"histogram",
|
|
3165
|
+
"How long the calls from this node to the cluster coordinator took.",
|
|
3166
|
+
metrics.coordinatorRequests.flatMap((figure) => {
|
|
3167
|
+
const operation = ["operation", figure.operation];
|
|
3168
|
+
return [
|
|
3169
|
+
...figure.bucketBoundsSeconds.map(
|
|
3170
|
+
(bound, position) => `narsil_coordinator_request_seconds_bucket${labels([operation, ["le", String(bound)]])} ${figure.bucketCounts[position] ?? 0}`
|
|
3171
|
+
),
|
|
3172
|
+
`narsil_coordinator_request_seconds_bucket${labels([operation, ["le", "+Inf"]])} ${figure.count}`,
|
|
3173
|
+
`narsil_coordinator_request_seconds_sum${labels([operation])} ${figure.sumSeconds}`,
|
|
3174
|
+
`narsil_coordinator_request_seconds_count${labels([operation])} ${figure.count}`
|
|
3175
|
+
];
|
|
3176
|
+
})
|
|
3177
|
+
)
|
|
3178
|
+
];
|
|
3179
|
+
return `${lines.join("\n")}
|
|
3180
|
+
`;
|
|
3181
|
+
}
|
|
3182
|
+
function createMetricsHandler(deps) {
|
|
3183
|
+
return async function metrics(ctx) {
|
|
3184
|
+
try {
|
|
3185
|
+
if (deps.cluster === void 0) {
|
|
3186
|
+
throw new NarsilError(
|
|
3187
|
+
ErrorCodes.CLUSTER_OPERATION_UNSUPPORTED,
|
|
3188
|
+
"This server fronts a single engine, so it reports no cluster metrics",
|
|
3189
|
+
{}
|
|
3062
3190
|
);
|
|
3063
3191
|
}
|
|
3064
|
-
await
|
|
3192
|
+
const text = prometheusText(await deps.cluster.getMetrics());
|
|
3193
|
+
if (ctx.abort.aborted) return;
|
|
3194
|
+
sendText(ctx.res, text, PROMETHEUS_TEXT_CONTENT_TYPE, ctx.abort);
|
|
3195
|
+
} catch (err) {
|
|
3196
|
+
respondError(ctx, err);
|
|
3065
3197
|
}
|
|
3066
3198
|
};
|
|
3067
3199
|
}
|
|
3068
3200
|
|
|
3069
|
-
// src/server/
|
|
3070
|
-
var
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3075
|
-
|
|
3076
|
-
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
ErrorCodes.CONFIG_INVALID,
|
|
3080
|
-
`${feed}.${field} must be a whole number of at least 1; however, this configuration sets it to ${String(value)}`,
|
|
3081
|
-
{ field, value: String(value) }
|
|
3082
|
-
);
|
|
3201
|
+
// src/server/handlers/query-suggestions.ts
|
|
3202
|
+
var PREVIEW_LIMIT_PATTERN = /^\d{1,7}$/;
|
|
3203
|
+
function readObject(ctx) {
|
|
3204
|
+
const body = parseJson(ctx);
|
|
3205
|
+
if (body === void 0) return void 0;
|
|
3206
|
+
if (!isRecord(body)) {
|
|
3207
|
+
badRequest(ctx.res, "The body must be a JSON object");
|
|
3208
|
+
return void 0;
|
|
3209
|
+
}
|
|
3210
|
+
return body;
|
|
3083
3211
|
}
|
|
3084
|
-
function
|
|
3085
|
-
if (
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
|
|
3091
|
-
|
|
3212
|
+
function readQuery(ctx, body) {
|
|
3213
|
+
if (typeof body.query === "string") return body.query;
|
|
3214
|
+
badRequest(ctx.res, 'The body must carry the phrase as a string in "query"');
|
|
3215
|
+
return void 0;
|
|
3216
|
+
}
|
|
3217
|
+
function isStringList(value) {
|
|
3218
|
+
return Array.isArray(value) && value.every((entry) => typeof entry === "string");
|
|
3219
|
+
}
|
|
3220
|
+
function isIdentity(value) {
|
|
3221
|
+
return isRecord(value) && typeof value.person === "string" && typeof value.group === "string";
|
|
3222
|
+
}
|
|
3223
|
+
function previewLimitOf(raw) {
|
|
3224
|
+
if (raw === null) return void 0;
|
|
3225
|
+
return PREVIEW_LIMIT_PATTERN.test(raw) ? Number(raw) : null;
|
|
3226
|
+
}
|
|
3227
|
+
async function identityOf(options, ctx) {
|
|
3228
|
+
if (ctx.hookContext === null) return void 0;
|
|
3229
|
+
try {
|
|
3230
|
+
const identity = await options.identify(ctx.hookContext);
|
|
3231
|
+
if (identity === null) return null;
|
|
3232
|
+
return isIdentity(identity) ? { person: identity.person, group: identity.group } : void 0;
|
|
3233
|
+
} catch {
|
|
3234
|
+
return void 0;
|
|
3092
3235
|
}
|
|
3093
|
-
const settings = value === true ? {} : value;
|
|
3094
|
-
for (const field of FEED_OPTION_FIELDS) assertCount(feed, field, settings[field]);
|
|
3095
|
-
return {
|
|
3096
|
-
windowBytes: settings.windowBytes ?? DEFAULT_FEED_WINDOW_BYTES,
|
|
3097
|
-
maxConnections: settings.maxConnections ?? DEFAULT_MAX_FEED_CONNECTIONS,
|
|
3098
|
-
maxCatchUps: settings.maxCatchUps ?? DEFAULT_MAX_FEED_CATCH_UPS,
|
|
3099
|
-
maxBufferedBytes: settings.maxBufferedBytes ?? DEFAULT_MAX_FEED_BUFFERED_BYTES
|
|
3100
|
-
};
|
|
3101
3236
|
}
|
|
3102
|
-
function
|
|
3103
|
-
|
|
3104
|
-
|
|
3237
|
+
function noIdentify(indexName) {
|
|
3238
|
+
return new NarsilError(
|
|
3239
|
+
ErrorCodes.QUERY_SUGGESTIONS_NOT_ENABLED,
|
|
3240
|
+
"This server has no identify function for popular query suggestions, so it counts no search",
|
|
3241
|
+
{ indexName }
|
|
3242
|
+
);
|
|
3105
3243
|
}
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
async function
|
|
3244
|
+
function createQuerySuggestionHandlers(deps, options) {
|
|
3245
|
+
const { engine, tasks } = deps;
|
|
3246
|
+
const suggestions = () => querySuggestions(engine);
|
|
3247
|
+
async function update(ctx) {
|
|
3248
|
+
const body = parseJson(ctx);
|
|
3249
|
+
if (body === void 0) return;
|
|
3250
|
+
try {
|
|
3251
|
+
await suggestions().updateSetting(ctx.params[0], body);
|
|
3252
|
+
respondJson(ctx, { ok: true });
|
|
3253
|
+
} catch (err) {
|
|
3254
|
+
respondError(ctx, err);
|
|
3255
|
+
}
|
|
3256
|
+
}
|
|
3257
|
+
async function status(ctx) {
|
|
3258
|
+
try {
|
|
3259
|
+
respondJson(ctx, await suggestions().status(ctx.params[0]));
|
|
3260
|
+
} catch (err) {
|
|
3261
|
+
respondError(ctx, err);
|
|
3262
|
+
}
|
|
3263
|
+
}
|
|
3264
|
+
async function record(ctx) {
|
|
3265
|
+
const body = readObject(ctx);
|
|
3266
|
+
if (body === void 0) return;
|
|
3267
|
+
const extra = Object.keys(body).filter((key) => key !== "query");
|
|
3268
|
+
if (extra.length > 0) {
|
|
3269
|
+
badRequest(ctx.res, 'The body carries the search in "query" alone, because the server identifies the person', {
|
|
3270
|
+
fields: extra
|
|
3271
|
+
});
|
|
3272
|
+
return;
|
|
3273
|
+
}
|
|
3274
|
+
const query = readQuery(ctx, body);
|
|
3275
|
+
if (query === void 0) return;
|
|
3276
|
+
const indexName = ctx.params[0];
|
|
3110
3277
|
try {
|
|
3111
|
-
|
|
3278
|
+
if (options === void 0) throw noIdentify(indexName);
|
|
3279
|
+
const identity = await identityOf(options, ctx);
|
|
3280
|
+
if (identity === void 0) {
|
|
3281
|
+
if (!ctx.abort.aborted) {
|
|
3282
|
+
sendError(ctx.res, 500, ServerErrorCodes.HOOK_ERROR, "The identify function of query suggestions failed");
|
|
3283
|
+
}
|
|
3284
|
+
return;
|
|
3285
|
+
}
|
|
3286
|
+
if (identity === null) {
|
|
3287
|
+
respondJson(ctx, { recorded: false, outcome: "unidentified" });
|
|
3288
|
+
return;
|
|
3289
|
+
}
|
|
3290
|
+
respondJson(ctx, await suggestions().recordQuery(indexName, { query, ...identity }));
|
|
3112
3291
|
} catch (err) {
|
|
3113
3292
|
respondError(ctx, err);
|
|
3114
3293
|
}
|
|
3115
3294
|
}
|
|
3116
|
-
async function
|
|
3117
|
-
const
|
|
3118
|
-
if (body === void 0) return;
|
|
3295
|
+
async function build(ctx) {
|
|
3296
|
+
const indexName = ctx.params[0];
|
|
3119
3297
|
try {
|
|
3120
|
-
|
|
3121
|
-
|
|
3298
|
+
const operations = suggestions();
|
|
3299
|
+
const current = await operations.status(indexName);
|
|
3300
|
+
if (current.setting?.enabled !== true) {
|
|
3301
|
+
throw new NarsilError(
|
|
3302
|
+
ErrorCodes.QUERY_SUGGESTIONS_NOT_ENABLED,
|
|
3303
|
+
`Index "${indexName}" has popular query suggestions off`,
|
|
3304
|
+
{ indexName }
|
|
3305
|
+
);
|
|
3306
|
+
}
|
|
3307
|
+
const task = await tasks.start("querySuggestionsBuild", indexName, async (context) => {
|
|
3308
|
+
const finished = await operations.build(indexName);
|
|
3309
|
+
context.reportBuild(finished);
|
|
3310
|
+
if (finished.status === "failed") {
|
|
3311
|
+
throw new NarsilError(
|
|
3312
|
+
finished.errorCode ?? ErrorCodes.WORKER_ACTION_FAILED,
|
|
3313
|
+
"The build of popular query suggestions failed",
|
|
3314
|
+
{ indexName, buildId: finished.buildId }
|
|
3315
|
+
);
|
|
3316
|
+
}
|
|
3317
|
+
});
|
|
3318
|
+
respondJson(ctx, task, 202);
|
|
3122
3319
|
} catch (err) {
|
|
3123
3320
|
respondError(ctx, err);
|
|
3124
3321
|
}
|
|
3125
3322
|
}
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
var InMemoryTaskStore = class {
|
|
3137
|
-
/**
|
|
3138
|
-
* Builds a store that keeps task records in this process alone.
|
|
3139
|
-
*
|
|
3140
|
-
* @param maxRetained - Records to keep before the oldest finished ones are
|
|
3141
|
-
* dropped. Defaults to 1000.
|
|
3142
|
-
*/
|
|
3143
|
-
constructor(maxRetained = DEFAULT_MAX_RETAINED_TASKS) {
|
|
3144
|
-
this.maxRetained = maxRetained;
|
|
3145
|
-
}
|
|
3146
|
-
records = /* @__PURE__ */ new Map();
|
|
3147
|
-
/**
|
|
3148
|
-
* Stores a record, replacing whatever its id already held.
|
|
3149
|
-
*
|
|
3150
|
-
* @param record - The record to store. A copy is kept, so changing the
|
|
3151
|
-
* object afterwards leaves the store untouched.
|
|
3152
|
-
*/
|
|
3153
|
-
async set(record) {
|
|
3154
|
-
this.records.set(record.id, { ...record });
|
|
3155
|
-
this.prune();
|
|
3156
|
-
}
|
|
3157
|
-
/**
|
|
3158
|
-
* Reads one record back.
|
|
3159
|
-
*
|
|
3160
|
-
* @param id - The task to read.
|
|
3161
|
-
* @returns A copy of the record, or `null` when the store holds no such id.
|
|
3162
|
-
*/
|
|
3163
|
-
async get(id) {
|
|
3164
|
-
const record = this.records.get(id);
|
|
3165
|
-
return record ? { ...record } : null;
|
|
3166
|
-
}
|
|
3167
|
-
/**
|
|
3168
|
-
* Lists every record the store holds.
|
|
3169
|
-
*
|
|
3170
|
-
* @returns A copy of each record, oldest first.
|
|
3171
|
-
*/
|
|
3172
|
-
async list() {
|
|
3173
|
-
return [...this.records.values()].map((record) => ({ ...record }));
|
|
3174
|
-
}
|
|
3175
|
-
/**
|
|
3176
|
-
* Removes one record. An id the store never held is not an error.
|
|
3177
|
-
*
|
|
3178
|
-
* @param id - The task to remove.
|
|
3179
|
-
*/
|
|
3180
|
-
async delete(id) {
|
|
3181
|
-
this.records.delete(id);
|
|
3182
|
-
}
|
|
3183
|
-
prune() {
|
|
3184
|
-
if (this.records.size <= this.maxRetained) return;
|
|
3185
|
-
for (const [id, record] of this.records) {
|
|
3186
|
-
if (this.records.size <= this.maxRetained) break;
|
|
3187
|
-
if (isTerminalTaskStatus(record.status)) this.records.delete(id);
|
|
3323
|
+
async function preview(ctx) {
|
|
3324
|
+
const limit = previewLimitOf(ctx.query.get("limit"));
|
|
3325
|
+
if (limit === null) {
|
|
3326
|
+
badRequest(ctx.res, 'Query parameter "limit" must be a whole number', { value: ctx.query.get("limit") });
|
|
3327
|
+
return;
|
|
3328
|
+
}
|
|
3329
|
+
try {
|
|
3330
|
+
respondJson(ctx, await suggestions().preview(ctx.params[0], limit));
|
|
3331
|
+
} catch (err) {
|
|
3332
|
+
respondError(ctx, err);
|
|
3188
3333
|
}
|
|
3189
3334
|
}
|
|
3190
|
-
|
|
3191
|
-
|
|
3192
|
-
|
|
3193
|
-
|
|
3194
|
-
|
|
3195
|
-
function matchesQuery(record, query) {
|
|
3196
|
-
if (query.indexName !== void 0 && record.indexName !== query.indexName) return false;
|
|
3197
|
-
if (query.type !== void 0 && query.type.length > 0 && !query.type.includes(record.type)) return false;
|
|
3198
|
-
if (query.status !== void 0 && query.status.length > 0 && !query.status.includes(record.status)) return false;
|
|
3199
|
-
return true;
|
|
3200
|
-
}
|
|
3201
|
-
var TaskRegistry = class {
|
|
3202
|
-
constructor(store, instanceId, maxConcurrent = 0) {
|
|
3203
|
-
this.store = store;
|
|
3204
|
-
this.instanceId = instanceId;
|
|
3205
|
-
this.maxConcurrent = maxConcurrent;
|
|
3206
|
-
}
|
|
3207
|
-
live = /* @__PURE__ */ new Map();
|
|
3208
|
-
/** Records a task as running and drives `op` to completion in the background.
|
|
3209
|
-
* Returns the record once it is persisted so the caller can respond 202.
|
|
3210
|
-
* @throws A `NarsilError` with `TOO_MANY_REQUESTS` where this instance
|
|
3211
|
-
* already drives {@link ServerLimits.maxConcurrentTasks} tasks, because each
|
|
3212
|
-
* one holds its own working set until it finishes. */
|
|
3213
|
-
async start(type, indexName, op, progress, backupName) {
|
|
3214
|
-
if (this.maxConcurrent > 0 && this.live.size >= this.maxConcurrent) {
|
|
3215
|
-
throw new NarsilError(
|
|
3216
|
-
ServerErrorCodes.TOO_MANY_REQUESTS,
|
|
3217
|
-
`This instance already drives ${this.maxConcurrent} tasks; poll one to completion before starting another`,
|
|
3218
|
-
{ running: this.live.size, limit: this.maxConcurrent }
|
|
3219
|
-
);
|
|
3335
|
+
async function readBans(ctx) {
|
|
3336
|
+
try {
|
|
3337
|
+
respondJson(ctx, { bans: await suggestions().bans(ctx.params[0]) });
|
|
3338
|
+
} catch (err) {
|
|
3339
|
+
respondError(ctx, err);
|
|
3220
3340
|
}
|
|
3221
|
-
const now = Date.now();
|
|
3222
|
-
const record = {
|
|
3223
|
-
id: randomUUID(),
|
|
3224
|
-
type,
|
|
3225
|
-
indexName,
|
|
3226
|
-
...backupName === void 0 ? {} : { backupName },
|
|
3227
|
-
owner: this.instanceId,
|
|
3228
|
-
status: "running",
|
|
3229
|
-
createdAt: now,
|
|
3230
|
-
startedAt: now,
|
|
3231
|
-
...progress === void 0 ? {} : { progress }
|
|
3232
|
-
};
|
|
3233
|
-
const task = {
|
|
3234
|
-
record,
|
|
3235
|
-
controller: new AbortController(),
|
|
3236
|
-
writes: Promise.resolve(),
|
|
3237
|
-
lastProgressWriteAt: now,
|
|
3238
|
-
settled: false
|
|
3239
|
-
};
|
|
3240
|
-
this.live.set(record.id, task);
|
|
3241
|
-
await this.enqueueWrite(task, record, RUNNING_TASK_TTL_MS);
|
|
3242
|
-
void this.drive(task, op);
|
|
3243
|
-
return record;
|
|
3244
3341
|
}
|
|
3245
|
-
|
|
3246
|
-
|
|
3342
|
+
async function replaceBans(ctx) {
|
|
3343
|
+
const body = readObject(ctx);
|
|
3344
|
+
if (body === void 0) return;
|
|
3345
|
+
if (!isStringList(body.bans)) {
|
|
3346
|
+
badRequest(ctx.res, 'The body must carry the ban list as an array of strings in "bans"');
|
|
3347
|
+
return;
|
|
3348
|
+
}
|
|
3349
|
+
try {
|
|
3350
|
+
respondJson(ctx, { bans: await suggestions().updateBans(ctx.params[0], body.bans) });
|
|
3351
|
+
} catch (err) {
|
|
3352
|
+
respondError(ctx, err);
|
|
3353
|
+
}
|
|
3247
3354
|
}
|
|
3248
|
-
|
|
3249
|
-
|
|
3250
|
-
|
|
3251
|
-
|
|
3252
|
-
|
|
3253
|
-
|
|
3254
|
-
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3355
|
+
async function remove(ctx) {
|
|
3356
|
+
const body = readObject(ctx);
|
|
3357
|
+
if (body === void 0) return;
|
|
3358
|
+
const query = readQuery(ctx, body);
|
|
3359
|
+
if (query === void 0) return;
|
|
3360
|
+
try {
|
|
3361
|
+
const held = await suggestions().removeQuery(ctx.params[0], query);
|
|
3362
|
+
respondJson(ctx, { suggestionIndexes: held });
|
|
3363
|
+
} catch (err) {
|
|
3364
|
+
respondError(ctx, err);
|
|
3365
|
+
}
|
|
3258
3366
|
}
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
|
|
3262
|
-
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
if (
|
|
3266
|
-
|
|
3367
|
+
async function manual(ctx) {
|
|
3368
|
+
const body = readObject(ctx);
|
|
3369
|
+
if (body === void 0) return;
|
|
3370
|
+
const query = readQuery(ctx, body);
|
|
3371
|
+
if (query === void 0) return;
|
|
3372
|
+
const { group, count } = body;
|
|
3373
|
+
if (typeof group !== "string" || count !== void 0 && typeof count !== "number") {
|
|
3374
|
+
badRequest(ctx.res, 'The body must carry the group as a string in "group", and any count as a number in "count"');
|
|
3375
|
+
return;
|
|
3267
3376
|
}
|
|
3268
|
-
|
|
3269
|
-
|
|
3270
|
-
|
|
3271
|
-
|
|
3377
|
+
try {
|
|
3378
|
+
const name = await suggestions().addSuggestion(ctx.params[0], {
|
|
3379
|
+
group,
|
|
3380
|
+
query,
|
|
3381
|
+
...count === void 0 ? {} : { count }
|
|
3382
|
+
});
|
|
3383
|
+
respondJson(ctx, { indexName: name });
|
|
3384
|
+
} catch (err) {
|
|
3385
|
+
respondError(ctx, err);
|
|
3386
|
+
}
|
|
3387
|
+
}
|
|
3388
|
+
return { update, status, record, build, preview, readBans, replaceBans, remove, manual };
|
|
3389
|
+
}
|
|
3390
|
+
|
|
3391
|
+
// src/server/handlers/search.ts
|
|
3392
|
+
function createSearchHandlers(deps) {
|
|
3393
|
+
const { engine, limits, allowRegex } = deps;
|
|
3394
|
+
async function search(ctx) {
|
|
3395
|
+
const params = readQueryBody(ctx, limits, allowRegex, "read");
|
|
3396
|
+
if (!params) return;
|
|
3397
|
+
try {
|
|
3398
|
+
const query = withServerQueryLimits(params, limits);
|
|
3399
|
+
const result = await searchWithSignal(ctx, query);
|
|
3400
|
+
await respondAnswer(ctx, encodeQueryResult(result), "hits", `a search on index '${ctx.params[0]}'`);
|
|
3401
|
+
} catch (err) {
|
|
3402
|
+
respondError(ctx, err);
|
|
3272
3403
|
}
|
|
3273
|
-
task.controller.abort(new Error("The task was cancelled"));
|
|
3274
|
-
const record = { ...task.record, cancelRequestedAt: Date.now() };
|
|
3275
|
-
task.record = record;
|
|
3276
|
-
await this.enqueueWrite(task, record, RUNNING_TASK_TTL_MS);
|
|
3277
|
-
return { outcome: "cancelling", record };
|
|
3278
3404
|
}
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
* a prior life are dead and must not keep showing a stale running status. This
|
|
3282
|
-
* is effective only with a stable instanceId; with the default random id it
|
|
3283
|
-
* finds none of its own prior tasks, which is also correct for the in-memory
|
|
3284
|
-
* default (which starts empty). */
|
|
3285
|
-
async reconcile() {
|
|
3286
|
-
let records;
|
|
3405
|
+
async function searchWithSignal(ctx, params) {
|
|
3406
|
+
bindQuerySignal(params, requestSignal(ctx.abort));
|
|
3287
3407
|
try {
|
|
3288
|
-
|
|
3289
|
-
}
|
|
3290
|
-
|
|
3408
|
+
return await engine.query(ctx.params[0], params);
|
|
3409
|
+
} finally {
|
|
3410
|
+
clearQuerySignal(params);
|
|
3291
3411
|
}
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
|
|
3297
|
-
|
|
3298
|
-
|
|
3299
|
-
error: { code: ServerErrorCodes.TASK_INTERRUPTED, message: "The server restarted while this task was running" }
|
|
3300
|
-
};
|
|
3412
|
+
}
|
|
3413
|
+
async function preflight(ctx) {
|
|
3414
|
+
const params = readQueryBody(ctx, limits, allowRegex, "read");
|
|
3415
|
+
if (!params) return;
|
|
3416
|
+
try {
|
|
3417
|
+
const query = withServerQueryLimits(params, limits);
|
|
3418
|
+
bindQuerySignal(query, requestSignal(ctx.abort));
|
|
3301
3419
|
try {
|
|
3302
|
-
await
|
|
3303
|
-
}
|
|
3420
|
+
respondJson(ctx, await engine.preflight(ctx.params[0], query));
|
|
3421
|
+
} finally {
|
|
3422
|
+
clearQuerySignal(query);
|
|
3304
3423
|
}
|
|
3424
|
+
} catch (err) {
|
|
3425
|
+
respondError(ctx, err);
|
|
3305
3426
|
}
|
|
3306
3427
|
}
|
|
3307
|
-
|
|
3308
|
-
|
|
3309
|
-
|
|
3310
|
-
|
|
3311
|
-
)
|
|
3312
|
-
|
|
3313
|
-
|
|
3428
|
+
async function suggest(ctx) {
|
|
3429
|
+
const params = parseJsonObject(ctx, "read");
|
|
3430
|
+
if (!params) return;
|
|
3431
|
+
const failure = validateSuggest(params);
|
|
3432
|
+
if (failure) {
|
|
3433
|
+
rejectInvalid(ctx, failure);
|
|
3434
|
+
return;
|
|
3435
|
+
}
|
|
3436
|
+
try {
|
|
3437
|
+
respondJson(ctx, await engine.suggest(ctx.params[0], params));
|
|
3438
|
+
} catch (err) {
|
|
3439
|
+
respondError(ctx, err);
|
|
3440
|
+
}
|
|
3314
3441
|
}
|
|
3315
|
-
|
|
3316
|
-
|
|
3317
|
-
|
|
3318
|
-
|
|
3319
|
-
|
|
3320
|
-
|
|
3321
|
-
|
|
3442
|
+
return { search, preflight, suggest };
|
|
3443
|
+
}
|
|
3444
|
+
|
|
3445
|
+
// src/server/sync/setting-handlers.ts
|
|
3446
|
+
function createSyncSettingHandlers(engine) {
|
|
3447
|
+
async function getSync(ctx) {
|
|
3448
|
+
try {
|
|
3449
|
+
respondJson(ctx, await engine.getSync(ctx.params[0]));
|
|
3450
|
+
} catch (err) {
|
|
3451
|
+
respondError(ctx, err);
|
|
3452
|
+
}
|
|
3322
3453
|
}
|
|
3323
|
-
async
|
|
3324
|
-
const
|
|
3325
|
-
|
|
3326
|
-
reportProgress: (progress) => this.reportProgress(task, { progress }),
|
|
3327
|
-
reportSteps: (steps) => this.reportProgress(task, { steps }),
|
|
3328
|
-
reportResult: (result) => {
|
|
3329
|
-
task.record = { ...task.record, result };
|
|
3330
|
-
},
|
|
3331
|
-
reportBuild: (build) => {
|
|
3332
|
-
task.record = { ...task.record, build };
|
|
3333
|
-
}
|
|
3334
|
-
};
|
|
3335
|
-
let settled;
|
|
3454
|
+
async function updateSync(ctx) {
|
|
3455
|
+
const body = parseJson(ctx);
|
|
3456
|
+
if (body === void 0) return;
|
|
3336
3457
|
try {
|
|
3337
|
-
await
|
|
3338
|
-
|
|
3458
|
+
await engine.updateSync(ctx.params[0], body);
|
|
3459
|
+
respondJson(ctx, { ok: true });
|
|
3339
3460
|
} catch (err) {
|
|
3340
|
-
|
|
3341
|
-
...task.record,
|
|
3342
|
-
status: "failed",
|
|
3343
|
-
error: err instanceof NarsilError ? serializeNarsilError(err) : { code: "INTERNAL_ERROR", message: "The operation failed" }
|
|
3344
|
-
};
|
|
3461
|
+
respondError(ctx, err);
|
|
3345
3462
|
}
|
|
3346
|
-
task.settled = true;
|
|
3347
|
-
task.record = { ...settled, completedAt: Date.now() };
|
|
3348
|
-
await this.enqueueWrite(task, task.record, TERMINAL_TASK_TTL_MS);
|
|
3349
|
-
this.live.delete(task.record.id);
|
|
3350
3463
|
}
|
|
3351
|
-
};
|
|
3352
|
-
|
|
3353
|
-
// src/server/create-server.ts
|
|
3354
|
-
function bodyBytesInFlightOf(fraction) {
|
|
3355
|
-
const processLimitBytes = readMemoryFigures()?.processLimitBytes;
|
|
3356
|
-
if (fraction === 0 || processLimitBytes === void 0) return Number.POSITIVE_INFINITY;
|
|
3357
|
-
return Math.floor(fraction * processLimitBytes);
|
|
3358
|
-
}
|
|
3359
|
-
function resolveLimits(limits, engineRerankWindow) {
|
|
3360
|
-
return {
|
|
3361
|
-
maxBodyBytesInFlight: bodyBytesInFlightOf(
|
|
3362
|
-
limits?.maxBodyBytesInFlightFraction ?? DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION
|
|
3363
|
-
),
|
|
3364
|
-
maxBodyBytes: limits?.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES,
|
|
3365
|
-
maxImportBytes: limits?.maxImportBytes ?? DEFAULT_MAX_IMPORT_BYTES,
|
|
3366
|
-
maxRestoreBytes: limits?.maxRestoreBytes ?? DEFAULT_MAX_RESTORE_BYTES,
|
|
3367
|
-
maxLineBytes: limits?.maxLineBytes ?? DEFAULT_MAX_LINE_BYTES,
|
|
3368
|
-
importBatchSize: limits?.importBatchSize ?? DEFAULT_IMPORT_BATCH_SIZE,
|
|
3369
|
-
maxConcurrentRequests: limits?.maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS,
|
|
3370
|
-
maxResultWindow: limits?.maxResultWindow ?? DEFAULT_MAX_RESULT_WINDOW,
|
|
3371
|
-
maxRerankWindow: limits?.maxRerankWindow ?? Math.min(DEFAULT_MAX_RERANK_WINDOW, engineRerankWindow ?? DEFAULT_MAX_RERANK_WINDOW),
|
|
3372
|
-
maxFetchDocuments: limits?.maxFetchDocuments ?? DEFAULT_MAX_FETCH_DOCUMENTS,
|
|
3373
|
-
maxImportErrors: limits?.maxImportErrors ?? DEFAULT_MAX_IMPORT_ERRORS,
|
|
3374
|
-
maxTaskPageSize: limits?.maxTaskPageSize ?? DEFAULT_MAX_TASK_PAGE_SIZE,
|
|
3375
|
-
maxConcurrentTasks: limits?.maxConcurrentTasks ?? DEFAULT_MAX_CONCURRENT_TASKS
|
|
3376
|
-
};
|
|
3377
|
-
}
|
|
3378
|
-
function resolveBuild(build) {
|
|
3379
|
-
return {
|
|
3380
|
-
version: build?.version ?? publishedPackageVersion(),
|
|
3381
|
-
gitSha: build?.gitSha ?? null,
|
|
3382
|
-
dirty: build?.dirty ?? false
|
|
3383
|
-
};
|
|
3384
|
-
}
|
|
3385
|
-
function isLoopback(host) {
|
|
3386
|
-
return host === "127.0.0.1" || host === "::1" || host === "localhost" || host.startsWith("127.");
|
|
3387
|
-
}
|
|
3388
|
-
function assertSecureBinding(host, options) {
|
|
3389
|
-
if (isLoopback(host) || options.onRequest || options.allowInsecure) return;
|
|
3390
|
-
throw new NarsilError(
|
|
3391
|
-
ErrorCodes.CONFIG_INVALID,
|
|
3392
|
-
`Refusing to start: bound to a non-loopback address ("${host}") with no onRequest auth hook. The server exposes destructive admin endpoints (restore, drop, clear, rebalance, optimize). Resolve one of: (1) set options.onRequest to authenticate requests, (2) bind to 127.0.0.1, or (3) set options.allowInsecure: true if this address is on a trusted private network.`,
|
|
3393
|
-
{ host }
|
|
3394
|
-
);
|
|
3464
|
+
return { getSync, updateSync };
|
|
3395
3465
|
}
|
|
3466
|
+
|
|
3467
|
+
// src/server/create-server/handlers.ts
|
|
3396
3468
|
function mainThreadHandlers(deps, syncServing) {
|
|
3397
3469
|
const idx = createIndexHandlers(deps);
|
|
3398
3470
|
const doc = createDocumentHandlers(deps);
|
|
@@ -3479,6 +3551,39 @@ function mainThreadHandlers(deps, syncServing) {
|
|
|
3479
3551
|
addManualSuggestion: suggestions.manual
|
|
3480
3552
|
};
|
|
3481
3553
|
}
|
|
3554
|
+
|
|
3555
|
+
// src/server/create-server/server.ts
|
|
3556
|
+
function bodyBytesInFlightOf(fraction) {
|
|
3557
|
+
const processLimitBytes = readMemoryFigures()?.processLimitBytes;
|
|
3558
|
+
if (fraction === 0 || processLimitBytes === void 0) return Number.POSITIVE_INFINITY;
|
|
3559
|
+
return Math.floor(fraction * processLimitBytes);
|
|
3560
|
+
}
|
|
3561
|
+
function resolveLimits(limits, engineRerankWindow) {
|
|
3562
|
+
return {
|
|
3563
|
+
maxBodyBytesInFlight: bodyBytesInFlightOf(
|
|
3564
|
+
limits?.maxBodyBytesInFlightFraction ?? DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION
|
|
3565
|
+
),
|
|
3566
|
+
maxBodyBytes: limits?.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES,
|
|
3567
|
+
maxImportBytes: limits?.maxImportBytes ?? DEFAULT_MAX_IMPORT_BYTES,
|
|
3568
|
+
maxRestoreBytes: limits?.maxRestoreBytes ?? DEFAULT_MAX_RESTORE_BYTES,
|
|
3569
|
+
maxLineBytes: limits?.maxLineBytes ?? DEFAULT_MAX_LINE_BYTES,
|
|
3570
|
+
importBatchSize: limits?.importBatchSize ?? DEFAULT_IMPORT_BATCH_SIZE,
|
|
3571
|
+
maxConcurrentRequests: limits?.maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS,
|
|
3572
|
+
maxResultWindow: limits?.maxResultWindow ?? DEFAULT_MAX_RESULT_WINDOW,
|
|
3573
|
+
maxRerankWindow: limits?.maxRerankWindow ?? Math.min(DEFAULT_MAX_RERANK_WINDOW, engineRerankWindow ?? DEFAULT_MAX_RERANK_WINDOW),
|
|
3574
|
+
maxFetchDocuments: limits?.maxFetchDocuments ?? DEFAULT_MAX_FETCH_DOCUMENTS,
|
|
3575
|
+
maxImportErrors: limits?.maxImportErrors ?? DEFAULT_MAX_IMPORT_ERRORS,
|
|
3576
|
+
maxTaskPageSize: limits?.maxTaskPageSize ?? DEFAULT_MAX_TASK_PAGE_SIZE,
|
|
3577
|
+
maxConcurrentTasks: limits?.maxConcurrentTasks ?? DEFAULT_MAX_CONCURRENT_TASKS
|
|
3578
|
+
};
|
|
3579
|
+
}
|
|
3580
|
+
function resolveBuild(build) {
|
|
3581
|
+
return {
|
|
3582
|
+
version: build?.version ?? publishedPackageVersion(),
|
|
3583
|
+
gitSha: build?.gitSha ?? null,
|
|
3584
|
+
dirty: build?.dirty ?? false
|
|
3585
|
+
};
|
|
3586
|
+
}
|
|
3482
3587
|
var NarsilHttpServer = class {
|
|
3483
3588
|
host;
|
|
3484
3589
|
port;
|
|
@@ -3486,6 +3591,8 @@ var NarsilHttpServer = class {
|
|
|
3486
3591
|
engine;
|
|
3487
3592
|
deps;
|
|
3488
3593
|
uws = null;
|
|
3594
|
+
app = null;
|
|
3595
|
+
closingRequests = null;
|
|
3489
3596
|
listenSocket = null;
|
|
3490
3597
|
ready = false;
|
|
3491
3598
|
threads = null;
|
|
@@ -3494,6 +3601,9 @@ var NarsilHttpServer = class {
|
|
|
3494
3601
|
syncServing = null;
|
|
3495
3602
|
feeds = null;
|
|
3496
3603
|
stopIdleHeapCleanup = () => void 0;
|
|
3604
|
+
listening = null;
|
|
3605
|
+
closing = null;
|
|
3606
|
+
cancelListen = null;
|
|
3497
3607
|
constructor(engine, options = {}) {
|
|
3498
3608
|
assertServerEngine(engine);
|
|
3499
3609
|
const engineRerankWindow = rerankWindowCapOf(engine);
|
|
@@ -3528,14 +3638,36 @@ var NarsilHttpServer = class {
|
|
|
3528
3638
|
querySuggestions: options.querySuggestions
|
|
3529
3639
|
};
|
|
3530
3640
|
}
|
|
3531
|
-
|
|
3641
|
+
listen() {
|
|
3642
|
+
if (this.listening !== null) return this.listening;
|
|
3643
|
+
const closing = this.closing;
|
|
3644
|
+
this.listening = (async () => {
|
|
3645
|
+
if (closing !== null) await closing;
|
|
3646
|
+
if (this.listenSocket !== null) return;
|
|
3647
|
+
let cancelled = false;
|
|
3648
|
+
const cancellation = new Promise((resolve) => {
|
|
3649
|
+
this.cancelListen = () => {
|
|
3650
|
+
cancelled = true;
|
|
3651
|
+
resolve();
|
|
3652
|
+
};
|
|
3653
|
+
});
|
|
3654
|
+
await this.startListening(cancellation, () => cancelled);
|
|
3655
|
+
})().finally(() => {
|
|
3656
|
+
this.listening = null;
|
|
3657
|
+
this.cancelListen = null;
|
|
3658
|
+
});
|
|
3659
|
+
return this.listening;
|
|
3660
|
+
}
|
|
3661
|
+
async startListening(cancellation, cancelled) {
|
|
3532
3662
|
assertFeedHasAuthentication("sync", this.sync, this.options);
|
|
3533
3663
|
assertFeedHasAuthentication("changes", this.changes, this.options);
|
|
3534
3664
|
assertSecureBinding(this.host, this.options);
|
|
3535
3665
|
const uws = await loadUWebSockets();
|
|
3536
3666
|
this.uws = uws;
|
|
3537
3667
|
const app = uws.App();
|
|
3668
|
+
this.app = app;
|
|
3538
3669
|
await this.registerRoutes(app);
|
|
3670
|
+
if (cancelled()) return;
|
|
3539
3671
|
await new Promise((resolve, reject) => {
|
|
3540
3672
|
app.listen(this.host, this.port, (socket) => {
|
|
3541
3673
|
if (socket) {
|
|
@@ -3546,11 +3678,24 @@ var NarsilHttpServer = class {
|
|
|
3546
3678
|
}
|
|
3547
3679
|
});
|
|
3548
3680
|
});
|
|
3549
|
-
await this.deps.tasks.reconcile();
|
|
3681
|
+
await Promise.race([this.deps.tasks.reconcile(cancelled), cancellation]);
|
|
3682
|
+
if (cancelled()) return;
|
|
3550
3683
|
this.stopIdleHeapCleanup = startIdleHeapCleanup();
|
|
3551
3684
|
this.ready = true;
|
|
3552
3685
|
}
|
|
3553
|
-
|
|
3686
|
+
close() {
|
|
3687
|
+
if (this.closing !== null) return this.closing;
|
|
3688
|
+
this.cancelListen?.();
|
|
3689
|
+
const listening = this.listening;
|
|
3690
|
+
this.closing = (async () => {
|
|
3691
|
+
if (listening !== null) await listening.catch(() => void 0);
|
|
3692
|
+
await this.stopListening();
|
|
3693
|
+
})().then(() => {
|
|
3694
|
+
this.closing = null;
|
|
3695
|
+
});
|
|
3696
|
+
return this.closing;
|
|
3697
|
+
}
|
|
3698
|
+
async stopListening() {
|
|
3554
3699
|
this.ready = false;
|
|
3555
3700
|
this.stopIdleHeapCleanup();
|
|
3556
3701
|
this.stopIdleHeapCleanup = () => void 0;
|
|
@@ -3560,15 +3705,23 @@ var NarsilHttpServer = class {
|
|
|
3560
3705
|
}
|
|
3561
3706
|
this.feeds?.close();
|
|
3562
3707
|
this.feeds = null;
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3708
|
+
const app = this.app;
|
|
3709
|
+
const closingRequests = this.closingRequests;
|
|
3710
|
+
const threads = this.threads;
|
|
3711
|
+
this.app = null;
|
|
3712
|
+
this.closingRequests = null;
|
|
3713
|
+
this.threads = null;
|
|
3714
|
+
try {
|
|
3715
|
+
await Promise.all([closingRequests?.close(SERVER_CLOSE_DRAIN_DEADLINE_MS), threads?.stop()]);
|
|
3716
|
+
} finally {
|
|
3717
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
3718
|
+
app?.close();
|
|
3719
|
+
this.syncServing = null;
|
|
3568
3720
|
}
|
|
3569
3721
|
}
|
|
3570
3722
|
async shutdown() {
|
|
3571
3723
|
await this.close();
|
|
3724
|
+
await this.deps.tasks.shutdown();
|
|
3572
3725
|
await this.engine.shutdown();
|
|
3573
3726
|
}
|
|
3574
3727
|
get listeningPort() {
|
|
@@ -3589,8 +3742,9 @@ var NarsilHttpServer = class {
|
|
|
3589
3742
|
this.syncServing = started.syncServing;
|
|
3590
3743
|
this.feeds = started.host;
|
|
3591
3744
|
const { sync, changes } = started.host.gateways;
|
|
3592
|
-
|
|
3593
|
-
if (
|
|
3745
|
+
const accept = this.closingRequests?.accept;
|
|
3746
|
+
if (sync !== void 0) registerFeedRoute(app, sync, SYNC_FEED_ROUTE, accept);
|
|
3747
|
+
if (changes !== void 0) registerFeedRoute(app, changes, CHANGES_FEED_ROUTE, accept);
|
|
3594
3748
|
}
|
|
3595
3749
|
async registerRoutes(app) {
|
|
3596
3750
|
const cors = resolveCors(this.options.cors);
|
|
@@ -3601,14 +3755,19 @@ var NarsilHttpServer = class {
|
|
|
3601
3755
|
const bodyBudget = createBodyBudgetBuffer(gateSlotOfWorker(threadCount));
|
|
3602
3756
|
const { limits, build, allowRegex } = this.deps;
|
|
3603
3757
|
const handlers = mainThreadHandlers(this.deps, () => this.syncServing);
|
|
3758
|
+
const closingRequests = watchClosingRequests();
|
|
3759
|
+
this.closingRequests = closingRequests;
|
|
3604
3760
|
this.startFeeds(app, authorize, core);
|
|
3605
3761
|
const run = createRouteRunner({
|
|
3762
|
+
accept: closingRequests.accept,
|
|
3606
3763
|
authorize,
|
|
3607
3764
|
gate: createSharedRequestGate(gate, limits.maxConcurrentRequests, MAIN_THREAD_GATE_SLOT),
|
|
3608
3765
|
bodyBudget: createSharedBodyBudget(bodyBudget, limits.maxBodyBytesInFlight, MAIN_THREAD_GATE_SLOT),
|
|
3609
|
-
corsHeaders: cors ? corsHeaderReader(cors) : void 0
|
|
3766
|
+
corsHeaders: cors ? corsHeaderReader(cors) : void 0,
|
|
3767
|
+
respondThrough: closingRequests.respondThrough,
|
|
3768
|
+
track: closingRequests.track
|
|
3610
3769
|
});
|
|
3611
|
-
registerServerRoutes(app, run, handlers, limits, cors);
|
|
3770
|
+
registerServerRoutes(app, run, handlers, limits, cors, closingRequests.accept);
|
|
3612
3771
|
if (core === void 0 || threadCount === 0 || !detectRuntime().supportsWorkerThreads) return;
|
|
3613
3772
|
try {
|
|
3614
3773
|
this.threads = await startRequestThreads({
|
|
@@ -3633,4 +3792,170 @@ function createServer(engine, options) {
|
|
|
3633
3792
|
return new NarsilHttpServer(engine, options);
|
|
3634
3793
|
}
|
|
3635
3794
|
|
|
3636
|
-
|
|
3795
|
+
// src/server/persistence-task-store.ts
|
|
3796
|
+
var RECORD_KEY_SUFFIX = ".json";
|
|
3797
|
+
var encoder = new TextEncoder();
|
|
3798
|
+
var decoder = new TextDecoder("utf-8", { fatal: true });
|
|
3799
|
+
function keyOf(id) {
|
|
3800
|
+
return `${encodeURIComponent(id)}${RECORD_KEY_SUFFIX}`;
|
|
3801
|
+
}
|
|
3802
|
+
function idOfKey(key) {
|
|
3803
|
+
if (!key.endsWith(RECORD_KEY_SUFFIX)) return null;
|
|
3804
|
+
try {
|
|
3805
|
+
const id = decodeURIComponent(key.slice(0, -RECORD_KEY_SUFFIX.length));
|
|
3806
|
+
return keyOf(id) === key ? id : null;
|
|
3807
|
+
} catch {
|
|
3808
|
+
return null;
|
|
3809
|
+
}
|
|
3810
|
+
}
|
|
3811
|
+
function isStoredRecord(value, id) {
|
|
3812
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
|
3813
|
+
const record = value;
|
|
3814
|
+
return record.id === id && isTaskType(record.type) && isTaskStatus(record.status) && typeof record.indexName === "string" && typeof record.owner === "string" && typeof record.createdAt === "number" && Number.isFinite(record.createdAt);
|
|
3815
|
+
}
|
|
3816
|
+
function byOldestFirst(left, right) {
|
|
3817
|
+
if (left.createdAt !== right.createdAt) return left.createdAt - right.createdAt;
|
|
3818
|
+
return left.id < right.id ? -1 : left.id > right.id ? 1 : 0;
|
|
3819
|
+
}
|
|
3820
|
+
var PersistenceTaskStore = class {
|
|
3821
|
+
/**
|
|
3822
|
+
* Builds a store over an adapter that it owns alone.
|
|
3823
|
+
*
|
|
3824
|
+
* @param adapter - The adapter that holds the records, such as one from
|
|
3825
|
+
* `createFilesystemPersistence` scoped to a folder that holds nothing else.
|
|
3826
|
+
* The store deletes every key of the adapter that holds no task record.
|
|
3827
|
+
* @param maxRetained - Records to keep before the oldest finished ones are
|
|
3828
|
+
* dropped. Defaults to 1000.
|
|
3829
|
+
*/
|
|
3830
|
+
constructor(adapter, maxRetained = DEFAULT_MAX_RETAINED_TASKS) {
|
|
3831
|
+
this.adapter = adapter;
|
|
3832
|
+
this.maxRetained = maxRetained;
|
|
3833
|
+
}
|
|
3834
|
+
records = /* @__PURE__ */ new Map();
|
|
3835
|
+
pendingWrites = /* @__PURE__ */ new Map();
|
|
3836
|
+
loading = null;
|
|
3837
|
+
/**
|
|
3838
|
+
* Stores a record and writes it through the adapter, replacing whatever its
|
|
3839
|
+
* id already held.
|
|
3840
|
+
*
|
|
3841
|
+
* @param record - The record to store. A copy is kept, so changing the
|
|
3842
|
+
* object afterwards leaves the store untouched.
|
|
3843
|
+
* @throws The adapter's error where the write fails. The store still
|
|
3844
|
+
* answers reads with the new record, and the next write of that id tries
|
|
3845
|
+
* again.
|
|
3846
|
+
*/
|
|
3847
|
+
async set(record) {
|
|
3848
|
+
const id = record.id;
|
|
3849
|
+
const json = JSON.stringify(record);
|
|
3850
|
+
await this.ensureLoaded();
|
|
3851
|
+
this.records.set(id, JSON.parse(json));
|
|
3852
|
+
const written = this.persist(id, json);
|
|
3853
|
+
this.prune();
|
|
3854
|
+
await written;
|
|
3855
|
+
}
|
|
3856
|
+
/**
|
|
3857
|
+
* Reads one record back.
|
|
3858
|
+
*
|
|
3859
|
+
* @param id - The task to read.
|
|
3860
|
+
* @returns A copy of the record, or `null` when the store holds no such id.
|
|
3861
|
+
*/
|
|
3862
|
+
async get(id) {
|
|
3863
|
+
await this.ensureLoaded();
|
|
3864
|
+
const record = this.records.get(id);
|
|
3865
|
+
return record ? structuredClone(record) : null;
|
|
3866
|
+
}
|
|
3867
|
+
/**
|
|
3868
|
+
* Lists every record the store holds.
|
|
3869
|
+
*
|
|
3870
|
+
* @returns A copy of each record, oldest first.
|
|
3871
|
+
*/
|
|
3872
|
+
async list() {
|
|
3873
|
+
await this.ensureLoaded();
|
|
3874
|
+
return [...this.records.values()].map((record) => structuredClone(record));
|
|
3875
|
+
}
|
|
3876
|
+
/**
|
|
3877
|
+
* Removes one record and deletes its key. An id the store never held is not
|
|
3878
|
+
* an error.
|
|
3879
|
+
*
|
|
3880
|
+
* @param id - The task to remove.
|
|
3881
|
+
*/
|
|
3882
|
+
async delete(id) {
|
|
3883
|
+
await this.ensureLoaded();
|
|
3884
|
+
this.records.delete(id);
|
|
3885
|
+
await this.persist(id, null);
|
|
3886
|
+
}
|
|
3887
|
+
/** Resolves once every write that the store has started reaches the adapter, failed or not. */
|
|
3888
|
+
async shutdown() {
|
|
3889
|
+
await this.loading;
|
|
3890
|
+
while (this.pendingWrites.size > 0) {
|
|
3891
|
+
await Promise.allSettled([...this.pendingWrites.values()]);
|
|
3892
|
+
}
|
|
3893
|
+
}
|
|
3894
|
+
ensureLoaded() {
|
|
3895
|
+
if (this.loading === null) {
|
|
3896
|
+
this.loading = this.load().catch((err) => {
|
|
3897
|
+
this.loading = null;
|
|
3898
|
+
throw err;
|
|
3899
|
+
});
|
|
3900
|
+
}
|
|
3901
|
+
return this.loading;
|
|
3902
|
+
}
|
|
3903
|
+
async load() {
|
|
3904
|
+
const keys2 = await this.adapter.list("");
|
|
3905
|
+
const loaded = [];
|
|
3906
|
+
const malformed = [];
|
|
3907
|
+
let next = 0;
|
|
3908
|
+
const readNext = async () => {
|
|
3909
|
+
while (next < keys2.length) {
|
|
3910
|
+
const key = keys2[next];
|
|
3911
|
+
next += 1;
|
|
3912
|
+
const outcome = await this.loadKey(key);
|
|
3913
|
+
if (outcome.kind === "record") loaded.push(outcome.record);
|
|
3914
|
+
else if (outcome.kind === "malformed") malformed.push(key);
|
|
3915
|
+
}
|
|
3916
|
+
};
|
|
3917
|
+
await Promise.all(Array.from({ length: Math.min(TASK_RECORD_LOAD_CONCURRENCY, keys2.length) }, readNext));
|
|
3918
|
+
loaded.sort(byOldestFirst);
|
|
3919
|
+
for (const record of loaded) this.records.set(record.id, record);
|
|
3920
|
+
await Promise.allSettled(malformed.map((key) => this.adapter.delete(key)));
|
|
3921
|
+
this.prune();
|
|
3922
|
+
}
|
|
3923
|
+
async loadKey(key) {
|
|
3924
|
+
const id = idOfKey(key);
|
|
3925
|
+
if (id === null) return { kind: "malformed" };
|
|
3926
|
+
let bytes;
|
|
3927
|
+
try {
|
|
3928
|
+
bytes = await this.adapter.load(key);
|
|
3929
|
+
} catch {
|
|
3930
|
+
return { kind: "unreadable" };
|
|
3931
|
+
}
|
|
3932
|
+
if (bytes === null) return { kind: "unreadable" };
|
|
3933
|
+
try {
|
|
3934
|
+
const parsed = JSON.parse(decoder.decode(bytes));
|
|
3935
|
+
return isStoredRecord(parsed, id) ? { kind: "record", record: parsed } : { kind: "malformed" };
|
|
3936
|
+
} catch {
|
|
3937
|
+
return { kind: "malformed" };
|
|
3938
|
+
}
|
|
3939
|
+
}
|
|
3940
|
+
persist(id, json) {
|
|
3941
|
+
const key = keyOf(id);
|
|
3942
|
+
const previous = this.pendingWrites.get(id) ?? Promise.resolve();
|
|
3943
|
+
const write = previous.catch(() => void 0).then(() => json === null ? this.adapter.delete(key) : this.adapter.save(key, encoder.encode(json)));
|
|
3944
|
+
this.pendingWrites.set(id, write);
|
|
3945
|
+
write.finally(() => {
|
|
3946
|
+
if (this.pendingWrites.get(id) === write) this.pendingWrites.delete(id);
|
|
3947
|
+
}).catch(() => void 0);
|
|
3948
|
+
return write;
|
|
3949
|
+
}
|
|
3950
|
+
prune() {
|
|
3951
|
+
if (this.records.size <= this.maxRetained) return;
|
|
3952
|
+
for (const [id, record] of this.records) {
|
|
3953
|
+
if (this.records.size <= this.maxRetained) break;
|
|
3954
|
+
if (!isTerminalTaskStatus(record.status)) continue;
|
|
3955
|
+
this.records.delete(id);
|
|
3956
|
+
this.persist(id, null).catch(() => void 0);
|
|
3957
|
+
}
|
|
3958
|
+
}
|
|
3959
|
+
};
|
|
3960
|
+
|
|
3961
|
+
export { InMemoryTaskStore, PersistenceTaskStore, createServer };
|