@delali/narsil 0.3.1-next.89 → 0.3.1-next.91

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/browser/{chunk-JACCR3BJ.mjs → chunk-66KYAOIK.mjs} +1 -1
  2. package/dist/browser/{chunk-3GZQKGVQ.mjs → chunk-B6XE7KXY.mjs} +11 -3
  3. package/dist/browser/{chunk-RAHUQOJL.mjs → chunk-NOUZ6RH3.mjs} +2219 -2005
  4. package/dist/browser/{durability-wiring-VSUBLUDK.mjs → durability-wiring-F4OGQUSU.mjs} +3 -3
  5. package/dist/browser/{snapshot-5MXBEDXF.mjs → snapshot-5EJ7I7QY.mjs} +12 -6
  6. package/dist/browser/{write-6SBYTS62.mjs → write-K5V4XQ72.mjs} +2 -2
  7. package/dist/{capabilities-SJPttcgQ.d.ts → capabilities-CvExBThq.d.ts} +11 -3
  8. package/dist/{chunk-MWJIAREY.mjs → chunk-3RDS4UHC.mjs} +2 -2
  9. package/dist/{chunk-QAMUC2BJ.mjs → chunk-3TTYK5A7.mjs} +7 -6
  10. package/dist/{chunk-URGLSNKR.mjs → chunk-43ORORN6.mjs} +201 -15
  11. package/dist/{chunk-A3UB2MXL.mjs → chunk-5ER245FI.mjs} +2 -505
  12. package/dist/{chunk-VY4DXZ5P.mjs → chunk-67Z2V6TC.mjs} +1 -1
  13. package/dist/{chunk-EMXAYPTQ.mjs → chunk-6VZ3Q73N.mjs} +2265 -3461
  14. package/dist/{chunk-S4LQDSUL.mjs → chunk-7OQHSDK6.mjs} +3 -3
  15. package/dist/chunk-7X5JST5L.mjs +114 -0
  16. package/dist/{chunk-R6BMMQXH.mjs → chunk-A4G7JSYO.mjs} +106 -41
  17. package/dist/{chunk-DCK4P5BI.mjs → chunk-BJ2YKRNX.mjs} +16 -7
  18. package/dist/{chunk-J2BPXTD7.mjs → chunk-CO4MRKST.mjs} +1 -1
  19. package/dist/{chunk-CL4OM7GB.mjs → chunk-EONIHG3V.mjs} +8 -4
  20. package/dist/chunk-MKWUXWQH.mjs +35 -0
  21. package/dist/chunk-NVL4ZR3K.mjs +426 -0
  22. package/dist/{chunk-GCM3JFVA.mjs → chunk-OHP7QR33.mjs} +2 -1
  23. package/dist/{chunk-YRX6XMOQ.mjs → chunk-R6PR4WYZ.mjs} +1 -0
  24. package/dist/{chunk-OOSIDHA3.mjs → chunk-RSRVGAIQ.mjs} +1 -1
  25. package/dist/{chunk-V7YUSKKZ.mjs → chunk-T4DB5YFB.mjs} +7 -3
  26. package/dist/{chunk-XSZVS4S3.mjs → chunk-UXQE2ROB.mjs} +6 -2
  27. package/dist/chunk-X5CV2E2B.mjs +1 -0
  28. package/dist/chunk-XLJ2S2BH.mjs +3572 -0
  29. package/dist/chunk-ZIJ63TBT.mjs +507 -0
  30. package/dist/{client-BMK53IR2.d.ts → client-Cc42b42j.d.ts} +27 -3
  31. package/dist/client.d.ts +9 -8
  32. package/dist/client.mjs +15 -2
  33. package/dist/{config-BUR-7eVQ.d.ts → config-BYne-yCI.d.ts} +2 -2
  34. package/dist/{config-UJx1dyX1.d.ts → config-DyRowvVP.d.ts} +2 -2
  35. package/dist/{direct-executor-EGESD26S.mjs → direct-executor-CHXVCBN7.mjs} +9 -5
  36. package/dist/distribution/coordinator/etcd.d.ts +3 -2
  37. package/dist/distribution/coordinator/etcd.mjs +4 -0
  38. package/dist/distribution/coordinator/in-memory.d.ts +3 -2
  39. package/dist/distribution/coordinator/in-memory.mjs +4 -0
  40. package/dist/distribution/transport/grpc.mjs +2 -2
  41. package/dist/distribution/transport/in-memory.mjs +1 -1
  42. package/dist/distribution/transport/tcp.mjs +2 -2
  43. package/dist/distribution.d.ts +43 -9
  44. package/dist/distribution.mjs +4590 -4263
  45. package/dist/{durability-wiring-ZLRQU6FG.mjs → durability-wiring-6AN4QTT3.mjs} +20 -16
  46. package/dist/{engine-9X6W5naj.d.ts → engine-96_kLtSY.d.ts} +43 -7
  47. package/dist/{index-vCMu8OeQ.d.ts → index-ONAEaqtx.d.ts} +8 -1
  48. package/dist/index.browser.mjs +1 -1
  49. package/dist/index.d.ts +44 -25
  50. package/dist/index.mjs +13 -9
  51. package/dist/{memory-CV9NAxLY.d.ts → memory-dd3GESdc.d.ts} +3 -3
  52. package/dist/partial-words.mjs +2 -1
  53. package/dist/persistence/durability/checkpoint-worker.d.ts +60 -58
  54. package/dist/persistence/durability/checkpoint-worker.mjs +18 -14
  55. package/dist/react.d.ts +8 -7
  56. package/dist/{schema-DtsgmNfU.d.ts → schema-DttzM-ln.d.ts} +12 -0
  57. package/dist/server.d.ts +11 -10
  58. package/dist/server.mjs +33 -6
  59. package/dist/{snapshot-22CUEYYW.mjs → snapshot-JFZVOGZV.mjs} +28 -18
  60. package/dist/synonyms-CpOAHPa-.d.ts +52 -0
  61. package/dist/synonyms.d.ts +22 -0
  62. package/dist/synonyms.mjs +12 -0
  63. package/dist/{tasks-BGaZJyoY.d.ts → tasks-DA1Ff0G4.d.ts} +4 -1
  64. package/dist/{thread-ZP3UMIZT.mjs → thread-VTDLLYXC.mjs} +7 -5
  65. package/dist/vector/search-worker.d.ts +2 -1
  66. package/dist/workers/entry.mjs +4 -4
  67. package/dist/{write-PXDKPG2R.mjs → write-OJR3H4BI.mjs} +14 -10
  68. package/package.json +15 -7
  69. package/dist/chunk-PR4O7CAP.mjs +0 -18
  70. package/dist/chunk-X4SEXL4U.mjs +0 -2310
@@ -1,7 +1,7 @@
1
1
  import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
2
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, readMetadataEnvelope, adapterCheckpointStore, readSegmentManifest, reclaimUnlistedKeys, snapshotBundleKey, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readCurrentManifest, DIRECTORY_LOCK_SETTLE_MS, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, SINGLE_NODE_PRIMARY_TERM, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, isId, isUint32, isSequenceNumber, readWalSegments, durableRecordMissing, growPartitionsTo, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, VECTOR_FILE_ENVELOPE_VERSION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry } from './chunk-3GZQKGVQ.mjs';
3
- import { applyDeleteEntry, applyIndexEntry } from './chunk-JACCR3BJ.mjs';
4
- import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-RAHUQOJL.mjs';
2
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, readMetadataEnvelope, adapterCheckpointStore, readSegmentManifest, reclaimUnlistedKeys, snapshotBundleKey, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readCurrentManifest, DIRECTORY_LOCK_SETTLE_MS, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, SINGLE_NODE_PRIMARY_TERM, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, isId, isUint32, isSequenceNumber, readWalSegments, durableRecordMissing, growPartitionsTo, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, VECTOR_FILE_ENVELOPE_VERSION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry } from './chunk-B6XE7KXY.mjs';
3
+ import { applyDeleteEntry, applyIndexEntry } from './chunk-66KYAOIK.mjs';
4
+ import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-NOUZ6RH3.mjs';
5
5
  import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-NQW2YXEX.mjs';
6
6
  import { NarsilError, ErrorCodes, compareCodePoints, detectRuntime, MIN_M, resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-FHFFWLEK.mjs';
7
7
  import { decode } from '@msgpack/msgpack';
@@ -1,7 +1,7 @@
1
1
  import { vectorsPerFile } from './chunk-3ZRMZIH4.mjs';
2
- import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, documentsOfPayloadV2, deserializePayloadV1, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts } from './chunk-3GZQKGVQ.mjs';
3
- import './chunk-JACCR3BJ.mjs';
4
- import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, setNestedValue, decodeIndexConfiguration, flattenSchema } from './chunk-RAHUQOJL.mjs';
2
+ import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, documentsOfPayloadV2, deserializePayloadV1, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts } from './chunk-B6XE7KXY.mjs';
3
+ import './chunk-66KYAOIK.mjs';
4
+ import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, loadVectorCodeFor, prepareStoredSynonyms, rejectSnapshot, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, setNestedValue, decodeIndexConfiguration, flattenSchema } from './chunk-NOUZ6RH3.mjs';
5
5
  import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, decodeVectorIndexParts, bytesToVectors } from './chunk-NQW2YXEX.mjs';
6
6
  import { NarsilError, ErrorCodes, MAX_FILE_BYTES, BATCH_CHUNK_SIZE } from './chunk-FHFFWLEK.mjs';
7
7
  import { decode } from '@msgpack/msgpack';
@@ -534,7 +534,12 @@ function prepareRestoredIndex(configuration, deps, reject) {
534
534
  }
535
535
  }
536
536
  const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
537
- return { config, language, embeddingAdapter, adapterName, storedRevision };
537
+ const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
538
+ return { config, language, embeddingAdapter, adapterName, storedRevision, synonyms };
539
+ }
540
+ function restoredSynonyms(raw, config, language, reject) {
541
+ if (raw === void 0 || raw === null) return null;
542
+ return prepareStoredSynonyms(raw, config, language, (message) => reject(`synonyms: ${message}`));
538
543
  }
539
544
  async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
540
545
  const loadingVectorCode = loadVectorCodeFor(prepared.config.schema);
@@ -551,7 +556,7 @@ async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
551
556
  currentRevision: prepared.language.revision
552
557
  });
553
558
  }
554
- deps.executor.createIndex(indexName, prepared.config, prepared.language);
559
+ deps.executor.createIndex(indexName, prepared.config, prepared.language, false, prepared.synonyms?.lookup ?? null);
555
560
  deps.indexRegistry.set(indexName, {
556
561
  config: prepared.config,
557
562
  language: prepared.language,
@@ -562,7 +567,8 @@ async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
562
567
  heldPartitions: null,
563
568
  documentCount: 0,
564
569
  partitionCount,
565
- analysisRevision: prepared.storedRevision
570
+ analysisRevision: prepared.storedRevision,
571
+ synonyms: prepared.synonyms?.encoded ?? null
566
572
  });
567
573
  await deps.indexState.registerOpen(indexName);
568
574
  const release = await deps.indexState.acquire(indexName, false);
@@ -1,5 +1,5 @@
1
- import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-JACCR3BJ.mjs';
2
- import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-RAHUQOJL.mjs';
1
+ import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-66KYAOIK.mjs';
2
+ import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-NOUZ6RH3.mjs';
3
3
  import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-NQW2YXEX.mjs';
4
4
  import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-FHFFWLEK.mjs';
5
5
  import { decode } from '@msgpack/msgpack';
@@ -1,6 +1,6 @@
1
1
  import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
2
- import { C as ClusterNamespace } from './config-UJx1dyX1.js';
3
- import { h as TaskStore } from './tasks-BGaZJyoY.js';
2
+ import { C as ClusterNamespace } from './config-DyRowvVP.js';
3
+ import { h as TaskStore } from './tasks-DA1Ff0G4.js';
4
4
 
5
5
  /**
6
6
  * Context handed to {@link OnRequestHook} for every inbound request, captured
@@ -284,6 +284,14 @@ declare const INDEX_LIFECYCLE_CAPABILITY = "indexes.lifecycle";
284
284
  * @public
285
285
  */
286
286
  declare const DOCUMENT_EXPORT_CAPABILITY = "indexes.documentExport";
287
+ /**
288
+ * Marks a server that reads the synonym set of an index at
289
+ * `GET /indexes/{name}/synonyms` and replaces it at
290
+ * `POST /indexes/{name}/synonyms`.
291
+ *
292
+ * @public
293
+ */
294
+ declare const SYNONYMS_CAPABILITY = "indexes.synonyms";
287
295
  /**
288
296
  * Every capability this server announces at `/capabilities`.
289
297
  *
@@ -295,4 +303,4 @@ declare const DOCUMENT_EXPORT_CAPABILITY = "indexes.documentExport";
295
303
  */
296
304
  declare const SERVER_CAPABILITIES: readonly string[];
297
305
 
298
- export { ASYNC_IMPORT_CAPABILITY as A, type CorsOptions as C, DOCUMENT_EXPORT_CAPABILITY as D, type ErrorEnvelope as E, INDEX_LIFECYCLE_CAPABILITY as I, type NarsilServer as N, type OnRequestHook as O, REBUILD_ANALYSIS_CAPABILITY as R, SERVER_CAPABILITIES as S, TASK_CANCEL_CAPABILITY as T, TASK_FILTER_CAPABILITY as a, type ServerOptions as b, type RequestContext as c, type RequestDenial as d, type ServerLimits as e };
306
+ export { ASYNC_IMPORT_CAPABILITY as A, type CorsOptions as C, DOCUMENT_EXPORT_CAPABILITY as D, type ErrorEnvelope as E, INDEX_LIFECYCLE_CAPABILITY as I, type NarsilServer as N, type OnRequestHook as O, REBUILD_ANALYSIS_CAPABILITY as R, SERVER_CAPABILITIES as S, TASK_CANCEL_CAPABILITY as T, SYNONYMS_CAPABILITY as a, TASK_FILTER_CAPABILITY as b, type ServerOptions as c, type RequestContext as d, type RequestDenial as e, type ServerLimits as f };
@@ -1,5 +1,5 @@
1
- import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-R6BMMQXH.mjs';
2
- import { getNestedValue } from './chunk-A3UB2MXL.mjs';
1
+ import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-A4G7JSYO.mjs';
2
+ import { getNestedValue } from './chunk-5ER245FI.mjs';
3
3
  import { requireValidFieldTypes } from './chunk-CVQWQVFX.mjs';
4
4
  import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
5
5
  import { ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
@@ -1,12 +1,13 @@
1
- import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-MWJIAREY.mjs';
2
- import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
1
+ import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-3RDS4UHC.mjs';
2
+ import { ReplicationMessageTypes } from './chunk-OHP7QR33.mjs';
3
3
  import { WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
4
- import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-R6BMMQXH.mjs';
5
- import { isCompositePartition, generateId, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-EMXAYPTQ.mjs';
6
- import { fnv1a } from './chunk-X4SEXL4U.mjs';
7
- import { getLanguage } from './chunk-PR4O7CAP.mjs';
4
+ import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-A4G7JSYO.mjs';
8
5
  import { MAX_FILE_BYTES } from './chunk-RVM7AODE.mjs';
6
+ import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-XLJ2S2BH.mjs';
7
+ import { fnv1a, isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-6VZ3Q73N.mjs';
8
+ import { generateId, getLanguage } from './chunk-MKWUXWQH.mjs';
9
9
  import { extractVectorFieldsFromSchema, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-CVQWQVFX.mjs';
10
+ import { resolveIndexAnalysis } from './chunk-NVL4ZR3K.mjs';
10
11
  import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
11
12
  import { encode, decode } from '@msgpack/msgpack';
12
13
 
@@ -1,11 +1,142 @@
1
1
  import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-P36MUFEJ.mjs';
2
- import { readSharedConfigFields, restoredEmbedding, restoredPartitionConfig } from './chunk-J2BPXTD7.mjs';
3
- import { getLanguage } from './chunk-PR4O7CAP.mjs';
2
+ import { readSharedConfigFields, restoredEmbedding, restoredPartitionConfig } from './chunk-CO4MRKST.mjs';
3
+ import { generateId, getLanguage } from './chunk-MKWUXWQH.mjs';
4
4
  import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-Q5DC6WAB.mjs';
5
+ import { decodeSynonymRules } from './chunk-7X5JST5L.mjs';
5
6
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
6
7
  import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
7
8
  import { encode, decode } from '@msgpack/msgpack';
8
9
 
10
+ // src/distribution/cluster/constants.ts
11
+ var LEADERSHIP_IMBALANCE_THRESHOLD = 2;
12
+ var ESTIMATED_PARTITION_BYTES = 50 * 1024 * 1024;
13
+ var REBALANCE_THRESHOLD = 0.1;
14
+ var BOOTSTRAP_CAS_ATTEMPTS = 5;
15
+ var ALLOCATION_CAS_ATTEMPTS = 5;
16
+ var ALLOCATION_RETRY_DELAY_MS = 1e3;
17
+ var TEARDOWN_CAS_ATTEMPTS = 5;
18
+ var TEARDOWN_RETRY_DELAY_MS = 500;
19
+ var TEARDOWN_RETRY_ROUNDS = 3;
20
+ var RECOVERY_CAS_ATTEMPTS = 5;
21
+ var PARTITION_STORES_TIMEOUT_MS = 5e3;
22
+ var DEFAULT_CONTROLLER_CONFIG = {
23
+ leaseTtlMs: 15e3,
24
+ standbyRetryMs: 5e3
25
+ };
26
+ var DEFAULT_NODE_LIFECYCLE_CONFIG = {
27
+ bootstrapRetryBaseMs: 1e3,
28
+ bootstrapRetryMaxMs: 5e3,
29
+ bootstrapMaxRetries: 10,
30
+ allocationDebounceMs: 250,
31
+ nodeHeartbeatIntervalMs: 1e4
32
+ };
33
+ var MAX_SYNONYM_GENERATION_LENGTH = 128;
34
+
35
+ // src/distribution/payload-guards.ts
36
+ function isRecord(value) {
37
+ return value !== null && typeof value === "object" && !Array.isArray(value);
38
+ }
39
+ function isValidInteger(value) {
40
+ return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
41
+ }
42
+
43
+ // src/distribution/cluster/synonym-set.ts
44
+ var SYNONYM_CHUNK_BYTES = 1048576;
45
+ var SYNONYM_SET_PREFIX = "_narsil/synonyms/";
46
+ var MAX_UINT32 = 4294967295;
47
+ var DIGEST_PATTERN = /^[0-9a-f]{64}$/;
48
+ var FIRST_PRINTABLE_CODE = 32;
49
+ function isKeySegment(value) {
50
+ for (let i = 0; i < value.length; i++) {
51
+ const code = value.charCodeAt(i);
52
+ if (code < FIRST_PRINTABLE_CODE || value[i] === "/" || value[i] === "\\") return false;
53
+ }
54
+ return true;
55
+ }
56
+ function synonymChunkKey(indexUuid, generation, chunk) {
57
+ return `${SYNONYM_SET_PREFIX}${indexUuid}/${generation}/${chunk}`;
58
+ }
59
+ function isUint32(value) {
60
+ return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= MAX_UINT32;
61
+ }
62
+ async function digestOf(bytes) {
63
+ const view = bytes.buffer instanceof ArrayBuffer ? new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) : Uint8Array.from(bytes);
64
+ const hash = new Uint8Array(await crypto.subtle.digest("SHA-256", view));
65
+ return Array.from(hash, (byte) => byte.toString(16).padStart(2, "0")).join("");
66
+ }
67
+ function synonymSetRefOf(raw) {
68
+ if (!isRecord(raw)) return "synonyms must be an object";
69
+ const { generation, digest, chunkCount, byteLength } = raw;
70
+ if (typeof generation !== "string" || generation.length === 0 || generation.length > MAX_SYNONYM_GENERATION_LENGTH || !isKeySegment(generation)) {
71
+ return "synonyms.generation must hold 1 to 128 characters, with no control character, slash, or backslash";
72
+ }
73
+ if (typeof digest !== "string" || !DIGEST_PATTERN.test(digest)) {
74
+ return "synonyms.digest must be a lowercase hexadecimal SHA-256";
75
+ }
76
+ if (!isUint32(byteLength) || byteLength === 0) {
77
+ return "synonyms.byteLength must be a positive whole number";
78
+ }
79
+ if (!isUint32(chunkCount) || chunkCount !== Math.ceil(byteLength / SYNONYM_CHUNK_BYTES)) {
80
+ return "synonyms.chunkCount must match synonyms.byteLength";
81
+ }
82
+ return { generation, digest, chunkCount, byteLength };
83
+ }
84
+ function decodeSynonymSetRef(raw, reject) {
85
+ const ref = synonymSetRefOf(raw);
86
+ return typeof ref === "string" ? reject(ref) : ref;
87
+ }
88
+ function synonymSetRefOrNull(raw) {
89
+ const ref = synonymSetRefOf(raw);
90
+ return typeof ref === "string" ? null : ref;
91
+ }
92
+ async function deleteSynonymSet(coordinator, indexUuid, ref) {
93
+ for (let chunk = 0; chunk < ref.chunkCount; chunk++) {
94
+ await coordinator.delete(synonymChunkKey(indexUuid, ref.generation, chunk));
95
+ }
96
+ }
97
+ async function storeSynonymSet(coordinator, indexUuid, encoded) {
98
+ const ref = {
99
+ generation: generateId(),
100
+ digest: await digestOf(encoded),
101
+ chunkCount: Math.ceil(encoded.byteLength / SYNONYM_CHUNK_BYTES),
102
+ byteLength: encoded.byteLength
103
+ };
104
+ try {
105
+ for (let chunk = 0; chunk < ref.chunkCount; chunk++) {
106
+ const start = chunk * SYNONYM_CHUNK_BYTES;
107
+ const bytes = encoded.subarray(start, Math.min(start + SYNONYM_CHUNK_BYTES, encoded.byteLength));
108
+ if (!await coordinator.compareAndSet(synonymChunkKey(indexUuid, ref.generation, chunk), null, bytes)) {
109
+ throw new NarsilError(
110
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
111
+ `Synonym set chunk ${chunk} of a fresh copy already exists in the coordinator`,
112
+ { indexUuid, generation: ref.generation, chunk }
113
+ );
114
+ }
115
+ }
116
+ } catch (err) {
117
+ await deleteSynonymSet(coordinator, indexUuid, ref).catch(() => void 0);
118
+ throw err;
119
+ }
120
+ return ref;
121
+ }
122
+ async function readSynonymSet(coordinator, indexUuid, ref, reject) {
123
+ const encoded = new Uint8Array(ref.byteLength);
124
+ let filled = 0;
125
+ for (let chunk = 0; chunk < ref.chunkCount; chunk++) {
126
+ const bytes = await coordinator.get(synonymChunkKey(indexUuid, ref.generation, chunk));
127
+ if (bytes === null) return null;
128
+ if (bytes.byteLength > encoded.byteLength - filled) return reject("the stored synonym set exceeds its byteLength");
129
+ encoded.set(bytes, filled);
130
+ filled += bytes.byteLength;
131
+ }
132
+ if (filled !== ref.byteLength) return reject("the stored synonym set falls short of its byteLength");
133
+ if (await digestOf(encoded) !== ref.digest) return reject("the stored synonym set does not match its digest");
134
+ return encoded;
135
+ }
136
+ function storedSynonymSetRules(encoded, reject) {
137
+ return decodeSynonymRules(encoded, (message) => reject(message));
138
+ }
139
+
9
140
  // src/distribution/cluster/index-settings.ts
10
141
  function partitionLimitsOf(partitions) {
11
142
  if (partitions === void 0) {
@@ -71,7 +202,7 @@ function indexSettingsOf(config) {
71
202
  ...settingsFieldsOf(config)
72
203
  };
73
204
  }
74
- function isRecord(value) {
205
+ function isRecord2(value) {
75
206
  return value !== null && typeof value === "object" && !Array.isArray(value);
76
207
  }
77
208
  function settingsRejection(indexName) {
@@ -88,7 +219,7 @@ function decodeIndexSettings(raw, indexName) {
88
219
  return void 0;
89
220
  }
90
221
  const reject = settingsRejection(indexName);
91
- if (!isRecord(raw)) {
222
+ if (!isRecord2(raw)) {
92
223
  return reject("settings must be an object");
93
224
  }
94
225
  if (typeof raw.language !== "string" || raw.language.length === 0) {
@@ -96,13 +227,15 @@ function decodeIndexSettings(raw, indexName) {
96
227
  }
97
228
  const embedding = restoredEmbedding(raw.embedding, reject);
98
229
  const partitions = restoredPartitionConfig(raw.partitionConfig, reject);
230
+ const synonyms = raw.synonyms === void 0 ? void 0 : decodeSynonymSetRef(raw.synonyms, reject);
99
231
  return {
100
232
  language: raw.language,
101
233
  ...settingsFieldsOf({
102
234
  ...readSharedConfigFields(raw, reject),
103
235
  ...partitions !== void 0 ? { partitions } : {},
104
236
  ...embedding !== void 0 ? { embedding } : {}
105
- })
237
+ }),
238
+ ...synonyms !== void 0 ? { synonyms } : {}
106
239
  };
107
240
  }
108
241
  function withPartitionCount(config, partitionCount) {
@@ -112,7 +245,7 @@ function indexConfigFromSettings(schema, settings, indexName) {
112
245
  if (settings === void 0) {
113
246
  return { schema };
114
247
  }
115
- const { language, embedding, partitionConfig, ...fields } = settings;
248
+ const { language, embedding, partitionConfig, synonyms: _synonyms, ...fields } = settings;
116
249
  return {
117
250
  schema,
118
251
  language,
@@ -153,14 +286,14 @@ function validateIndexName(indexName) {
153
286
  );
154
287
  }
155
288
  }
156
- function isValidInteger(value) {
289
+ function isValidInteger2(value) {
157
290
  return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
158
291
  }
159
292
  function indexConfigKey(indexName) {
160
293
  return `${INDEX_CONFIG_PREFIX}${indexName}${INDEX_CONFIG_SUFFIX}`;
161
294
  }
162
295
  function validateDecodedMetadata(decoded, indexName) {
163
- if (!isRecord(decoded)) {
296
+ if (!isRecord2(decoded)) {
164
297
  throw new NarsilError(
165
298
  ErrorCodes.CONTROLLER_METADATA_INVALID,
166
299
  `Index metadata for '${indexName}' is not an object`,
@@ -181,21 +314,21 @@ function validateDecodedMetadata(decoded, indexName) {
181
314
  { indexName, received: truncateForDisplay(decoded.indexName) }
182
315
  );
183
316
  }
184
- if (!isValidInteger(decoded.partitionCount) || decoded.partitionCount <= 0 || decoded.partitionCount > MAX_PARTITION_COUNT) {
317
+ if (!isValidInteger2(decoded.partitionCount) || decoded.partitionCount <= 0 || decoded.partitionCount > MAX_PARTITION_COUNT) {
185
318
  throw new NarsilError(
186
319
  ErrorCodes.CONTROLLER_METADATA_INVALID,
187
320
  `Index metadata for '${indexName}' has invalid partitionCount (must be an integer between 1 and ${MAX_PARTITION_COUNT})`,
188
321
  { indexName, partitionCount: truncateForDisplay(decoded.partitionCount) }
189
322
  );
190
323
  }
191
- if (!isValidInteger(decoded.replicationFactor) || decoded.replicationFactor < 0 || decoded.replicationFactor > MAX_REPLICATION_FACTOR) {
324
+ if (!isValidInteger2(decoded.replicationFactor) || decoded.replicationFactor < 0 || decoded.replicationFactor > MAX_REPLICATION_FACTOR) {
192
325
  throw new NarsilError(
193
326
  ErrorCodes.CONTROLLER_METADATA_INVALID,
194
327
  `Index metadata for '${indexName}' has invalid replicationFactor (must be an integer between 0 and ${MAX_REPLICATION_FACTOR})`,
195
328
  { indexName, replicationFactor: truncateForDisplay(decoded.replicationFactor) }
196
329
  );
197
330
  }
198
- if (!isRecord(decoded.constraints)) {
331
+ if (!isRecord2(decoded.constraints)) {
199
332
  throw new NarsilError(
200
333
  ErrorCodes.CONTROLLER_METADATA_INVALID,
201
334
  `Index metadata for '${indexName}' has invalid constraints`,
@@ -244,14 +377,67 @@ async function readStoredMetadata(coordinator, indexName) {
244
377
  }
245
378
  const decoded = decode(raw);
246
379
  const record = validateDecodedMetadata(decoded, indexName);
247
- return { record, rawSettings: isRecord(decoded) ? decoded.settings : void 0 };
380
+ return { bytes: raw, record, rawSettings: isRecord2(decoded) ? decoded.settings : void 0 };
381
+ }
382
+ async function readIndexMetadataSnapshot(coordinator, indexName) {
383
+ const stored = await readStoredMetadata(coordinator, indexName);
384
+ if (stored === null) return null;
385
+ return { bytes: stored.bytes, record: stored.record, settings: decodeIndexSettings(stored.rawSettings, indexName) };
386
+ }
387
+ function metadataWithSynonymSet(snapshot, ref) {
388
+ const decoded = decode(snapshot.bytes);
389
+ const fields = isRecord2(decoded) ? decoded : {};
390
+ const { synonyms: _replaced, ...settings } = isRecord2(fields.settings) ? fields.settings : { language: snapshot.settings?.language ?? getLanguage("english").name };
391
+ const nextSettings = ref === null ? settings : { ...settings, synonyms: ref };
392
+ return new Uint8Array(encode({ ...fields, settings: nextSettings }, { ignoreUndefined: true }));
248
393
  }
249
394
  async function getIndexMetadata(coordinator, indexName) {
250
395
  return (await readStoredMetadata(coordinator, indexName))?.record ?? null;
251
396
  }
397
+ async function readIndexState(coordinator, indexName) {
398
+ const reject = settingsRejection(indexName);
399
+ let stored = await readStoredMetadata(coordinator, indexName);
400
+ while (stored !== null) {
401
+ const { record } = stored;
402
+ const settings = decodeIndexSettings(stored.rawSettings, indexName);
403
+ const ref = settings?.synonyms;
404
+ if (ref === void 0) return { record, settings, synonyms: [] };
405
+ const encoded = await readSynonymSet(coordinator, record.indexUuid, ref, reject);
406
+ if (encoded !== null) return { record, settings, synonyms: storedSynonymSetRules(encoded, reject) };
407
+ stored = await readStoredMetadata(coordinator, indexName);
408
+ if (stored !== null && referencesSameSet(stored, record.indexUuid, ref, indexName)) {
409
+ return reject(`a chunk of synonym set ${ref.generation} is missing`);
410
+ }
411
+ }
412
+ return null;
413
+ }
414
+ function referencesSameSet(stored, indexUuid, ref, indexName) {
415
+ if (stored.record.indexUuid !== indexUuid) return false;
416
+ return decodeIndexSettings(stored.rawSettings, indexName)?.synonyms?.generation === ref.generation;
417
+ }
252
418
  async function getClusterIndexConfig(coordinator, indexName, schema) {
253
- const stored = await readStoredMetadata(coordinator, indexName);
254
- return indexConfigFromSettings(schema, decodeIndexSettings(stored?.rawSettings, indexName), indexName);
419
+ const state = await readIndexState(coordinator, indexName);
420
+ const config = indexConfigFromSettings(schema, state?.settings, indexName);
421
+ return state === null || state.synonyms.length === 0 ? config : { ...config, synonyms: state.synonyms };
422
+ }
423
+ function decodedMetadataOf(bytes) {
424
+ let decoded;
425
+ try {
426
+ decoded = decode(bytes);
427
+ } catch {
428
+ return null;
429
+ }
430
+ return isRecord2(decoded) ? decoded : null;
431
+ }
432
+ function indexUuidOfStoredMetadata(bytes) {
433
+ const decoded = decodedMetadataOf(bytes);
434
+ return typeof decoded?.indexUuid === "string" ? decoded.indexUuid : null;
435
+ }
436
+ function storedSynonymSetOf(bytes) {
437
+ const decoded = decodedMetadataOf(bytes);
438
+ if (decoded === null || typeof decoded.indexUuid !== "string" || !isRecord2(decoded.settings)) return null;
439
+ const ref = synonymSetRefOrNull(decoded.settings.synonyms);
440
+ return ref === null ? null : { indexUuid: decoded.indexUuid, ref };
255
441
  }
256
442
 
257
- export { getClusterIndexConfig, getIndexMetadata, indexConfigKey, indexSettingsOf, putIndexMetadata, validateIndexName, withPartitionCount };
443
+ export { ALLOCATION_CAS_ATTEMPTS, ALLOCATION_RETRY_DELAY_MS, BOOTSTRAP_CAS_ATTEMPTS, DEFAULT_CONTROLLER_CONFIG, DEFAULT_NODE_LIFECYCLE_CONFIG, ESTIMATED_PARTITION_BYTES, LEADERSHIP_IMBALANCE_THRESHOLD, PARTITION_STORES_TIMEOUT_MS, REBALANCE_THRESHOLD, RECOVERY_CAS_ATTEMPTS, TEARDOWN_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, TEARDOWN_RETRY_ROUNDS, deleteSynonymSet, getClusterIndexConfig, getIndexMetadata, indexConfigFromSettings, indexConfigKey, indexSettingsOf, indexUuidOfStoredMetadata, isRecord, isValidInteger, metadataWithSynonymSet, putIndexMetadata, readIndexMetadataSnapshot, readIndexState, storeSynonymSet, storedSynonymSetOf, validateIndexName, withPartitionCount };