@delali/narsil 0.3.1-next.89 → 0.3.1-next.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-JACCR3BJ.mjs → chunk-66KYAOIK.mjs} +1 -1
- package/dist/browser/{chunk-3GZQKGVQ.mjs → chunk-B6XE7KXY.mjs} +11 -3
- package/dist/browser/{chunk-RAHUQOJL.mjs → chunk-NOUZ6RH3.mjs} +2219 -2005
- package/dist/browser/{durability-wiring-VSUBLUDK.mjs → durability-wiring-F4OGQUSU.mjs} +3 -3
- package/dist/browser/{snapshot-5MXBEDXF.mjs → snapshot-5EJ7I7QY.mjs} +12 -6
- package/dist/browser/{write-6SBYTS62.mjs → write-K5V4XQ72.mjs} +2 -2
- package/dist/{capabilities-SJPttcgQ.d.ts → capabilities-CvExBThq.d.ts} +11 -3
- package/dist/{chunk-MWJIAREY.mjs → chunk-3RDS4UHC.mjs} +2 -2
- package/dist/{chunk-QAMUC2BJ.mjs → chunk-3TTYK5A7.mjs} +7 -6
- package/dist/{chunk-URGLSNKR.mjs → chunk-43ORORN6.mjs} +201 -15
- package/dist/{chunk-A3UB2MXL.mjs → chunk-5ER245FI.mjs} +2 -505
- package/dist/{chunk-VY4DXZ5P.mjs → chunk-67Z2V6TC.mjs} +1 -1
- package/dist/{chunk-EMXAYPTQ.mjs → chunk-6VZ3Q73N.mjs} +2265 -3461
- package/dist/{chunk-S4LQDSUL.mjs → chunk-7OQHSDK6.mjs} +3 -3
- package/dist/chunk-7X5JST5L.mjs +114 -0
- package/dist/{chunk-R6BMMQXH.mjs → chunk-A4G7JSYO.mjs} +106 -41
- package/dist/{chunk-DCK4P5BI.mjs → chunk-BJ2YKRNX.mjs} +16 -7
- package/dist/{chunk-J2BPXTD7.mjs → chunk-CO4MRKST.mjs} +1 -1
- package/dist/{chunk-CL4OM7GB.mjs → chunk-EONIHG3V.mjs} +8 -4
- package/dist/chunk-MKWUXWQH.mjs +35 -0
- package/dist/chunk-NVL4ZR3K.mjs +426 -0
- package/dist/{chunk-GCM3JFVA.mjs → chunk-OHP7QR33.mjs} +2 -1
- package/dist/{chunk-YRX6XMOQ.mjs → chunk-R6PR4WYZ.mjs} +1 -0
- package/dist/{chunk-OOSIDHA3.mjs → chunk-RSRVGAIQ.mjs} +1 -1
- package/dist/{chunk-V7YUSKKZ.mjs → chunk-T4DB5YFB.mjs} +7 -3
- package/dist/{chunk-XSZVS4S3.mjs → chunk-UXQE2ROB.mjs} +6 -2
- package/dist/chunk-X5CV2E2B.mjs +1 -0
- package/dist/chunk-XLJ2S2BH.mjs +3572 -0
- package/dist/chunk-ZIJ63TBT.mjs +507 -0
- package/dist/{client-BMK53IR2.d.ts → client-Cc42b42j.d.ts} +27 -3
- package/dist/client.d.ts +9 -8
- package/dist/client.mjs +15 -2
- package/dist/{config-BUR-7eVQ.d.ts → config-BYne-yCI.d.ts} +2 -2
- package/dist/{config-UJx1dyX1.d.ts → config-DyRowvVP.d.ts} +2 -2
- package/dist/{direct-executor-EGESD26S.mjs → direct-executor-CHXVCBN7.mjs} +9 -5
- package/dist/distribution/coordinator/etcd.d.ts +3 -2
- package/dist/distribution/coordinator/etcd.mjs +4 -0
- package/dist/distribution/coordinator/in-memory.d.ts +3 -2
- package/dist/distribution/coordinator/in-memory.mjs +4 -0
- package/dist/distribution/transport/grpc.mjs +2 -2
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +2 -2
- package/dist/distribution.d.ts +43 -9
- package/dist/distribution.mjs +4590 -4263
- package/dist/{durability-wiring-ZLRQU6FG.mjs → durability-wiring-6AN4QTT3.mjs} +20 -16
- package/dist/{engine-9X6W5naj.d.ts → engine-96_kLtSY.d.ts} +43 -7
- package/dist/{index-vCMu8OeQ.d.ts → index-ONAEaqtx.d.ts} +8 -1
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +44 -25
- package/dist/index.mjs +13 -9
- package/dist/{memory-CV9NAxLY.d.ts → memory-dd3GESdc.d.ts} +3 -3
- package/dist/partial-words.mjs +2 -1
- package/dist/persistence/durability/checkpoint-worker.d.ts +60 -58
- package/dist/persistence/durability/checkpoint-worker.mjs +18 -14
- package/dist/react.d.ts +8 -7
- package/dist/{schema-DtsgmNfU.d.ts → schema-DttzM-ln.d.ts} +12 -0
- package/dist/server.d.ts +11 -10
- package/dist/server.mjs +33 -6
- package/dist/{snapshot-22CUEYYW.mjs → snapshot-JFZVOGZV.mjs} +28 -18
- package/dist/synonyms-CpOAHPa-.d.ts +52 -0
- package/dist/synonyms.d.ts +22 -0
- package/dist/synonyms.mjs +12 -0
- package/dist/{tasks-BGaZJyoY.d.ts → tasks-DA1Ff0G4.d.ts} +4 -1
- package/dist/{thread-ZP3UMIZT.mjs → thread-VTDLLYXC.mjs} +7 -5
- package/dist/vector/search-worker.d.ts +2 -1
- package/dist/workers/entry.mjs +4 -4
- package/dist/{write-PXDKPG2R.mjs → write-OJR3H4BI.mjs} +14 -10
- package/package.json +15 -7
- package/dist/chunk-PR4O7CAP.mjs +0 -18
- package/dist/chunk-X4SEXL4U.mjs +0 -2310
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
|
|
2
|
-
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, readMetadataEnvelope, adapterCheckpointStore, readSegmentManifest, reclaimUnlistedKeys, snapshotBundleKey, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readCurrentManifest, DIRECTORY_LOCK_SETTLE_MS, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, SINGLE_NODE_PRIMARY_TERM, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, isId, isUint32, isSequenceNumber, readWalSegments, durableRecordMissing, growPartitionsTo, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, VECTOR_FILE_ENVELOPE_VERSION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry } from './chunk-
|
|
3
|
-
import { applyDeleteEntry, applyIndexEntry } from './chunk-
|
|
4
|
-
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-
|
|
2
|
+
import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, readMetadataEnvelope, adapterCheckpointStore, readSegmentManifest, reclaimUnlistedKeys, snapshotBundleKey, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, writeMetadataEnvelope, metadataFromConfig, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readCurrentManifest, DIRECTORY_LOCK_SETTLE_MS, commitCheckpointManifest, removeCheckpointGarbage, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, SINGLE_NODE_PRIMARY_TERM, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, frameRecord, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, readCommitMarker, isId, isUint32, isSequenceNumber, readWalSegments, durableRecordMissing, growPartitionsTo, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, MAX_SEGMENTS_PER_PARTITION, VECTOR_FILE_ENVELOPE_VERSION, writeSegmentHeader, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, LOG_RECORDS_PER_CHECKPOINT, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, buildEntry } from './chunk-B6XE7KXY.mjs';
|
|
3
|
+
import { applyDeleteEntry, applyIndexEntry } from './chunk-66KYAOIK.mjs';
|
|
4
|
+
import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-NOUZ6RH3.mjs';
|
|
5
5
|
import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-NQW2YXEX.mjs';
|
|
6
6
|
import { NarsilError, ErrorCodes, compareCodePoints, detectRuntime, MIN_M, resolveWorkerEntry, MAX_FILE_BYTES } from './chunk-FHFFWLEK.mjs';
|
|
7
7
|
import { decode } from '@msgpack/msgpack';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { vectorsPerFile } from './chunk-3ZRMZIH4.mjs';
|
|
2
|
-
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, documentsOfPayloadV2, deserializePayloadV1, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts } from './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, setNestedValue, decodeIndexConfiguration, flattenSchema } from './chunk-
|
|
2
|
+
import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, documentsOfPayloadV2, deserializePayloadV1, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts } from './chunk-B6XE7KXY.mjs';
|
|
3
|
+
import './chunk-66KYAOIK.mjs';
|
|
4
|
+
import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, loadVectorCodeFor, prepareStoredSynonyms, rejectSnapshot, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, setNestedValue, decodeIndexConfiguration, flattenSchema } from './chunk-NOUZ6RH3.mjs';
|
|
5
5
|
import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, decodeVectorIndexParts, bytesToVectors } from './chunk-NQW2YXEX.mjs';
|
|
6
6
|
import { NarsilError, ErrorCodes, MAX_FILE_BYTES, BATCH_CHUNK_SIZE } from './chunk-FHFFWLEK.mjs';
|
|
7
7
|
import { decode } from '@msgpack/msgpack';
|
|
@@ -534,7 +534,12 @@ function prepareRestoredIndex(configuration, deps, reject) {
|
|
|
534
534
|
}
|
|
535
535
|
}
|
|
536
536
|
const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
|
|
537
|
-
|
|
537
|
+
const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
|
|
538
|
+
return { config, language, embeddingAdapter, adapterName, storedRevision, synonyms };
|
|
539
|
+
}
|
|
540
|
+
function restoredSynonyms(raw, config, language, reject) {
|
|
541
|
+
if (raw === void 0 || raw === null) return null;
|
|
542
|
+
return prepareStoredSynonyms(raw, config, language, (message) => reject(`synonyms: ${message}`));
|
|
538
543
|
}
|
|
539
544
|
async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
|
|
540
545
|
const loadingVectorCode = loadVectorCodeFor(prepared.config.schema);
|
|
@@ -551,7 +556,7 @@ async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
|
|
|
551
556
|
currentRevision: prepared.language.revision
|
|
552
557
|
});
|
|
553
558
|
}
|
|
554
|
-
deps.executor.createIndex(indexName, prepared.config, prepared.language);
|
|
559
|
+
deps.executor.createIndex(indexName, prepared.config, prepared.language, false, prepared.synonyms?.lookup ?? null);
|
|
555
560
|
deps.indexRegistry.set(indexName, {
|
|
556
561
|
config: prepared.config,
|
|
557
562
|
language: prepared.language,
|
|
@@ -562,7 +567,8 @@ async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
|
|
|
562
567
|
heldPartitions: null,
|
|
563
568
|
documentCount: 0,
|
|
564
569
|
partitionCount,
|
|
565
|
-
analysisRevision: prepared.storedRevision
|
|
570
|
+
analysisRevision: prepared.storedRevision,
|
|
571
|
+
synonyms: prepared.synonyms?.encoded ?? null
|
|
566
572
|
});
|
|
567
573
|
await deps.indexState.registerOpen(indexName);
|
|
568
574
|
const release = await deps.indexState.acquire(indexName, false);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-
|
|
2
|
-
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-
|
|
1
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-66KYAOIK.mjs';
|
|
2
|
+
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-NOUZ6RH3.mjs';
|
|
3
3
|
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-NQW2YXEX.mjs';
|
|
4
4
|
import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-FHFFWLEK.mjs';
|
|
5
5
|
import { decode } from '@msgpack/msgpack';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
3
|
-
import { h as TaskStore } from './tasks-
|
|
2
|
+
import { C as ClusterNamespace } from './config-DyRowvVP.js';
|
|
3
|
+
import { h as TaskStore } from './tasks-DA1Ff0G4.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Context handed to {@link OnRequestHook} for every inbound request, captured
|
|
@@ -284,6 +284,14 @@ declare const INDEX_LIFECYCLE_CAPABILITY = "indexes.lifecycle";
|
|
|
284
284
|
* @public
|
|
285
285
|
*/
|
|
286
286
|
declare const DOCUMENT_EXPORT_CAPABILITY = "indexes.documentExport";
|
|
287
|
+
/**
|
|
288
|
+
* Marks a server that reads the synonym set of an index at
|
|
289
|
+
* `GET /indexes/{name}/synonyms` and replaces it at
|
|
290
|
+
* `POST /indexes/{name}/synonyms`.
|
|
291
|
+
*
|
|
292
|
+
* @public
|
|
293
|
+
*/
|
|
294
|
+
declare const SYNONYMS_CAPABILITY = "indexes.synonyms";
|
|
287
295
|
/**
|
|
288
296
|
* Every capability this server announces at `/capabilities`.
|
|
289
297
|
*
|
|
@@ -295,4 +303,4 @@ declare const DOCUMENT_EXPORT_CAPABILITY = "indexes.documentExport";
|
|
|
295
303
|
*/
|
|
296
304
|
declare const SERVER_CAPABILITIES: readonly string[];
|
|
297
305
|
|
|
298
|
-
export { ASYNC_IMPORT_CAPABILITY as A, type CorsOptions as C, DOCUMENT_EXPORT_CAPABILITY as D, type ErrorEnvelope as E, INDEX_LIFECYCLE_CAPABILITY as I, type NarsilServer as N, type OnRequestHook as O, REBUILD_ANALYSIS_CAPABILITY as R, SERVER_CAPABILITIES as S, TASK_CANCEL_CAPABILITY as T,
|
|
306
|
+
export { ASYNC_IMPORT_CAPABILITY as A, type CorsOptions as C, DOCUMENT_EXPORT_CAPABILITY as D, type ErrorEnvelope as E, INDEX_LIFECYCLE_CAPABILITY as I, type NarsilServer as N, type OnRequestHook as O, REBUILD_ANALYSIS_CAPABILITY as R, SERVER_CAPABILITIES as S, TASK_CANCEL_CAPABILITY as T, SYNONYMS_CAPABILITY as a, TASK_FILTER_CAPABILITY as b, type ServerOptions as c, type RequestContext as d, type RequestDenial as e, type ServerLimits as f };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-
|
|
2
|
-
import { getNestedValue } from './chunk-
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors } from './chunk-A4G7JSYO.mjs';
|
|
2
|
+
import { getNestedValue } from './chunk-5ER245FI.mjs';
|
|
3
3
|
import { requireValidFieldTypes } from './chunk-CVQWQVFX.mjs';
|
|
4
4
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
5
5
|
import { ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
|
|
@@ -1,12 +1,13 @@
|
|
|
1
|
-
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-
|
|
2
|
-
import { ReplicationMessageTypes } from './chunk-
|
|
1
|
+
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-3RDS4UHC.mjs';
|
|
2
|
+
import { ReplicationMessageTypes } from './chunk-OHP7QR33.mjs';
|
|
3
3
|
import { WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
|
|
4
|
-
import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-
|
|
5
|
-
import { isCompositePartition, generateId, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-EMXAYPTQ.mjs';
|
|
6
|
-
import { fnv1a } from './chunk-X4SEXL4U.mjs';
|
|
7
|
-
import { getLanguage } from './chunk-PR4O7CAP.mjs';
|
|
4
|
+
import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-A4G7JSYO.mjs';
|
|
8
5
|
import { MAX_FILE_BYTES } from './chunk-RVM7AODE.mjs';
|
|
6
|
+
import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-XLJ2S2BH.mjs';
|
|
7
|
+
import { fnv1a, isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-6VZ3Q73N.mjs';
|
|
8
|
+
import { generateId, getLanguage } from './chunk-MKWUXWQH.mjs';
|
|
9
9
|
import { extractVectorFieldsFromSchema, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-CVQWQVFX.mjs';
|
|
10
|
+
import { resolveIndexAnalysis } from './chunk-NVL4ZR3K.mjs';
|
|
10
11
|
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
11
12
|
import { encode, decode } from '@msgpack/msgpack';
|
|
12
13
|
|
|
@@ -1,11 +1,142 @@
|
|
|
1
1
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-P36MUFEJ.mjs';
|
|
2
|
-
import { readSharedConfigFields, restoredEmbedding, restoredPartitionConfig } from './chunk-
|
|
3
|
-
import { getLanguage } from './chunk-
|
|
2
|
+
import { readSharedConfigFields, restoredEmbedding, restoredPartitionConfig } from './chunk-CO4MRKST.mjs';
|
|
3
|
+
import { generateId, getLanguage } from './chunk-MKWUXWQH.mjs';
|
|
4
4
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-Q5DC6WAB.mjs';
|
|
5
|
+
import { decodeSynonymRules } from './chunk-7X5JST5L.mjs';
|
|
5
6
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
6
7
|
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
7
8
|
import { encode, decode } from '@msgpack/msgpack';
|
|
8
9
|
|
|
10
|
+
// src/distribution/cluster/constants.ts
|
|
11
|
+
var LEADERSHIP_IMBALANCE_THRESHOLD = 2;
|
|
12
|
+
var ESTIMATED_PARTITION_BYTES = 50 * 1024 * 1024;
|
|
13
|
+
var REBALANCE_THRESHOLD = 0.1;
|
|
14
|
+
var BOOTSTRAP_CAS_ATTEMPTS = 5;
|
|
15
|
+
var ALLOCATION_CAS_ATTEMPTS = 5;
|
|
16
|
+
var ALLOCATION_RETRY_DELAY_MS = 1e3;
|
|
17
|
+
var TEARDOWN_CAS_ATTEMPTS = 5;
|
|
18
|
+
var TEARDOWN_RETRY_DELAY_MS = 500;
|
|
19
|
+
var TEARDOWN_RETRY_ROUNDS = 3;
|
|
20
|
+
var RECOVERY_CAS_ATTEMPTS = 5;
|
|
21
|
+
var PARTITION_STORES_TIMEOUT_MS = 5e3;
|
|
22
|
+
var DEFAULT_CONTROLLER_CONFIG = {
|
|
23
|
+
leaseTtlMs: 15e3,
|
|
24
|
+
standbyRetryMs: 5e3
|
|
25
|
+
};
|
|
26
|
+
var DEFAULT_NODE_LIFECYCLE_CONFIG = {
|
|
27
|
+
bootstrapRetryBaseMs: 1e3,
|
|
28
|
+
bootstrapRetryMaxMs: 5e3,
|
|
29
|
+
bootstrapMaxRetries: 10,
|
|
30
|
+
allocationDebounceMs: 250,
|
|
31
|
+
nodeHeartbeatIntervalMs: 1e4
|
|
32
|
+
};
|
|
33
|
+
var MAX_SYNONYM_GENERATION_LENGTH = 128;
|
|
34
|
+
|
|
35
|
+
// src/distribution/payload-guards.ts
|
|
36
|
+
function isRecord(value) {
|
|
37
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
38
|
+
}
|
|
39
|
+
function isValidInteger(value) {
|
|
40
|
+
return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// src/distribution/cluster/synonym-set.ts
|
|
44
|
+
var SYNONYM_CHUNK_BYTES = 1048576;
|
|
45
|
+
var SYNONYM_SET_PREFIX = "_narsil/synonyms/";
|
|
46
|
+
var MAX_UINT32 = 4294967295;
|
|
47
|
+
var DIGEST_PATTERN = /^[0-9a-f]{64}$/;
|
|
48
|
+
var FIRST_PRINTABLE_CODE = 32;
|
|
49
|
+
function isKeySegment(value) {
|
|
50
|
+
for (let i = 0; i < value.length; i++) {
|
|
51
|
+
const code = value.charCodeAt(i);
|
|
52
|
+
if (code < FIRST_PRINTABLE_CODE || value[i] === "/" || value[i] === "\\") return false;
|
|
53
|
+
}
|
|
54
|
+
return true;
|
|
55
|
+
}
|
|
56
|
+
function synonymChunkKey(indexUuid, generation, chunk) {
|
|
57
|
+
return `${SYNONYM_SET_PREFIX}${indexUuid}/${generation}/${chunk}`;
|
|
58
|
+
}
|
|
59
|
+
function isUint32(value) {
|
|
60
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= MAX_UINT32;
|
|
61
|
+
}
|
|
62
|
+
async function digestOf(bytes) {
|
|
63
|
+
const view = bytes.buffer instanceof ArrayBuffer ? new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) : Uint8Array.from(bytes);
|
|
64
|
+
const hash = new Uint8Array(await crypto.subtle.digest("SHA-256", view));
|
|
65
|
+
return Array.from(hash, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
66
|
+
}
|
|
67
|
+
function synonymSetRefOf(raw) {
|
|
68
|
+
if (!isRecord(raw)) return "synonyms must be an object";
|
|
69
|
+
const { generation, digest, chunkCount, byteLength } = raw;
|
|
70
|
+
if (typeof generation !== "string" || generation.length === 0 || generation.length > MAX_SYNONYM_GENERATION_LENGTH || !isKeySegment(generation)) {
|
|
71
|
+
return "synonyms.generation must hold 1 to 128 characters, with no control character, slash, or backslash";
|
|
72
|
+
}
|
|
73
|
+
if (typeof digest !== "string" || !DIGEST_PATTERN.test(digest)) {
|
|
74
|
+
return "synonyms.digest must be a lowercase hexadecimal SHA-256";
|
|
75
|
+
}
|
|
76
|
+
if (!isUint32(byteLength) || byteLength === 0) {
|
|
77
|
+
return "synonyms.byteLength must be a positive whole number";
|
|
78
|
+
}
|
|
79
|
+
if (!isUint32(chunkCount) || chunkCount !== Math.ceil(byteLength / SYNONYM_CHUNK_BYTES)) {
|
|
80
|
+
return "synonyms.chunkCount must match synonyms.byteLength";
|
|
81
|
+
}
|
|
82
|
+
return { generation, digest, chunkCount, byteLength };
|
|
83
|
+
}
|
|
84
|
+
function decodeSynonymSetRef(raw, reject) {
|
|
85
|
+
const ref = synonymSetRefOf(raw);
|
|
86
|
+
return typeof ref === "string" ? reject(ref) : ref;
|
|
87
|
+
}
|
|
88
|
+
function synonymSetRefOrNull(raw) {
|
|
89
|
+
const ref = synonymSetRefOf(raw);
|
|
90
|
+
return typeof ref === "string" ? null : ref;
|
|
91
|
+
}
|
|
92
|
+
async function deleteSynonymSet(coordinator, indexUuid, ref) {
|
|
93
|
+
for (let chunk = 0; chunk < ref.chunkCount; chunk++) {
|
|
94
|
+
await coordinator.delete(synonymChunkKey(indexUuid, ref.generation, chunk));
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
async function storeSynonymSet(coordinator, indexUuid, encoded) {
|
|
98
|
+
const ref = {
|
|
99
|
+
generation: generateId(),
|
|
100
|
+
digest: await digestOf(encoded),
|
|
101
|
+
chunkCount: Math.ceil(encoded.byteLength / SYNONYM_CHUNK_BYTES),
|
|
102
|
+
byteLength: encoded.byteLength
|
|
103
|
+
};
|
|
104
|
+
try {
|
|
105
|
+
for (let chunk = 0; chunk < ref.chunkCount; chunk++) {
|
|
106
|
+
const start = chunk * SYNONYM_CHUNK_BYTES;
|
|
107
|
+
const bytes = encoded.subarray(start, Math.min(start + SYNONYM_CHUNK_BYTES, encoded.byteLength));
|
|
108
|
+
if (!await coordinator.compareAndSet(synonymChunkKey(indexUuid, ref.generation, chunk), null, bytes)) {
|
|
109
|
+
throw new NarsilError(
|
|
110
|
+
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
111
|
+
`Synonym set chunk ${chunk} of a fresh copy already exists in the coordinator`,
|
|
112
|
+
{ indexUuid, generation: ref.generation, chunk }
|
|
113
|
+
);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
} catch (err) {
|
|
117
|
+
await deleteSynonymSet(coordinator, indexUuid, ref).catch(() => void 0);
|
|
118
|
+
throw err;
|
|
119
|
+
}
|
|
120
|
+
return ref;
|
|
121
|
+
}
|
|
122
|
+
async function readSynonymSet(coordinator, indexUuid, ref, reject) {
|
|
123
|
+
const encoded = new Uint8Array(ref.byteLength);
|
|
124
|
+
let filled = 0;
|
|
125
|
+
for (let chunk = 0; chunk < ref.chunkCount; chunk++) {
|
|
126
|
+
const bytes = await coordinator.get(synonymChunkKey(indexUuid, ref.generation, chunk));
|
|
127
|
+
if (bytes === null) return null;
|
|
128
|
+
if (bytes.byteLength > encoded.byteLength - filled) return reject("the stored synonym set exceeds its byteLength");
|
|
129
|
+
encoded.set(bytes, filled);
|
|
130
|
+
filled += bytes.byteLength;
|
|
131
|
+
}
|
|
132
|
+
if (filled !== ref.byteLength) return reject("the stored synonym set falls short of its byteLength");
|
|
133
|
+
if (await digestOf(encoded) !== ref.digest) return reject("the stored synonym set does not match its digest");
|
|
134
|
+
return encoded;
|
|
135
|
+
}
|
|
136
|
+
function storedSynonymSetRules(encoded, reject) {
|
|
137
|
+
return decodeSynonymRules(encoded, (message) => reject(message));
|
|
138
|
+
}
|
|
139
|
+
|
|
9
140
|
// src/distribution/cluster/index-settings.ts
|
|
10
141
|
function partitionLimitsOf(partitions) {
|
|
11
142
|
if (partitions === void 0) {
|
|
@@ -71,7 +202,7 @@ function indexSettingsOf(config) {
|
|
|
71
202
|
...settingsFieldsOf(config)
|
|
72
203
|
};
|
|
73
204
|
}
|
|
74
|
-
function
|
|
205
|
+
function isRecord2(value) {
|
|
75
206
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
76
207
|
}
|
|
77
208
|
function settingsRejection(indexName) {
|
|
@@ -88,7 +219,7 @@ function decodeIndexSettings(raw, indexName) {
|
|
|
88
219
|
return void 0;
|
|
89
220
|
}
|
|
90
221
|
const reject = settingsRejection(indexName);
|
|
91
|
-
if (!
|
|
222
|
+
if (!isRecord2(raw)) {
|
|
92
223
|
return reject("settings must be an object");
|
|
93
224
|
}
|
|
94
225
|
if (typeof raw.language !== "string" || raw.language.length === 0) {
|
|
@@ -96,13 +227,15 @@ function decodeIndexSettings(raw, indexName) {
|
|
|
96
227
|
}
|
|
97
228
|
const embedding = restoredEmbedding(raw.embedding, reject);
|
|
98
229
|
const partitions = restoredPartitionConfig(raw.partitionConfig, reject);
|
|
230
|
+
const synonyms = raw.synonyms === void 0 ? void 0 : decodeSynonymSetRef(raw.synonyms, reject);
|
|
99
231
|
return {
|
|
100
232
|
language: raw.language,
|
|
101
233
|
...settingsFieldsOf({
|
|
102
234
|
...readSharedConfigFields(raw, reject),
|
|
103
235
|
...partitions !== void 0 ? { partitions } : {},
|
|
104
236
|
...embedding !== void 0 ? { embedding } : {}
|
|
105
|
-
})
|
|
237
|
+
}),
|
|
238
|
+
...synonyms !== void 0 ? { synonyms } : {}
|
|
106
239
|
};
|
|
107
240
|
}
|
|
108
241
|
function withPartitionCount(config, partitionCount) {
|
|
@@ -112,7 +245,7 @@ function indexConfigFromSettings(schema, settings, indexName) {
|
|
|
112
245
|
if (settings === void 0) {
|
|
113
246
|
return { schema };
|
|
114
247
|
}
|
|
115
|
-
const { language, embedding, partitionConfig, ...fields } = settings;
|
|
248
|
+
const { language, embedding, partitionConfig, synonyms: _synonyms, ...fields } = settings;
|
|
116
249
|
return {
|
|
117
250
|
schema,
|
|
118
251
|
language,
|
|
@@ -153,14 +286,14 @@ function validateIndexName(indexName) {
|
|
|
153
286
|
);
|
|
154
287
|
}
|
|
155
288
|
}
|
|
156
|
-
function
|
|
289
|
+
function isValidInteger2(value) {
|
|
157
290
|
return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
|
|
158
291
|
}
|
|
159
292
|
function indexConfigKey(indexName) {
|
|
160
293
|
return `${INDEX_CONFIG_PREFIX}${indexName}${INDEX_CONFIG_SUFFIX}`;
|
|
161
294
|
}
|
|
162
295
|
function validateDecodedMetadata(decoded, indexName) {
|
|
163
|
-
if (!
|
|
296
|
+
if (!isRecord2(decoded)) {
|
|
164
297
|
throw new NarsilError(
|
|
165
298
|
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
166
299
|
`Index metadata for '${indexName}' is not an object`,
|
|
@@ -181,21 +314,21 @@ function validateDecodedMetadata(decoded, indexName) {
|
|
|
181
314
|
{ indexName, received: truncateForDisplay(decoded.indexName) }
|
|
182
315
|
);
|
|
183
316
|
}
|
|
184
|
-
if (!
|
|
317
|
+
if (!isValidInteger2(decoded.partitionCount) || decoded.partitionCount <= 0 || decoded.partitionCount > MAX_PARTITION_COUNT) {
|
|
185
318
|
throw new NarsilError(
|
|
186
319
|
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
187
320
|
`Index metadata for '${indexName}' has invalid partitionCount (must be an integer between 1 and ${MAX_PARTITION_COUNT})`,
|
|
188
321
|
{ indexName, partitionCount: truncateForDisplay(decoded.partitionCount) }
|
|
189
322
|
);
|
|
190
323
|
}
|
|
191
|
-
if (!
|
|
324
|
+
if (!isValidInteger2(decoded.replicationFactor) || decoded.replicationFactor < 0 || decoded.replicationFactor > MAX_REPLICATION_FACTOR) {
|
|
192
325
|
throw new NarsilError(
|
|
193
326
|
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
194
327
|
`Index metadata for '${indexName}' has invalid replicationFactor (must be an integer between 0 and ${MAX_REPLICATION_FACTOR})`,
|
|
195
328
|
{ indexName, replicationFactor: truncateForDisplay(decoded.replicationFactor) }
|
|
196
329
|
);
|
|
197
330
|
}
|
|
198
|
-
if (!
|
|
331
|
+
if (!isRecord2(decoded.constraints)) {
|
|
199
332
|
throw new NarsilError(
|
|
200
333
|
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
201
334
|
`Index metadata for '${indexName}' has invalid constraints`,
|
|
@@ -244,14 +377,67 @@ async function readStoredMetadata(coordinator, indexName) {
|
|
|
244
377
|
}
|
|
245
378
|
const decoded = decode(raw);
|
|
246
379
|
const record = validateDecodedMetadata(decoded, indexName);
|
|
247
|
-
return { record, rawSettings:
|
|
380
|
+
return { bytes: raw, record, rawSettings: isRecord2(decoded) ? decoded.settings : void 0 };
|
|
381
|
+
}
|
|
382
|
+
async function readIndexMetadataSnapshot(coordinator, indexName) {
|
|
383
|
+
const stored = await readStoredMetadata(coordinator, indexName);
|
|
384
|
+
if (stored === null) return null;
|
|
385
|
+
return { bytes: stored.bytes, record: stored.record, settings: decodeIndexSettings(stored.rawSettings, indexName) };
|
|
386
|
+
}
|
|
387
|
+
function metadataWithSynonymSet(snapshot, ref) {
|
|
388
|
+
const decoded = decode(snapshot.bytes);
|
|
389
|
+
const fields = isRecord2(decoded) ? decoded : {};
|
|
390
|
+
const { synonyms: _replaced, ...settings } = isRecord2(fields.settings) ? fields.settings : { language: snapshot.settings?.language ?? getLanguage("english").name };
|
|
391
|
+
const nextSettings = ref === null ? settings : { ...settings, synonyms: ref };
|
|
392
|
+
return new Uint8Array(encode({ ...fields, settings: nextSettings }, { ignoreUndefined: true }));
|
|
248
393
|
}
|
|
249
394
|
async function getIndexMetadata(coordinator, indexName) {
|
|
250
395
|
return (await readStoredMetadata(coordinator, indexName))?.record ?? null;
|
|
251
396
|
}
|
|
397
|
+
async function readIndexState(coordinator, indexName) {
|
|
398
|
+
const reject = settingsRejection(indexName);
|
|
399
|
+
let stored = await readStoredMetadata(coordinator, indexName);
|
|
400
|
+
while (stored !== null) {
|
|
401
|
+
const { record } = stored;
|
|
402
|
+
const settings = decodeIndexSettings(stored.rawSettings, indexName);
|
|
403
|
+
const ref = settings?.synonyms;
|
|
404
|
+
if (ref === void 0) return { record, settings, synonyms: [] };
|
|
405
|
+
const encoded = await readSynonymSet(coordinator, record.indexUuid, ref, reject);
|
|
406
|
+
if (encoded !== null) return { record, settings, synonyms: storedSynonymSetRules(encoded, reject) };
|
|
407
|
+
stored = await readStoredMetadata(coordinator, indexName);
|
|
408
|
+
if (stored !== null && referencesSameSet(stored, record.indexUuid, ref, indexName)) {
|
|
409
|
+
return reject(`a chunk of synonym set ${ref.generation} is missing`);
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
return null;
|
|
413
|
+
}
|
|
414
|
+
function referencesSameSet(stored, indexUuid, ref, indexName) {
|
|
415
|
+
if (stored.record.indexUuid !== indexUuid) return false;
|
|
416
|
+
return decodeIndexSettings(stored.rawSettings, indexName)?.synonyms?.generation === ref.generation;
|
|
417
|
+
}
|
|
252
418
|
async function getClusterIndexConfig(coordinator, indexName, schema) {
|
|
253
|
-
const
|
|
254
|
-
|
|
419
|
+
const state = await readIndexState(coordinator, indexName);
|
|
420
|
+
const config = indexConfigFromSettings(schema, state?.settings, indexName);
|
|
421
|
+
return state === null || state.synonyms.length === 0 ? config : { ...config, synonyms: state.synonyms };
|
|
422
|
+
}
|
|
423
|
+
function decodedMetadataOf(bytes) {
|
|
424
|
+
let decoded;
|
|
425
|
+
try {
|
|
426
|
+
decoded = decode(bytes);
|
|
427
|
+
} catch {
|
|
428
|
+
return null;
|
|
429
|
+
}
|
|
430
|
+
return isRecord2(decoded) ? decoded : null;
|
|
431
|
+
}
|
|
432
|
+
function indexUuidOfStoredMetadata(bytes) {
|
|
433
|
+
const decoded = decodedMetadataOf(bytes);
|
|
434
|
+
return typeof decoded?.indexUuid === "string" ? decoded.indexUuid : null;
|
|
435
|
+
}
|
|
436
|
+
function storedSynonymSetOf(bytes) {
|
|
437
|
+
const decoded = decodedMetadataOf(bytes);
|
|
438
|
+
if (decoded === null || typeof decoded.indexUuid !== "string" || !isRecord2(decoded.settings)) return null;
|
|
439
|
+
const ref = synonymSetRefOrNull(decoded.settings.synonyms);
|
|
440
|
+
return ref === null ? null : { indexUuid: decoded.indexUuid, ref };
|
|
255
441
|
}
|
|
256
442
|
|
|
257
|
-
export { getClusterIndexConfig, getIndexMetadata, indexConfigKey, indexSettingsOf, putIndexMetadata, validateIndexName, withPartitionCount };
|
|
443
|
+
export { ALLOCATION_CAS_ATTEMPTS, ALLOCATION_RETRY_DELAY_MS, BOOTSTRAP_CAS_ATTEMPTS, DEFAULT_CONTROLLER_CONFIG, DEFAULT_NODE_LIFECYCLE_CONFIG, ESTIMATED_PARTITION_BYTES, LEADERSHIP_IMBALANCE_THRESHOLD, PARTITION_STORES_TIMEOUT_MS, REBALANCE_THRESHOLD, RECOVERY_CAS_ATTEMPTS, TEARDOWN_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, TEARDOWN_RETRY_ROUNDS, deleteSynonymSet, getClusterIndexConfig, getIndexMetadata, indexConfigFromSettings, indexConfigKey, indexSettingsOf, indexUuidOfStoredMetadata, isRecord, isValidInteger, metadataWithSynonymSet, putIndexMetadata, readIndexMetadataSnapshot, readIndexState, storeSynonymSet, storedSynonymSetOf, validateIndexName, withPartitionCount };
|