@delali/narsil 0.3.1-next.91 → 0.3.1-next.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-66KYAOIK.mjs → chunk-3U636ADD.mjs} +2 -2
- package/dist/browser/{chunk-NOUZ6RH3.mjs → chunk-D3CC2NMZ.mjs} +238 -114
- package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-FYFQJKFW.mjs} +208 -78
- package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
- package/dist/browser/{chunk-B6XE7KXY.mjs → chunk-LULHQXCK.mjs} +4 -4
- package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
- package/dist/browser/{durability-wiring-F4OGQUSU.mjs → durability-wiring-3ZRKSPJD.mjs} +498 -71
- package/dist/browser/{module-E25UDADM.mjs → module-IEOQI5GI.mjs} +102 -54
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-RBULSOVX.mjs +35 -0
- package/dist/browser/{snapshot-5EJ7I7QY.mjs → snapshot-Y3NGUJHX.mjs} +362 -103
- package/dist/browser/{write-K5V4XQ72.mjs → write-ZUBDNLCH.mjs} +4 -4
- package/dist/{capabilities-CvExBThq.d.ts → capabilities-BvxvFzVA.d.ts} +2 -2
- package/dist/{chunk-NVL4ZR3K.mjs → chunk-2GZGBOQI.mjs} +2 -2
- package/dist/{chunk-XLJ2S2BH.mjs → chunk-3LYY62M6.mjs} +40 -21
- package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
- package/dist/{chunk-T4DB5YFB.mjs → chunk-54HYC5FT.mjs} +6 -6
- package/dist/{chunk-ZIJ63TBT.mjs → chunk-7KC56574.mjs} +1 -1
- package/dist/{chunk-67Z2V6TC.mjs → chunk-7QONURHW.mjs} +3 -3
- package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
- package/dist/{chunk-6VZ3Q73N.mjs → chunk-CHOGQSHK.mjs} +44 -25
- package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
- package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
- package/dist/{chunk-43ORORN6.mjs → chunk-GJZHU4FC.mjs} +3 -3
- package/dist/{chunk-R7OSDAFK.mjs → chunk-HBVLF3HA.mjs} +1 -1
- package/dist/{chunk-ULRPNYAG.mjs → chunk-ICL2AELD.mjs} +120 -62
- package/dist/{chunk-3RDS4UHC.mjs → chunk-J7SDAOTP.mjs} +4 -4
- package/dist/{chunk-A4G7JSYO.mjs → chunk-JFUE6F2Y.mjs} +98 -49
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-7OQHSDK6.mjs → chunk-L26LEDHO.mjs} +7 -7
- package/dist/{chunk-EONIHG3V.mjs → chunk-MASE2LTJ.mjs} +7 -7
- package/dist/{chunk-MXS5HWIH.mjs → chunk-MWT5WCJ5.mjs} +89 -17
- package/dist/{chunk-3TTYK5A7.mjs → chunk-Q4UTU5ZE.mjs} +9 -9
- package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
- package/dist/chunk-QY7GC2MW.mjs +33 -0
- package/dist/{chunk-BJ2YKRNX.mjs → chunk-R36MIGBU.mjs} +86 -54
- package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
- package/dist/{chunk-UXQE2ROB.mjs → chunk-RUIXODYU.mjs} +3 -2
- package/dist/{chunk-5ER245FI.mjs → chunk-SBCEHIPL.mjs} +3 -3
- package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
- package/dist/{chunk-UKSUENHK.mjs → chunk-T2QN2FA4.mjs} +40 -26
- package/dist/{chunk-CO4MRKST.mjs → chunk-WLOF6HSP.mjs} +2 -2
- package/dist/{chunk-MKWUXWQH.mjs → chunk-ZHS44YKF.mjs} +1 -1
- package/dist/{client-Cc42b42j.d.ts → client-DKwterMb.d.ts} +2 -2
- package/dist/client.d.ts +6 -6
- package/dist/client.mjs +1 -0
- package/dist/{config-DyRowvVP.d.ts → config-6YBtxekG.d.ts} +1 -1
- package/dist/{config-BYne-yCI.d.ts → config-DPphZfuM.d.ts} +1 -1
- package/dist/direct-executor-5KJGVQHM.mjs +22 -0
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +26 -26
- package/dist/{durability-wiring-6AN4QTT3.mjs → durability-wiring-FRPV6JLZ.mjs} +518 -91
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-96_kLtSY.d.ts → engine-BGt9lI1V.d.ts} +11 -11
- package/dist/index.browser.mjs +2 -2
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +18 -18
- package/dist/{memory-dd3GESdc.d.ts → memory-COdoGthn.d.ts} +96 -76
- package/dist/{module-WIDYIJLY.mjs → module-IUTNXWMV.mjs} +72 -38
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +21 -21
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/restore-queue-I5SI6AUQ.mjs +35 -0
- package/dist/server.d.ts +8 -8
- package/dist/server.mjs +10 -10
- package/dist/snapshot-YC6TS4OP.mjs +738 -0
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-DA1Ff0G4.d.ts → tasks-BJmamiC4.d.ts} +1 -1
- package/dist/{thread-VTDLLYXC.mjs → thread-U7XYJWGG.mjs} +15 -12
- package/dist/vector/search-worker.mjs +6 -6
- package/dist/workers/entry.mjs +5 -5
- package/dist/{write-OJR3H4BI.mjs → write-M2OALSA4.mjs} +20 -20
- package/package.json +7 -7
- package/dist/chunk-Y2F66IQJ.mjs +0 -74
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/direct-executor-CHXVCBN7.mjs +0 -22
- package/dist/snapshot-JFZVOGZV.mjs +0 -479
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-
|
|
2
|
-
import { generateId, getLanguage } from './chunk-
|
|
3
|
-
import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-
|
|
4
|
-
import { tokenizeOptions, getNestedValue, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-
|
|
5
|
-
import { schemaFieldsOf, partialFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-
|
|
1
|
+
import { requirePatternSearchFor, requirePartialWordSearchFor, createPartitionIndex, buildPartialWordTableFrom, recordValueSize, patternIndexOf, encodeFieldIndexes, encodePartialSpellingsOf, countPartialSpelling, copyVectorsInto, isCompositePartition, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, documentValueBytes, cloneProjected, projectIncludedPaths, valueWithOwnBuffers, createSurfaceRegistry, MAX_TERM_FREQUENCY, joinPatternWorkMeter, fanOutQuery, sanitizeGlobalStats, countsWithoutScores, fanOutMatchCount, createCompositePartition, createGeoIndex, documentWithOwnBuffers, readProjectedPath, createNumericIndex, createBooleanIndex, createEnumIndex, ValueAffixes } from './chunk-CHOGQSHK.mjs';
|
|
2
|
+
import { generateId, getLanguage } from './chunk-ZHS44YKF.mjs';
|
|
3
|
+
import { GrowableUint32, bitsetHas, bitsetSet, createBitSet } from './chunk-QM3VPX4Z.mjs';
|
|
4
|
+
import { tokenizeOptions, getNestedValue, sameFields, visitDocumentSurfaceSpellings, visitDocumentSpellings } from './chunk-SBCEHIPL.mjs';
|
|
5
|
+
import { schemaFieldsOf, partialFieldsOf, encodeDocument, validateDocument, validateDocumentStrict, extractVectorFieldsFromSchema } from './chunk-4ZJWRETC.mjs';
|
|
6
6
|
import { fuzzyPrefixOf, boundedLevenshtein } from './chunk-GES7U2PM.mjs';
|
|
7
|
-
import { isWordIndexedBase } from './chunk-
|
|
8
|
-
import { resolveIndexAnalysis, synonymSearchModule } from './chunk-
|
|
9
|
-
import { visitTokens, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION } from './chunk-
|
|
7
|
+
import { isWordIndexedBase } from './chunk-BB7MASRQ.mjs';
|
|
8
|
+
import { resolveIndexAnalysis, synonymSearchModule } from './chunk-2GZGBOQI.mjs';
|
|
9
|
+
import { visitTokens, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, FROZEN_REMOVAL_INDEX_MAX_PROCESS_FRACTION } from './chunk-7KC56574.mjs';
|
|
10
10
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
11
|
-
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-
|
|
11
|
+
import { loadedVectorCode, loadVectorCode, loadVectorCodeFor, requireLoadedVectorCode } from './chunk-HBVLF3HA.mjs';
|
|
12
12
|
import { PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
|
|
13
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
13
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
14
14
|
import { totalmem } from 'os';
|
|
15
15
|
import { getHeapStatistics } from 'v8';
|
|
16
16
|
import { Decoder, decode } from '@msgpack/msgpack';
|
|
@@ -699,6 +699,9 @@ function createFrozenDocTable(idTable, lengthColumns, documents, tombstones) {
|
|
|
699
699
|
internalIdCapacity() {
|
|
700
700
|
return capacity;
|
|
701
701
|
},
|
|
702
|
+
numberingGeneration() {
|
|
703
|
+
return 0;
|
|
704
|
+
},
|
|
702
705
|
contentBytes() {
|
|
703
706
|
return documents.byteLength;
|
|
704
707
|
},
|
|
@@ -2237,6 +2240,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
2237
2240
|
lastSchemaRef: null,
|
|
2238
2241
|
sortColumns: null,
|
|
2239
2242
|
scoreBuffer: null,
|
|
2243
|
+
vectorOrdinalMaps: null,
|
|
2240
2244
|
liveDocumentCount() {
|
|
2241
2245
|
return source.documentCount - tombstones.size;
|
|
2242
2246
|
},
|
|
@@ -3296,7 +3300,7 @@ function heldVectorOf(held, fieldName, docId) {
|
|
|
3296
3300
|
return held.fields.get(fieldName)?.view.vectorOf(docId);
|
|
3297
3301
|
}
|
|
3298
3302
|
|
|
3299
|
-
// src/workers/direct-executor.ts
|
|
3303
|
+
// src/workers/direct-executor/index.ts
|
|
3300
3304
|
var NO_VECTOR_WORKER_COPIES = { enabled: false };
|
|
3301
3305
|
function createDirectExecutor(options) {
|
|
3302
3306
|
const indexes = /* @__PURE__ */ new Map();
|
|
@@ -3312,7 +3316,7 @@ function createDirectExecutor(options) {
|
|
|
3312
3316
|
}
|
|
3313
3317
|
return entry;
|
|
3314
3318
|
}
|
|
3315
|
-
function vectorIndexesFor(indexName, config) {
|
|
3319
|
+
function vectorIndexesFor(indexName, config, copies = vectorWorkerCopies) {
|
|
3316
3320
|
const vectorIndexes = /* @__PURE__ */ new Map();
|
|
3317
3321
|
const vectorFields = extractVectorFieldsFromSchema(config.schema);
|
|
3318
3322
|
if (vectorFields.size === 0) return vectorIndexes;
|
|
@@ -3324,7 +3328,7 @@ function createDirectExecutor(options) {
|
|
|
3324
3328
|
fieldPath,
|
|
3325
3329
|
dim,
|
|
3326
3330
|
config.vectorPromotion,
|
|
3327
|
-
|
|
3331
|
+
copies,
|
|
3328
3332
|
indexName,
|
|
3329
3333
|
config.vectorPromotion?.storage ?? vectorStorage
|
|
3330
3334
|
)
|
|
@@ -3344,11 +3348,16 @@ function createDirectExecutor(options) {
|
|
|
3344
3348
|
indexName
|
|
3345
3349
|
});
|
|
3346
3350
|
}
|
|
3351
|
+
const entry = buildEntry(indexName, config, language, analysisStale, awaitedCopyLoad, vectorWorkerCopies);
|
|
3352
|
+
indexes.set(indexName, entry);
|
|
3353
|
+
return entry.manager;
|
|
3354
|
+
}
|
|
3355
|
+
function buildEntry(indexName, config, language, analysisStale, awaitedCopyLoad, copies) {
|
|
3347
3356
|
const router = createPartitionRouter();
|
|
3348
3357
|
const partitionCount = config.partitions?.maxPartitions ?? 1;
|
|
3349
|
-
const vectorIndexes = vectorIndexesFor(indexName, config);
|
|
3358
|
+
const vectorIndexes = vectorIndexesFor(indexName, config, copies);
|
|
3350
3359
|
const manager = createPartitionManager(indexName, config, language, router, partitionCount, vectorIndexes);
|
|
3351
|
-
|
|
3360
|
+
return {
|
|
3352
3361
|
manager,
|
|
3353
3362
|
config,
|
|
3354
3363
|
language,
|
|
@@ -3364,11 +3373,9 @@ function createDirectExecutor(options) {
|
|
|
3364
3373
|
vectorCopies: createHeldVectorCopies(),
|
|
3365
3374
|
analysisStale,
|
|
3366
3375
|
awaitedCopyLoad
|
|
3367
|
-
}
|
|
3368
|
-
return manager;
|
|
3376
|
+
};
|
|
3369
3377
|
}
|
|
3370
|
-
function
|
|
3371
|
-
const entry = requireIndex(indexName);
|
|
3378
|
+
function disposeEntry(entry) {
|
|
3372
3379
|
for (const vectorIndex of entry.vectorIndexes.values()) {
|
|
3373
3380
|
vectorIndex.dispose();
|
|
3374
3381
|
}
|
|
@@ -3376,8 +3383,16 @@ function createDirectExecutor(options) {
|
|
|
3376
3383
|
for (const partition of entry.manager.getAllPartitions()) {
|
|
3377
3384
|
partition.clear();
|
|
3378
3385
|
}
|
|
3386
|
+
}
|
|
3387
|
+
function dropIndex(indexName) {
|
|
3388
|
+
disposeEntry(requireIndex(indexName));
|
|
3379
3389
|
indexes.delete(indexName);
|
|
3380
3390
|
}
|
|
3391
|
+
function installDetachedIndex(detached) {
|
|
3392
|
+
const previous = indexes.get(detached.indexName);
|
|
3393
|
+
indexes.set(detached.indexName, detached.entry);
|
|
3394
|
+
if (previous !== void 0 && previous !== detached.entry) disposeEntry(previous);
|
|
3395
|
+
}
|
|
3381
3396
|
function listIndexes() {
|
|
3382
3397
|
return Array.from(indexes.keys());
|
|
3383
3398
|
}
|
|
@@ -3565,8 +3580,12 @@ function createDirectExecutor(options) {
|
|
|
3565
3580
|
},
|
|
3566
3581
|
createIndex,
|
|
3567
3582
|
dropIndex,
|
|
3568
|
-
listIndexes
|
|
3583
|
+
listIndexes,
|
|
3584
|
+
vectorWorkerCopies,
|
|
3585
|
+
createDetachedEntry: (indexName, config, language, copies) => buildEntry(indexName, config, language, false, null, copies),
|
|
3586
|
+
installDetachedIndex,
|
|
3587
|
+
discardDetachedIndex: (detached) => disposeEntry(detached.entry)
|
|
3569
3588
|
};
|
|
3570
3589
|
}
|
|
3571
3590
|
|
|
3572
|
-
export { availableSynonymModule, buildSegmentPayload, createDirectExecutor, createFrozenSegment, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, freezeSegmentShared, isEmptySynonymSet, mergeFrozenSegments, mergeSegmentContents, readHeapStatistics, registerSynonyms, resolvePartitionInsertOptions, segmentArraysOf, selectionKeeps, shareSegmentArrays, synonymModule };
|
|
3591
|
+
export { availableSynonymModule, buildSegmentPayload, createDirectExecutor, createFrozenSegment, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, freezeSegmentShared, isEmptySynonymSet, mergeFrozenSegments, mergeSegmentContents, readHeapStatistics, readResidentMemory, registerSynonyms, resolvePartitionInsertOptions, segmentArraysOf, selectionKeeps, shareSegmentArrays, synonymModule };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { EARLIER_RELEASE_MIN_M, MIN_M, MAX_M } from './chunk-KEHK2O6P.mjs';
|
|
2
|
-
import { isWordIndexedBase, parseFieldType } from './chunk-
|
|
2
|
+
import { isWordIndexedBase, parseFieldType } from './chunk-BB7MASRQ.mjs';
|
|
3
3
|
import { compareCodePoints, holdsUnpairedSurrogate, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
4
4
|
import { DEFAULT_PATTERN_VALUE_LIMIT, isPlainObject, MAX_TEXT_FIELDS, MAX_VECTOR_DIMENSION, MAX_DOCUMENT_BYTES, MAX_NESTING_DEPTH, PROTOTYPE_POLLUTION_KEYS, FIELD_NAME_PATTERN, RESERVED_ROOT_FIELDS, MAX_DOCUMENT_NESTING_DEPTH, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
5
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
5
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
6
6
|
import { encode } from '@msgpack/msgpack';
|
|
7
7
|
|
|
8
8
|
// src/schema/validator/schema.ts
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { decodeSegmentManifest, loadPartitionSegments, loadVectorField } from './chunk-
|
|
2
|
-
import { manifestKey, segmentsPrefix } from './chunk-
|
|
3
|
-
import { readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader } from './chunk-
|
|
4
|
-
import { VALID_HNSW_METRICS } from './chunk-
|
|
5
|
-
import { requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-
|
|
1
|
+
import { decodeSegmentManifest, loadPartitionSegments, loadVectorField } from './chunk-MASE2LTJ.mjs';
|
|
2
|
+
import { manifestKey, segmentsPrefix } from './chunk-Q4UTU5ZE.mjs';
|
|
3
|
+
import { readHeader, HEADER_SIZE, crc32, engineVersionParts, writeHeader } from './chunk-JFUE6F2Y.mjs';
|
|
4
|
+
import { VALID_HNSW_METRICS } from './chunk-RDJKYKMQ.mjs';
|
|
5
|
+
import { requireValidFieldTypes, isQuantizationMode, isStorageMode } from './chunk-4ZJWRETC.mjs';
|
|
6
6
|
import { VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
7
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
7
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
8
8
|
import { decode, encode } from '@msgpack/msgpack';
|
|
9
9
|
|
|
10
10
|
// src/serialization/stored-vector-values.ts
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
2
|
-
import { createNarsilError, ErrorCodes } from './chunk-
|
|
2
|
+
import { createNarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
3
3
|
|
|
4
4
|
// src/core/tokenizer/constants.ts
|
|
5
5
|
var TOKEN_CACHE_SIZE_FLOOR = 5e4;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { nodeThreadId } from './chunk-C7SLA6CM.mjs';
|
|
2
|
-
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-
|
|
2
|
+
import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-Q4UTU5ZE.mjs';
|
|
3
3
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
5
5
|
import { openSync, close, readFileSync } from 'fs';
|
|
6
6
|
import { rename, readFile, writeFile, unlink, realpath, readdir, readlink } from 'fs/promises';
|
|
7
7
|
import { endianness } from 'os';
|
|
@@ -628,4 +628,4 @@ async function listRecursive(dir, base, pathMod, fs) {
|
|
|
628
628
|
return results;
|
|
629
629
|
}
|
|
630
630
|
|
|
631
|
-
export { createDurableDirectory };
|
|
631
|
+
export { createDurableDirectory, getFs };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { PARSED_FIELD_TYPE_CACHE_ENTRIES, VECTOR_PATTERN, MAX_STORED_VECTOR_DIMENSION } from './chunk-L5ORED7V.mjs';
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
3
3
|
|
|
4
4
|
// src/schema/validator/field-type.ts
|
|
5
5
|
var FIELD_OPTION_ORDER = ["sortable", "pattern", "partial"];
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, patternSearchModule, DEFAULT_PATTERN_WORK_CAP, bitsetHas, createBitSet, bitsetSet, bitsetIsEmpty, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES } from './chunk-
|
|
1
|
+
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, patternSearchModule, DEFAULT_PATTERN_WORK_CAP, bitsetHas, createBitSet, bitsetSet, bitsetIsEmpty, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES } from './chunk-QM3VPX4Z.mjs';
|
|
2
2
|
import { GEO_UNIT_METRES, MAX_FILTER_DEPTH, MAX_FILTER_TESTS, FILTER_OPERAND_COPY_DEPTH } from './chunk-WQBCBKY3.mjs';
|
|
3
|
-
import { expandTermPrefix, suggestDisplayTerms, getNestedValue, textFieldsChanged, getFlatSchema, tokenizeOptions, partialWordSearchModule, getOrCreateFieldNameIndex, partitionDocumentFrequency } from './chunk-
|
|
4
|
-
import { isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, partialFieldsOf, schemaFieldsOf, flattenSchema, validateDocument, validateDocumentStrict, ANTIMERIDIAN_SPAN_DEGREES, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, RADIUS_BOX_MARGIN_RATIO, RADIUS_BOX_MARGIN_RADIANS, DEG_TO_RAD, VINCENTY_MAX_ITERATIONS } from './chunk-
|
|
3
|
+
import { expandTermPrefix, suggestDisplayTerms, getNestedValue, textFieldsChanged, getFlatSchema, tokenizeOptions, partialWordSearchModule, getOrCreateFieldNameIndex, partitionDocumentFrequency } from './chunk-SBCEHIPL.mjs';
|
|
4
|
+
import { isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, partialFieldsOf, schemaFieldsOf, flattenSchema, validateDocument, validateDocumentStrict, ANTIMERIDIAN_SPAN_DEGREES, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, RADIUS_BOX_MARGIN_RATIO, RADIUS_BOX_MARGIN_RADIANS, DEG_TO_RAD, VINCENTY_MAX_ITERATIONS } from './chunk-4ZJWRETC.mjs';
|
|
5
5
|
import { MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, RESULT_WINDOW, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_PAGE_SIZE, PREFIX_MAX_EXPANSIONS } from './chunk-FGMGUDHX.mjs';
|
|
6
6
|
import { fuzzyPrefixOf, boundedLevenshtein, fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
7
|
-
import { parsedTypeOf, isStringValuedBase, withSortableOption, isWordIndexedBase, isWordIndexedFieldType, isListBase } from './chunk-
|
|
8
|
-
import { tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, visitTokens, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT } from './chunk-
|
|
7
|
+
import { parsedTypeOf, isStringValuedBase, withSortableOption, isWordIndexedBase, isWordIndexedFieldType, isListBase } from './chunk-BB7MASRQ.mjs';
|
|
8
|
+
import { tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, visitTokens, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT } from './chunk-7KC56574.mjs';
|
|
9
9
|
import { singleFoldTable, appendFoldedCodePoint, sortModeOf, defaultSortMode, SORT_VALUE_MAX_CODE_POINTS, toReducedSortValue, readSortField, compareComparableKeys, toComparableSortValue, isSortMode, SORT_MODES, compareSortValues, compareComparableValues, truncateSortString } from './chunk-R6A773WK.mjs';
|
|
10
10
|
import { compareCodePoints, exceedsCodePoints, codePointCount, holdsUnpairedSurrogate } from './chunk-YTJCGKM4.mjs';
|
|
11
11
|
import { isGeopointOrVectorType, PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
|
|
12
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
12
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
13
13
|
|
|
14
14
|
// src/core/partition/partial-words/platform.ts
|
|
15
15
|
function platformPartialWordSearch() {
|
|
@@ -423,6 +423,7 @@ function createDocumentStore() {
|
|
|
423
423
|
const reverseMap = [];
|
|
424
424
|
const lengthColumns = /* @__PURE__ */ new Map();
|
|
425
425
|
let nextInternalId = 0;
|
|
426
|
+
let numberingResets = 0;
|
|
426
427
|
let sortedIds = null;
|
|
427
428
|
let storedContentBytes = 0;
|
|
428
429
|
function replaceStored(docId, document, fieldLengths) {
|
|
@@ -461,6 +462,7 @@ function createDocumentStore() {
|
|
|
461
462
|
forwardMap.clear();
|
|
462
463
|
reverseMap.length = 0;
|
|
463
464
|
nextInternalId = 0;
|
|
465
|
+
numberingResets++;
|
|
464
466
|
sortedIds = null;
|
|
465
467
|
}
|
|
466
468
|
const idResolver = {
|
|
@@ -569,6 +571,9 @@ function createDocumentStore() {
|
|
|
569
571
|
internalIdCapacity() {
|
|
570
572
|
return nextInternalId;
|
|
571
573
|
},
|
|
574
|
+
numberingGeneration() {
|
|
575
|
+
return numberingResets;
|
|
576
|
+
},
|
|
572
577
|
contentBytes() {
|
|
573
578
|
return storedContentBytes;
|
|
574
579
|
},
|
|
@@ -4905,7 +4910,7 @@ function estimatePartitionBytes(state) {
|
|
|
4905
4910
|
bytes += docCount * state.sizeIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
4906
4911
|
for (const index of state.patternIndexes.values()) bytes += index.bytes();
|
|
4907
4912
|
bytes += state.partialWords.bytes();
|
|
4908
|
-
return bytes + sortColumnBytes(state);
|
|
4913
|
+
return bytes + sortColumnBytes(state) + (state.vectorOrdinalMaps?.bytes ?? 0);
|
|
4909
4914
|
}
|
|
4910
4915
|
|
|
4911
4916
|
// src/core/partition/merge.ts
|
|
@@ -6329,7 +6334,8 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
6329
6334
|
lastSchemaRef: null,
|
|
6330
6335
|
trackPositions,
|
|
6331
6336
|
sortColumns: null,
|
|
6332
|
-
scoreBuffer: null
|
|
6337
|
+
scoreBuffer: null,
|
|
6338
|
+
vectorOrdinalMaps: null
|
|
6333
6339
|
};
|
|
6334
6340
|
function clearAll() {
|
|
6335
6341
|
state.invertedIdx.clear();
|
|
@@ -6351,6 +6357,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
6351
6357
|
state.lastSchemaRef = null;
|
|
6352
6358
|
state.sortColumns = null;
|
|
6353
6359
|
state.scoreBuffer = null;
|
|
6360
|
+
state.vectorOrdinalMaps = null;
|
|
6354
6361
|
}
|
|
6355
6362
|
const partition = {
|
|
6356
6363
|
get partitionId() {
|
|
@@ -6505,6 +6512,9 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
6505
6512
|
estimateMemoryBytes() {
|
|
6506
6513
|
return estimatePartitionBytes(state);
|
|
6507
6514
|
},
|
|
6515
|
+
readStates() {
|
|
6516
|
+
return [state];
|
|
6517
|
+
},
|
|
6508
6518
|
searchFulltext(params) {
|
|
6509
6519
|
return searchFulltext(state, params);
|
|
6510
6520
|
},
|
|
@@ -6664,7 +6674,7 @@ function frozenSegmentBytes(segment) {
|
|
|
6664
6674
|
bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
|
|
6665
6675
|
}
|
|
6666
6676
|
for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
|
|
6667
|
-
return bytes + segment.partialWords.bytes() + segment.removalBytes();
|
|
6677
|
+
return bytes + segment.partialWords.bytes() + segment.removalBytes() + (segment.vectorOrdinalMaps?.bytes ?? 0);
|
|
6668
6678
|
}
|
|
6669
6679
|
|
|
6670
6680
|
// src/core/partition/composite/compaction.ts
|
|
@@ -6737,6 +6747,24 @@ function compositeFieldValues(frozen, live, docId, fieldPaths) {
|
|
|
6737
6747
|
}
|
|
6738
6748
|
return live.fieldValues(docId, fieldPaths);
|
|
6739
6749
|
}
|
|
6750
|
+
function* compositeDocIds(frozen, live) {
|
|
6751
|
+
for (const segment of frozen) {
|
|
6752
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
6753
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
6754
|
+
if (docId !== void 0) yield docId;
|
|
6755
|
+
}
|
|
6756
|
+
}
|
|
6757
|
+
yield* live.docIds();
|
|
6758
|
+
}
|
|
6759
|
+
function compositeSortedDocIds(frozen, live) {
|
|
6760
|
+
const merged = [];
|
|
6761
|
+
for (const segment of frozen) {
|
|
6762
|
+
merged.push(...segment.docStore.sortedDocIds());
|
|
6763
|
+
}
|
|
6764
|
+
merged.push(...live.sortedDocIds());
|
|
6765
|
+
merged.sort(compareCodePoints);
|
|
6766
|
+
return merged;
|
|
6767
|
+
}
|
|
6740
6768
|
|
|
6741
6769
|
// src/core/partition/composite/filters.ts
|
|
6742
6770
|
var WORD_BITS = 32;
|
|
@@ -7412,23 +7440,11 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
7412
7440
|
}
|
|
7413
7441
|
return total;
|
|
7414
7442
|
},
|
|
7415
|
-
|
|
7416
|
-
|
|
7417
|
-
for (const internalId of segment.docStore.allInternalIds()) {
|
|
7418
|
-
const docId = segment.docStore.getExternalId(internalId);
|
|
7419
|
-
if (docId !== void 0) yield docId;
|
|
7420
|
-
}
|
|
7421
|
-
}
|
|
7422
|
-
yield* live.docIds();
|
|
7443
|
+
docIds() {
|
|
7444
|
+
return compositeDocIds(frozen, live);
|
|
7423
7445
|
},
|
|
7424
7446
|
sortedDocIds() {
|
|
7425
|
-
|
|
7426
|
-
for (const segment of frozen) {
|
|
7427
|
-
merged.push(...segment.docStore.sortedDocIds());
|
|
7428
|
-
}
|
|
7429
|
-
merged.push(...live.sortedDocIds());
|
|
7430
|
-
merged.sort(compareCodePoints);
|
|
7431
|
-
return merged;
|
|
7447
|
+
return compositeSortedDocIds(frozen, live);
|
|
7432
7448
|
},
|
|
7433
7449
|
releaseSortedDocIds() {
|
|
7434
7450
|
for (const segment of frozen) {
|
|
@@ -7442,6 +7458,9 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
7442
7458
|
total += frozenSegmentBytes(segment);
|
|
7443
7459
|
}
|
|
7444
7460
|
return total;
|
|
7461
|
+
},
|
|
7462
|
+
readStates() {
|
|
7463
|
+
return subs();
|
|
7445
7464
|
}
|
|
7446
7465
|
};
|
|
7447
7466
|
}
|
|
@@ -8436,4 +8455,4 @@ function fanOutMatchCount(manager, params, language, schema, options) {
|
|
|
8436
8455
|
return count;
|
|
8437
8456
|
}
|
|
8438
8457
|
|
|
8439
|
-
export { FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, MAX_TERM_FREQUENCY, PATTERN_HOST, ValueAffixes, applyPagination, applyProjection, applySorting, availablePatternSearch, buildPartialWordTableFrom, clampRowCount, cloneProjected, collectQueryTermStats, copyVectorsInto, countPartialSpelling, countsWithoutScores, createBooleanIndex, createCompositePartition, createEnumIndex, createFrozenPartialWords, createGeoIndex, createNumericIndex, createPartitionIndex, createPatternWorkMeter, createSurfaceRegistry, decodePageCursor, documentValueBytes, documentWithOwnBuffers, encodeFieldIndexes, encodePageCursor, encodePartialSpellingsOf, encodeSegmentState, everyValueFacetConfig, fanOutMatchCount, fanOutQuery, filterTestCount, fnv1a, fnv1aBytes, fnv1aUint32s, fulltextMatches, isCompositePartition, joinPatternWorkMeter, kWayMerge, mergeFacets, mergePartitionStats, normalizeSort, oversampledFacetConfig, oversampledShardSize, partialSpellingsOfDocuments, partialWordHighlighter, partialWordsToHighlight, partitionStateOf, partitionsIn, patternIndexOf, projectIncludedPaths, projectionKeepsField, pruneStatsToQueryTerms, readProjectedPath, readSortValues, recordValueSize, registerPartialWordSearch, registerPatternSearch, requireMatchingCursor, requirePartialWordSearchFor, requirePatternSearch, requirePatternSearchFor, requireSortableFields, requireWithinResultWindow, resolveProjection, sanitizeGlobalStats, setNestedValue, sortModesOf, sortSignatureEntry, sortSignatureOf, survivorDocumentsOf, undeclaredFieldOnStrictIndex, validatedFilterOf, valueWithOwnBuffers };
|
|
8458
|
+
export { FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, MAX_TERM_FREQUENCY, PATTERN_HOST, ValueAffixes, applyPagination, applyPartitionFiltersBitset, applyProjection, applySorting, availablePatternSearch, buildPartialWordTableFrom, clampRowCount, cloneProjected, collectQueryTermStats, copyVectorsInto, countPartialSpelling, countsWithoutScores, createBooleanIndex, createCompositePartition, createEnumIndex, createFrozenPartialWords, createGeoIndex, createNumericIndex, createPartitionIndex, createPatternWorkMeter, createSurfaceRegistry, decodePageCursor, documentValueBytes, documentWithOwnBuffers, encodeFieldIndexes, encodePageCursor, encodePartialSpellingsOf, encodeSegmentState, everyValueFacetConfig, fanOutMatchCount, fanOutQuery, filterTestCount, fnv1a, fnv1aBytes, fnv1aUint32s, fulltextMatches, isCompositePartition, joinPatternWorkMeter, kWayMerge, mergeFacets, mergePartitionStats, normalizeSort, oversampledFacetConfig, oversampledShardSize, partialSpellingsOfDocuments, partialWordHighlighter, partialWordsToHighlight, partitionStateOf, partitionsIn, patternIndexOf, projectIncludedPaths, projectionKeepsField, pruneStatsToQueryTerms, readProjectedPath, readSortValues, recordValueSize, registerPartialWordSearch, registerPatternSearch, requireMatchingCursor, requirePartialWordSearchFor, requirePatternSearch, requirePatternSearchFor, requireSortableFields, requireWithinResultWindow, resolveProjection, sanitizeGlobalStats, setNestedValue, sortModesOf, sortSignatureEntry, sortSignatureOf, survivorDocumentsOf, undeclaredFieldOnStrictIndex, validatedFilterOf, valueWithOwnBuffers };
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
2
2
|
import { existsSync } from 'fs';
|
|
3
3
|
import { createRequire } from 'module';
|
|
4
4
|
import { join, dirname } from 'path';
|
|
5
5
|
|
|
6
|
-
var NATIVE_CORE_ABI_VERSION =
|
|
6
|
+
var NATIVE_CORE_ABI_VERSION = 4;
|
|
7
7
|
var BINARY_FILE_NAME = "narsil-core.node";
|
|
8
8
|
var MUSL_DIRECTORY = "musl";
|
|
9
9
|
var PLATFORMS_WITH_A_BINARY = /* @__PURE__ */ new Set([
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { osqRecordBytes } from './chunk-37CD3OKS.mjs';
|
|
2
2
|
import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-KEHK2O6P.mjs';
|
|
3
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
4
4
|
|
|
5
5
|
// src/vector/vector-index/dead-bits.ts
|
|
6
6
|
var NO_LIVE_VECTOR_AT_POSITION = -1;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-P36MUFEJ.mjs';
|
|
2
|
-
import { readSharedConfigFields, restoredEmbedding, restoredPartitionConfig } from './chunk-
|
|
3
|
-
import { generateId, getLanguage } from './chunk-MKWUXWQH.mjs';
|
|
2
|
+
import { readSharedConfigFields, restoredEmbedding, restoredPartitionConfig } from './chunk-WLOF6HSP.mjs';
|
|
4
3
|
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-Q5DC6WAB.mjs';
|
|
4
|
+
import { generateId, getLanguage } from './chunk-ZHS44YKF.mjs';
|
|
5
5
|
import { decodeSynonymRules } from './chunk-7X5JST5L.mjs';
|
|
6
6
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
7
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
7
|
+
import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
|
|
8
8
|
import { encode, decode } from '@msgpack/msgpack';
|
|
9
9
|
|
|
10
10
|
// src/distribution/cluster/constants.ts
|
|
@@ -14,7 +14,7 @@ function requireLoadedVectorCode() {
|
|
|
14
14
|
function loadVectorCode() {
|
|
15
15
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
16
16
|
if (loading === null) {
|
|
17
|
-
loading = import('./module-
|
|
17
|
+
loading = import('./module-IUTNXWMV.mjs').then(
|
|
18
18
|
(code) => {
|
|
19
19
|
loaded = code;
|
|
20
20
|
return code;
|