@delali/narsil 0.3.1-next.87 → 0.3.1-next.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-BBDAYUCO.mjs → chunk-3GZQKGVQ.mjs} +25 -9
- package/dist/browser/{chunk-3BXQPIXK.mjs → chunk-JACCR3BJ.mjs} +3 -3
- package/dist/browser/{chunk-TBMNPGFO.mjs → chunk-RAHUQOJL.mjs} +2074 -881
- package/dist/browser/{durability-wiring-7RUN7LNS.mjs → durability-wiring-VSUBLUDK.mjs} +3 -3
- package/dist/browser/{snapshot-A42DRERF.mjs → snapshot-5MXBEDXF.mjs} +3 -3
- package/dist/browser/{write-4U6Q26TJ.mjs → write-6SBYTS62.mjs} +2 -2
- package/dist/{chunk-PSUJCOSE.mjs → chunk-A3UB2MXL.mjs} +172 -122
- package/dist/{chunk-JFJBA3PD.mjs → chunk-CL4OM7GB.mjs} +3 -3
- package/dist/{chunk-AXDAS2VA.mjs → chunk-DCK4P5BI.mjs} +28 -30
- package/dist/{chunk-XBXKPWMR.mjs → chunk-EMXAYPTQ.mjs} +1735 -651
- package/dist/{chunk-IKST36MA.mjs → chunk-MWJIAREY.mjs} +4 -4
- package/dist/{chunk-T7DMUVTL.mjs → chunk-QAMUC2BJ.mjs} +27 -11
- package/dist/{chunk-AQU2QHSY.mjs → chunk-R6BMMQXH.mjs} +114 -45
- package/dist/{chunk-K4OWQ2M3.mjs → chunk-S4LQDSUL.mjs} +3 -3
- package/dist/{chunk-MKK4OZ5W.mjs → chunk-V7YUSKKZ.mjs} +3 -3
- package/dist/{chunk-WQSHDZ7Z.mjs → chunk-VY4DXZ5P.mjs} +1 -1
- package/dist/{chunk-SYRFSF6V.mjs → chunk-X4SEXL4U.mjs} +49 -11
- package/dist/{direct-executor-LD6JW57R.mjs → direct-executor-EGESD26S.mjs} +3 -3
- package/dist/distribution.mjs +14 -27
- package/dist/{durability-wiring-5S32KDDC.mjs → durability-wiring-ZLRQU6FG.mjs} +10 -10
- package/dist/index.browser.mjs +1 -1
- package/dist/index.mjs +5 -5
- package/dist/partial-words.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +8 -8
- package/dist/{snapshot-6BYSUPWL.mjs → snapshot-22CUEYYW.mjs} +10 -10
- package/dist/{thread-GS65E6RM.mjs → thread-ZP3UMIZT.mjs} +3 -3
- package/dist/workers/entry.mjs +3 -3
- package/dist/{write-HLEZTL2L.mjs → write-PXDKPG2R.mjs} +6 -6
- package/package.json +13 -8
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-
|
|
2
|
-
import { VERSION, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment,
|
|
1
|
+
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes } from './chunk-JACCR3BJ.mjs';
|
|
2
|
+
import { VERSION, flattenSchema, packChecksummedEnvelope, openChecksummedEnvelope, requireValidFieldTypes, isCompositePartition, crc32, reconstructSchemaFromMetadata, extractVectorFieldsFromSchema, readHeader, HEADER_SIZE, engineVersionParts, writeHeader, partialFieldsOf, getLanguage, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, VECTOR_PATTERN, isQuantizationMode, isStorageMode, survivorDocumentsOf, partitionStateOf, freezeSegmentShared, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, fnv1a, selectionKeeps, decodeMapWithoutFields, prepareDocumentVectors, messagePackUpperBoundBytes, segmentArraysOf, mergeSegmentContents, resolveIndexAnalysis } from './chunk-RAHUQOJL.mjs';
|
|
3
3
|
import { decodeVectorGraphPartPayload, assembleGraphParts, decodeVectorFilePayload, countDead, VECTOR_FILE_MAX_VECTORS, deadBytesFor } from './chunk-NQW2YXEX.mjs';
|
|
4
4
|
import { compareCodePoints, NarsilError, ErrorCodes, MAX_FILE_BYTES, VALID_HNSW_METRICS } from './chunk-FHFFWLEK.mjs';
|
|
5
5
|
import { decode, encode } from '@msgpack/msgpack';
|
|
@@ -2176,7 +2176,7 @@ function noSpellingsOf(analysis) {
|
|
|
2176
2176
|
return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
|
|
2177
2177
|
}
|
|
2178
2178
|
function contentsOfGroup(group, request, tombstones) {
|
|
2179
|
-
const { schema, partialWordAnalysis } = request;
|
|
2179
|
+
const { schema, partialWordAnalysis, textAnalysis } = request;
|
|
2180
2180
|
if (group === null) {
|
|
2181
2181
|
const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
|
|
2182
2182
|
return {
|
|
@@ -2185,7 +2185,7 @@ function contentsOfGroup(group, request, tombstones) {
|
|
|
2185
2185
|
arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
|
|
2186
2186
|
};
|
|
2187
2187
|
}
|
|
2188
|
-
const merged = mergeSegmentContents(group.selections, partialWordAnalysis);
|
|
2188
|
+
const merged = mergeSegmentContents(group.selections, partialWordAnalysis, textAnalysis);
|
|
2189
2189
|
return {
|
|
2190
2190
|
schema,
|
|
2191
2191
|
tombstones,
|
|
@@ -2320,6 +2320,7 @@ async function compactPartitionSegments(input) {
|
|
|
2320
2320
|
nextSegmentId: input.nextSegmentId,
|
|
2321
2321
|
schema: input.schema,
|
|
2322
2322
|
partialWordAnalysis: input.partialWordAnalysis,
|
|
2323
|
+
textAnalysis: input.textAnalysis,
|
|
2323
2324
|
selections: frozen.map((segment) => ({ segment })),
|
|
2324
2325
|
tombstones: retainedTombstones(frozen, contents),
|
|
2325
2326
|
mergesIntoOneFile: true
|
|
@@ -2335,9 +2336,7 @@ async function compactPartitionSegments(input) {
|
|
|
2335
2336
|
}
|
|
2336
2337
|
|
|
2337
2338
|
// src/persistence/durability/segment/write.ts
|
|
2338
|
-
function
|
|
2339
|
-
const fields = partialFieldsOf(config.schema);
|
|
2340
|
-
if (fields.length === 0) return null;
|
|
2339
|
+
function textAnalysisOf(config) {
|
|
2341
2340
|
let resolved = null;
|
|
2342
2341
|
const resolve = () => {
|
|
2343
2342
|
resolved ??= {
|
|
@@ -2347,7 +2346,6 @@ function partialWordAnalysisOf(config) {
|
|
|
2347
2346
|
return resolved;
|
|
2348
2347
|
};
|
|
2349
2348
|
return {
|
|
2350
|
-
fields,
|
|
2351
2349
|
get language() {
|
|
2352
2350
|
return resolve().language;
|
|
2353
2351
|
},
|
|
@@ -2356,6 +2354,19 @@ function partialWordAnalysisOf(config) {
|
|
|
2356
2354
|
}
|
|
2357
2355
|
};
|
|
2358
2356
|
}
|
|
2357
|
+
function partialWordAnalysisOf(config, text) {
|
|
2358
|
+
const fields = partialFieldsOf(config.schema);
|
|
2359
|
+
if (fields.length === 0) return null;
|
|
2360
|
+
return {
|
|
2361
|
+
fields,
|
|
2362
|
+
get language() {
|
|
2363
|
+
return text.language;
|
|
2364
|
+
},
|
|
2365
|
+
get options() {
|
|
2366
|
+
return text.options;
|
|
2367
|
+
}
|
|
2368
|
+
};
|
|
2369
|
+
}
|
|
2359
2370
|
function resolveCompactionThreshold(configured) {
|
|
2360
2371
|
if (!Number.isInteger(configured) || configured <= 0 || configured > MAX_SEGMENTS_PER_PARTITION) {
|
|
2361
2372
|
return DEFAULT_COMPACTION_THRESHOLD;
|
|
@@ -2376,6 +2387,7 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
|
|
|
2376
2387
|
nextSegmentId: priorPartition?.nextSegmentId ?? 0,
|
|
2377
2388
|
schema: context.input.metadata.schema,
|
|
2378
2389
|
partialWordAnalysis: context.partialWordAnalysis,
|
|
2390
|
+
textAnalysis: context.textAnalysis,
|
|
2379
2391
|
selections: whole.selections,
|
|
2380
2392
|
tombstones: []
|
|
2381
2393
|
});
|
|
@@ -2399,6 +2411,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
2399
2411
|
nextSegmentId,
|
|
2400
2412
|
schema: context.input.metadata.schema,
|
|
2401
2413
|
partialWordAnalysis: context.partialWordAnalysis,
|
|
2414
|
+
textAnalysis: context.textAnalysis,
|
|
2402
2415
|
selections: built.selections,
|
|
2403
2416
|
tombstones: built.tombstones
|
|
2404
2417
|
});
|
|
@@ -2418,6 +2431,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
2418
2431
|
partitionId,
|
|
2419
2432
|
schema: context.input.metadata.schema,
|
|
2420
2433
|
partialWordAnalysis: context.partialWordAnalysis,
|
|
2434
|
+
textAnalysis: context.textAnalysis,
|
|
2421
2435
|
segments,
|
|
2422
2436
|
nextSegmentId,
|
|
2423
2437
|
compactionThreshold: context.compactionThreshold
|
|
@@ -2440,11 +2454,13 @@ function carryForwardUncheckpointedPartitions(prior, targets, partitions, checkp
|
|
|
2440
2454
|
}
|
|
2441
2455
|
async function writeCheckpointSegments(input) {
|
|
2442
2456
|
const config = reconstructSchemaFromMetadata(input.metadata);
|
|
2457
|
+
const textAnalysis = textAnalysisOf(config);
|
|
2443
2458
|
const context = {
|
|
2444
2459
|
input,
|
|
2445
2460
|
indexName: input.metadata.indexName,
|
|
2446
2461
|
config,
|
|
2447
|
-
partialWordAnalysis: partialWordAnalysisOf(config),
|
|
2462
|
+
partialWordAnalysis: partialWordAnalysisOf(config, textAnalysis),
|
|
2463
|
+
textAnalysis,
|
|
2448
2464
|
vectorFieldPaths: new Set(extractVectorFieldsFromSchema(config.schema).keys()),
|
|
2449
2465
|
compactionThreshold: resolveCompactionThreshold(input.compactionThreshold)
|
|
2450
2466
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-
|
|
1
|
+
import { removeDocumentVectors, prepareDocumentVectors, insertDocumentVectors, getNestedValue, requireValidFieldTypes, VECTOR_PATTERN } from './chunk-RAHUQOJL.mjs';
|
|
2
2
|
import { NarsilError, ErrorCodes } from './chunk-FHFFWLEK.mjs';
|
|
3
3
|
import { decode } from '@msgpack/msgpack';
|
|
4
4
|
|
|
@@ -28,12 +28,12 @@ function applyIndexEntry(entry, manager, vectorFieldPaths, vecIndexes) {
|
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
30
|
}
|
|
31
|
-
function applyDeleteEntry(entry, manager, vecIndexes) {
|
|
31
|
+
function applyDeleteEntry(entry, manager, vecIndexes, removedSpellings) {
|
|
32
32
|
removeDocumentVectors(entry.documentId, vecIndexes);
|
|
33
33
|
if (!manager.has(entry.documentId)) {
|
|
34
34
|
return;
|
|
35
35
|
}
|
|
36
|
-
manager.remove(entry.documentId);
|
|
36
|
+
manager.remove(entry.documentId, removedSpellings);
|
|
37
37
|
}
|
|
38
38
|
function restoreVectorFields(document, vectorFieldPaths) {
|
|
39
39
|
for (const fieldPath of vectorFieldPaths) {
|