@delali/narsil 0.3.1-next.77 → 0.3.1-next.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-HH6ALOUZ.mjs → chunk-55WEJKWT.mjs} +117 -17
- package/dist/{chunk-DFFQOA7S.mjs → chunk-DFW3U5UW.mjs} +134 -159
- package/dist/{chunk-V3B24XR7.mjs → chunk-IOZPBNSG.mjs} +2 -1
- package/dist/{chunk-D4EF7J3I.mjs → chunk-P2BEMMOD.mjs} +193 -58
- package/dist/{chunk-PXDY5VRT.mjs → chunk-PJWG547Z.mjs} +2 -2
- package/dist/{chunk-356UOTK2.mjs → chunk-PXDQFYD5.mjs} +7 -1
- package/dist/{chunk-G7IP3UDZ.mjs → chunk-R6WBLBTC.mjs} +45 -26
- package/dist/{chunk-PJU6EF22.mjs → chunk-RTR44V2A.mjs} +267 -108
- package/dist/{chunk-NO3ABFLE.mjs → chunk-WFVOBZS4.mjs} +2 -49
- package/dist/{chunk-BT62MUM5.mjs → chunk-XSZTF4ZQ.mjs} +1 -1
- package/dist/{chunk-BT6BI4GV.mjs → chunk-Z3BXCZXS.mjs} +1 -1
- package/dist/{direct-executor-Q7PNCGZB.mjs → direct-executor-WB2HKU4N.mjs} +6 -6
- package/dist/distribution.mjs +10 -10
- package/dist/index.browser.mjs +488 -260
- package/dist/index.mjs +9 -9
- package/dist/partial-words.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +9 -9
- package/dist/server.mjs +6 -6
- package/dist/{thread-YOTNJEVY.mjs → thread-XBDVXKUE.mjs} +6 -6
- package/dist/workers/entry.mjs +3 -3
- package/package.json +7 -7
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, queryBindingOf, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-
|
|
1
|
+
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, queryBindingOf, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-R6WBLBTC.mjs';
|
|
2
2
|
import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
|
|
3
|
-
import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-
|
|
4
|
-
import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, requirePartialWordSearchFor, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, createPartitionManager, buildSegmentPayload,
|
|
5
|
-
import { requirePatternSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, availablePatternSearch, kWayMerge, mergeFacets, PATTERN_HOST, setNestedValue } from './chunk-
|
|
3
|
+
import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-XSZTF4ZQ.mjs';
|
|
4
|
+
import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, requirePartialWordSearchFor, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, NO_LIVE_VECTOR_AT_POSITION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-DFW3U5UW.mjs';
|
|
5
|
+
import { requirePatternSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, availablePatternSearch, kWayMerge, mergeFacets, PATTERN_HOST, setNestedValue } from './chunk-P2BEMMOD.mjs';
|
|
6
6
|
import { detectRuntime, getLanguage } from './chunk-4MQ5F4IL.mjs';
|
|
7
|
-
import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, BATCH_CHUNK_SIZE, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, EXPORT_CHUNK_BYTES } from './chunk-
|
|
7
|
+
import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, BATCH_CHUNK_SIZE, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, EXPORT_CHUNK_BYTES } from './chunk-IOZPBNSG.mjs';
|
|
8
8
|
import { releaseLocksHeldBy } from './chunk-AMGDG4ZJ.mjs';
|
|
9
9
|
import { requireSearchBackendSettings } from './chunk-2E35UM3G.mjs';
|
|
10
10
|
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES, PATTERN_TEST_OPERATORS, PATTERN_EXPRESSION_OPERATORS, MATCHER_CODE_POINT_BYTES, MATCHER_SIMULATION_BYTES_PER_POSITION, MATCHER_STATE_OVERHEAD_BYTES } from './chunk-4344I4MR.mjs';
|
|
11
|
-
import { getNestedValue, parseVectorDimension } from './chunk-
|
|
12
|
-
import { extractVectorFieldsFromSchema, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, raiseEarlierReleaseGraphSetting, flattenSchema, isSingleStringFieldType, requireValidFieldTypes, encodeDocument, VECTOR_PATTERN, schemaFieldsOf, MAX_DOCUMENT_NESTING_DEPTH, isQuantizationMode, isStorageMode, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-
|
|
11
|
+
import { getNestedValue, parseVectorDimension } from './chunk-WFVOBZS4.mjs';
|
|
12
|
+
import { extractVectorFieldsFromSchema, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, raiseEarlierReleaseGraphSetting, partialFieldsOf, flattenSchema, isSingleStringFieldType, requireValidFieldTypes, encodeDocument, VECTOR_PATTERN, schemaFieldsOf, MAX_DOCUMENT_NESTING_DEPTH, isQuantizationMode, isStorageMode, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-PXDQFYD5.mjs';
|
|
13
13
|
import { MIN_M } from './chunk-KEHK2O6P.mjs';
|
|
14
14
|
import { compareCodePoints, foldedCodePointCount, codePointCount } from './chunk-IBR6634K.mjs';
|
|
15
15
|
import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-POWOTUKJ.mjs';
|
|
@@ -3505,7 +3505,7 @@ function selectionOfMask(segment, keep) {
|
|
|
3505
3505
|
}
|
|
3506
3506
|
function frozenLiveTail(live, shared) {
|
|
3507
3507
|
if (live.count() === 0) return null;
|
|
3508
|
-
const payload =
|
|
3508
|
+
const payload = live.encodeSegment();
|
|
3509
3509
|
const documents = survivorDocumentsOf(partitionStateOf(live), payload);
|
|
3510
3510
|
if (shared) {
|
|
3511
3511
|
const snapshot = freezeSegmentShared(payload, documents);
|
|
@@ -4325,6 +4325,23 @@ function requireEntriesAscend(blob, offsets, order, name) {
|
|
|
4325
4325
|
}
|
|
4326
4326
|
}
|
|
4327
4327
|
}
|
|
4328
|
+
function requireSpellingEntriesAscend(blob, offsets, tokens, name) {
|
|
4329
|
+
for (let at = 1; at < tokens.length; at++) {
|
|
4330
|
+
const order = compareEntries(blob, offsets, at - 1, at) || tokens[at - 1] - tokens[at];
|
|
4331
|
+
if (order >= 0) corrupt2(`${name} fails to ascend strictly at entry ${at}`, { field: name, at });
|
|
4332
|
+
}
|
|
4333
|
+
}
|
|
4334
|
+
function requireBelow(values, limit, name) {
|
|
4335
|
+
for (let at = 0; at < values.length; at++) {
|
|
4336
|
+
if (values[at] >= limit)
|
|
4337
|
+
corrupt2(`${name} holds ${values[at]} at entry ${at}, at or above ${limit}`, { field: name, at });
|
|
4338
|
+
}
|
|
4339
|
+
}
|
|
4340
|
+
function requireNonZero(values, name) {
|
|
4341
|
+
for (let at = 0; at < values.length; at++) {
|
|
4342
|
+
if (values[at] === 0) corrupt2(`${name} holds 0 at entry ${at}`, { field: name, at });
|
|
4343
|
+
}
|
|
4344
|
+
}
|
|
4328
4345
|
function requirePermutation(order, name) {
|
|
4329
4346
|
const seen = new Uint8Array(order.length);
|
|
4330
4347
|
for (let at = 0; at < order.length; at++) {
|
|
@@ -4464,6 +4481,34 @@ function decodeSurfaceForms(raw, context) {
|
|
|
4464
4481
|
requireUtf8Entries(blob, offsets, "surfaceForms.text");
|
|
4465
4482
|
return counts.length === 0 ? null : { blob, offsets, counts };
|
|
4466
4483
|
}
|
|
4484
|
+
function decodePartialWords(raw, tokenCount, context) {
|
|
4485
|
+
if (raw === null || raw === void 0) return null;
|
|
4486
|
+
const column = requireRecord(raw, "partialWords");
|
|
4487
|
+
const fields = requireStrings(column.fields, "partialWords.fields");
|
|
4488
|
+
const spellings = bytesFrom(requireBytes(column.text, "partialWords.text", 1), context.shared);
|
|
4489
|
+
const spellingOffsets = uint32Column(column, "offsets", "partialWords", context);
|
|
4490
|
+
const spellingTokens = uint32Column(column, "tokens", "partialWords", context);
|
|
4491
|
+
const spellingCounts = uint32Column(column, "counts", "partialWords", context);
|
|
4492
|
+
const entries = spellingTokens.length;
|
|
4493
|
+
requireCount(spellingOffsets.length, entries + 1, "partialWords.offsets");
|
|
4494
|
+
requireCount(spellingCounts.length, entries, "partialWords.counts");
|
|
4495
|
+
requireOffsets(spellingOffsets, spellings.length, "partialWords.offsets");
|
|
4496
|
+
requireUtf8Entries(spellings, spellingOffsets, "partialWords.text");
|
|
4497
|
+
requireBelow(spellingTokens, tokenCount, "partialWords.tokens");
|
|
4498
|
+
requireNonZero(spellingCounts, "partialWords.counts");
|
|
4499
|
+
requireSpellingEntriesAscend(spellings, spellingOffsets, spellingTokens, "partialWords");
|
|
4500
|
+
const runs = uint32sFrom(requireBytes(column.runs, "partialWords.runs", 4 * CODE_POINTS_PER_RUN), context.shared);
|
|
4501
|
+
requireRunsAscend(runs, "partialWords.runs");
|
|
4502
|
+
const runOffsets = uint32Column(column, "runOffsets", "partialWords", context);
|
|
4503
|
+
const runEntries = uint32Column(column, "runEntries", "partialWords", context);
|
|
4504
|
+
requireCount(runOffsets.length, runs.length / CODE_POINTS_PER_RUN + 1, "partialWords.runOffsets");
|
|
4505
|
+
requireOffsets(runOffsets, runEntries.length, "partialWords.runOffsets");
|
|
4506
|
+
requireBelow(runEntries, entries, "partialWords.runEntries");
|
|
4507
|
+
for (let run = 0; run + 1 < runOffsets.length; run++) {
|
|
4508
|
+
requireStrictlyAscending(runEntries, runOffsets[run], runOffsets[run + 1], "partialWords.runEntries");
|
|
4509
|
+
}
|
|
4510
|
+
return { fields, spellings, spellingOffsets, spellingTokens, spellingCounts, runs, runOffsets, runEntries };
|
|
4511
|
+
}
|
|
4467
4512
|
|
|
4468
4513
|
// src/serialization/keyword-segment/types.ts
|
|
4469
4514
|
var KEYWORD_SEGMENT_PAYLOAD_VERSION = 1;
|
|
@@ -4584,12 +4629,13 @@ function decodeKeywordSegment(payload, shared) {
|
|
|
4584
4629
|
corrupt2(`fieldNames holds ${fieldNames.length} names, above the limit of ${MAX_SEGMENT_FIELD_NAMES}`);
|
|
4585
4630
|
}
|
|
4586
4631
|
const context = { documentCount, shared };
|
|
4632
|
+
const postings = decodePostings(raw, fieldNames, context);
|
|
4587
4633
|
const arrays = {
|
|
4588
4634
|
documentCount,
|
|
4589
4635
|
fieldNames,
|
|
4590
4636
|
idTable: decodeIds(raw, context),
|
|
4591
4637
|
documentTable: decodeDocuments(raw, schema, context),
|
|
4592
|
-
...
|
|
4638
|
+
...postings,
|
|
4593
4639
|
...decodeFieldLengths(raw.fieldLengths, context),
|
|
4594
4640
|
numeric: decodeValueColumns(raw.numeric, "numeric", context),
|
|
4595
4641
|
sizes: decodeValueColumns(raw.sizes, "sizes", context),
|
|
@@ -4597,7 +4643,8 @@ function decodeKeywordSegment(payload, shared) {
|
|
|
4597
4643
|
enums: decodeEnumColumns(raw.enum, context),
|
|
4598
4644
|
geo: decodeGeopointColumns(raw.geopoint, context),
|
|
4599
4645
|
pattern: decodePatternColumns(raw.pattern, context),
|
|
4600
|
-
surfaceTable: decodeSurfaceForms(raw.surfaceForms, context)
|
|
4646
|
+
surfaceTable: decodeSurfaceForms(raw.surfaceForms, context),
|
|
4647
|
+
partialWords: decodePartialWords(raw.partialWords, postings.tokenTable.offsets.length - 1, context)
|
|
4601
4648
|
};
|
|
4602
4649
|
return { schema, tombstones, arrays };
|
|
4603
4650
|
}
|
|
@@ -4721,7 +4768,7 @@ function postingsInTokenOrder(arrays) {
|
|
|
4721
4768
|
}
|
|
4722
4769
|
|
|
4723
4770
|
// src/serialization/keyword-segment/encode.ts
|
|
4724
|
-
var TOP_LEVEL_ENTRIES =
|
|
4771
|
+
var TOP_LEVEL_ENTRIES = 28;
|
|
4725
4772
|
function writeFieldLengths(out, arrays) {
|
|
4726
4773
|
out.array(arrays.fieldLengthNames.length);
|
|
4727
4774
|
for (let f = 0; f < arrays.fieldLengthNames.length; f++) {
|
|
@@ -4783,6 +4830,21 @@ function writeSurfaceForms(out, arrays) {
|
|
|
4783
4830
|
out.key("offsets").binary(littleEndianBytes(table.offsets));
|
|
4784
4831
|
out.key("counts").binary(littleEndianBytes(table.counts));
|
|
4785
4832
|
}
|
|
4833
|
+
function writePartialWords(out, arrays) {
|
|
4834
|
+
const table = arrays.partialWords;
|
|
4835
|
+
if (table === null) {
|
|
4836
|
+
out.value(null);
|
|
4837
|
+
return;
|
|
4838
|
+
}
|
|
4839
|
+
out.map(8).key("fields").value(table.fields);
|
|
4840
|
+
out.key("text").binary(table.spellings);
|
|
4841
|
+
out.key("offsets").binary(littleEndianBytes(table.spellingOffsets));
|
|
4842
|
+
out.key("tokens").binary(littleEndianBytes(table.spellingTokens));
|
|
4843
|
+
out.key("counts").binary(littleEndianBytes(table.spellingCounts));
|
|
4844
|
+
out.key("runs").binary(littleEndianBytes(table.runs));
|
|
4845
|
+
out.key("runOffsets").binary(littleEndianBytes(table.runOffsets));
|
|
4846
|
+
out.key("runEntries").binary(littleEndianBytes(table.runEntries));
|
|
4847
|
+
}
|
|
4786
4848
|
function encodeKeywordSegment(contents) {
|
|
4787
4849
|
const { arrays } = contents;
|
|
4788
4850
|
const postings = postingsInTokenOrder(arrays);
|
|
@@ -4823,6 +4885,8 @@ function encodeKeywordSegment(contents) {
|
|
|
4823
4885
|
writePatternColumns(out, arrays.pattern);
|
|
4824
4886
|
out.key("surfaceForms");
|
|
4825
4887
|
writeSurfaceForms(out, arrays);
|
|
4888
|
+
out.key("partialWords");
|
|
4889
|
+
writePartialWords(out, arrays);
|
|
4826
4890
|
return out;
|
|
4827
4891
|
}
|
|
4828
4892
|
|
|
@@ -4901,7 +4965,7 @@ function fixedBytesOf(segment, chargesTokens) {
|
|
|
4901
4965
|
const surfaces = segment.sharedSnapshot?.surfaceTable ?? null;
|
|
4902
4966
|
if (surfaces !== null) bytes += surfaces.blob.byteLength + surfaces.offsets.byteLength + surfaces.counts.byteLength;
|
|
4903
4967
|
else bytes += messagePackUpperBoundBytes(arrays.readSurfaceForms());
|
|
4904
|
-
return bytes;
|
|
4968
|
+
return bytes + segment.partialWords.bytes();
|
|
4905
4969
|
}
|
|
4906
4970
|
function keptDocumentsOf(selection, chargesTokens) {
|
|
4907
4971
|
const { segment } = selection;
|
|
@@ -5030,16 +5094,26 @@ function emptyPayload() {
|
|
|
5030
5094
|
sizes: []
|
|
5031
5095
|
};
|
|
5032
5096
|
}
|
|
5033
|
-
function
|
|
5097
|
+
function noSpellingsOf(analysis) {
|
|
5098
|
+
if (analysis === null || analysis.fields.length === 0) return null;
|
|
5099
|
+
return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
|
|
5100
|
+
}
|
|
5101
|
+
function contentsOfGroup(group, request, tombstones) {
|
|
5102
|
+
const { schema, partialWordAnalysis } = request;
|
|
5034
5103
|
if (group === null) {
|
|
5104
|
+
const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
|
|
5035
5105
|
return {
|
|
5036
5106
|
schema,
|
|
5037
5107
|
tombstones,
|
|
5038
|
-
arrays: segmentArraysOf(
|
|
5108
|
+
arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
|
|
5039
5109
|
};
|
|
5040
5110
|
}
|
|
5041
|
-
const merged = mergeSegmentContents(group.selections);
|
|
5042
|
-
return {
|
|
5111
|
+
const merged = mergeSegmentContents(group.selections, partialWordAnalysis);
|
|
5112
|
+
return {
|
|
5113
|
+
schema,
|
|
5114
|
+
tombstones,
|
|
5115
|
+
arrays: segmentArraysOf(merged.payload, merged.documents, merged.reusedPartialWords)
|
|
5116
|
+
};
|
|
5043
5117
|
}
|
|
5044
5118
|
function requireRoomForAnotherSegment(state) {
|
|
5045
5119
|
if (state.segments.length < MAX_SEGMENTS_PER_PARTITION) return;
|
|
@@ -5052,7 +5126,7 @@ function requireRoomForAnotherSegment(state) {
|
|
|
5052
5126
|
async function writeGroup(state, group) {
|
|
5053
5127
|
const { request } = state;
|
|
5054
5128
|
const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
|
|
5055
|
-
const contents = contentsOfGroup(group, request
|
|
5129
|
+
const contents = contentsOfGroup(group, request, tombstones);
|
|
5056
5130
|
const encoded = encodeKeywordSegment(contents);
|
|
5057
5131
|
if (HEADER_SIZE + encoded.byteLength > MAX_FILE_BYTES && group !== null && group.documentCount > 1) {
|
|
5058
5132
|
const [first, second] = halvesOf(group);
|
|
@@ -5168,6 +5242,7 @@ async function compactPartitionSegments(input) {
|
|
|
5168
5242
|
partitionId: input.partitionId,
|
|
5169
5243
|
nextSegmentId: input.nextSegmentId,
|
|
5170
5244
|
schema: input.schema,
|
|
5245
|
+
partialWordAnalysis: input.partialWordAnalysis,
|
|
5171
5246
|
selections: frozen.map((segment) => ({ segment })),
|
|
5172
5247
|
tombstones: retainedTombstones(frozen, contents),
|
|
5173
5248
|
mergesIntoOneFile: true
|
|
@@ -5183,6 +5258,27 @@ async function compactPartitionSegments(input) {
|
|
|
5183
5258
|
}
|
|
5184
5259
|
|
|
5185
5260
|
// src/persistence/durability/segment/write.ts
|
|
5261
|
+
function partialWordAnalysisOf(config) {
|
|
5262
|
+
const fields = partialFieldsOf(config.schema);
|
|
5263
|
+
if (fields.length === 0) return null;
|
|
5264
|
+
let resolved = null;
|
|
5265
|
+
const resolve = () => {
|
|
5266
|
+
resolved ??= {
|
|
5267
|
+
language: getLanguage(config.language ?? "english"),
|
|
5268
|
+
options: resolvePartitionInsertOptions(config, resolveIndexAnalysis(config))
|
|
5269
|
+
};
|
|
5270
|
+
return resolved;
|
|
5271
|
+
};
|
|
5272
|
+
return {
|
|
5273
|
+
fields,
|
|
5274
|
+
get language() {
|
|
5275
|
+
return resolve().language;
|
|
5276
|
+
},
|
|
5277
|
+
get options() {
|
|
5278
|
+
return resolve().options;
|
|
5279
|
+
}
|
|
5280
|
+
};
|
|
5281
|
+
}
|
|
5186
5282
|
function resolveCompactionThreshold(configured) {
|
|
5187
5283
|
if (!Number.isInteger(configured) || configured <= 0 || configured > MAX_SEGMENTS_PER_PARTITION) {
|
|
5188
5284
|
return DEFAULT_COMPACTION_THRESHOLD;
|
|
@@ -5202,6 +5298,7 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
|
|
|
5202
5298
|
partitionId,
|
|
5203
5299
|
nextSegmentId: priorPartition?.nextSegmentId ?? 0,
|
|
5204
5300
|
schema: context.input.metadata.schema,
|
|
5301
|
+
partialWordAnalysis: context.partialWordAnalysis,
|
|
5205
5302
|
selections: whole.selections,
|
|
5206
5303
|
tombstones: []
|
|
5207
5304
|
});
|
|
@@ -5224,6 +5321,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
5224
5321
|
partitionId,
|
|
5225
5322
|
nextSegmentId,
|
|
5226
5323
|
schema: context.input.metadata.schema,
|
|
5324
|
+
partialWordAnalysis: context.partialWordAnalysis,
|
|
5227
5325
|
selections: built.selections,
|
|
5228
5326
|
tombstones: built.tombstones
|
|
5229
5327
|
});
|
|
@@ -5242,6 +5340,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
|
|
|
5242
5340
|
indexName: context.indexName,
|
|
5243
5341
|
partitionId,
|
|
5244
5342
|
schema: context.input.metadata.schema,
|
|
5343
|
+
partialWordAnalysis: context.partialWordAnalysis,
|
|
5245
5344
|
segments,
|
|
5246
5345
|
nextSegmentId,
|
|
5247
5346
|
compactionThreshold: context.compactionThreshold
|
|
@@ -5268,6 +5367,7 @@ async function writeCheckpointSegments(input) {
|
|
|
5268
5367
|
input,
|
|
5269
5368
|
indexName: input.metadata.indexName,
|
|
5270
5369
|
config,
|
|
5370
|
+
partialWordAnalysis: partialWordAnalysisOf(config),
|
|
5271
5371
|
vectorFieldPaths: new Set(extractVectorFieldsFromSchema(config.schema).keys()),
|
|
5272
5372
|
compactionThreshold: resolveCompactionThreshold(input.compactionThreshold)
|
|
5273
5373
|
};
|