@delali/narsil 0.3.1-next.77 → 0.3.1-next.78

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -516,6 +516,7 @@ var MAX_VECTOR_DIMENSION = 4096;
516
516
  var MAX_STORED_VECTOR_DIMENSION = 65535;
517
517
  var MAX_TEXT_FIELDS = 65536;
518
518
  var MAX_DOCUMENT_BYTES = 16777216;
519
+ var PARSED_FIELD_TYPE_CACHE_ENTRIES = 1024;
519
520
 
520
521
  // src/schema/validator/shared.ts
521
522
  var VECTOR_PATTERN = /^vector\[(\d+)]$/;
@@ -606,10 +607,15 @@ function parseFieldType(type, path) {
606
607
  dimension
607
608
  };
608
609
  }
610
+ var parsedTypesByName = /* @__PURE__ */ new Map();
609
611
  function parsedTypeOf(type) {
610
612
  if (type === void 0) return void 0;
613
+ const known = parsedTypesByName.get(type);
614
+ if (known !== void 0) return known;
611
615
  try {
612
- return parseFieldType(type, "");
616
+ const parsed = Object.freeze(parseFieldType(type, ""));
617
+ if (parsedTypesByName.size < PARSED_FIELD_TYPE_CACHE_ENTRIES) parsedTypesByName.set(type, parsed);
618
+ return parsed;
613
619
  } catch {
614
620
  return void 0;
615
621
  }
@@ -828,7 +834,9 @@ var EMPTY_FROZEN_SLOT = {
828
834
  readerFor: () => null,
829
835
  spellings: () => null,
830
836
  bytes: () => 0,
831
- afterRemoval: () => void 0
837
+ afterRemoval: () => void 0,
838
+ tableOfEveryDocument: () => null,
839
+ countKeptSpellings: () => void 0
832
840
  };
833
841
  function createGrowingPartialWords(view) {
834
842
  return availablePartialWordSearch()?.createGrowingSlot(view) ?? EMPTY_GROWING_SLOT;
@@ -1289,53 +1297,6 @@ function tokenize(text, language, options) {
1289
1297
  });
1290
1298
  return { tokens, originalTokens, surfaces };
1291
1299
  }
1292
- function* tokenizeIterator(text, language, options) {
1293
- const {
1294
- stem: stem2 = true,
1295
- removeStopWords = true,
1296
- removeDiacritics = false,
1297
- collectSurfaces = false,
1298
- customTokenizer,
1299
- stopWordOverride
1300
- } = options ?? {};
1301
- if (customTokenizer) {
1302
- const customResult = customTokenizer.tokenize(text);
1303
- for (const entry of customResult) {
1304
- yield entry;
1305
- }
1306
- return;
1307
- }
1308
- const normalized = normalizeForSplitting(text);
1309
- const rawParts = splitText(normalized, language);
1310
- const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
1311
- const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
1312
- const stemsTokens = stem2 && language.stemmer !== null;
1313
- const wantSurfaces = collectSurfaces && stemsTokens;
1314
- const stopWords2 = removeStopWords ? resolveStopWords(language, stopWordOverride) : /* @__PURE__ */ new Set();
1315
- const stripPossessives = language.tokenizer?.stripPossessive ?? false;
1316
- let position = 0;
1317
- for (const part of rawParts) {
1318
- const candidate = stripPossessives ? stripPossessive(part) : part;
1319
- if (candidate.length < minLength) {
1320
- position++;
1321
- continue;
1322
- }
1323
- if (stopWords2.has(candidate)) {
1324
- position++;
1325
- continue;
1326
- }
1327
- const processed = transformToken(candidate, language, stem2, effectiveDiacritics);
1328
- if (processed.length > 0) {
1329
- if (wantSurfaces) {
1330
- const surface = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
1331
- yield { token: processed, position, surface: surface === processed ? void 0 : surface };
1332
- } else {
1333
- yield { token: processed, position };
1334
- }
1335
- }
1336
- position++;
1337
- }
1338
- }
1339
1300
 
1340
1301
  // src/languages/snowball/base-stemmer.ts
1341
1302
  var BaseStemmer = class {
@@ -3297,6 +3258,9 @@ function createDocumentStore() {
3297
3258
  },
3298
3259
  toInternal(externalId) {
3299
3260
  return forwardMap.get(externalId);
3261
+ },
3262
+ isLive(internalId) {
3263
+ return reverseMap[internalId] !== void 0;
3300
3264
  }
3301
3265
  };
3302
3266
  return {
@@ -5274,8 +5238,25 @@ function createNumericIndex() {
5274
5238
  };
5275
5239
  }
5276
5240
 
5241
+ // src/core/partition/partial-words/constants.ts
5242
+ var PARTIAL_WORD_RUN_CODE_POINTS = 3;
5243
+
5277
5244
  // src/core/partition/partial-words/spellings.ts
5245
+ var LAST_BASIC_PLANE_CODE_POINT2 = 65535;
5246
+ function codePointCountOf(text) {
5247
+ let count = 0;
5248
+ for (let index = 0; index < text.length; count++) {
5249
+ index += (text.codePointAt(index) ?? 0) > LAST_BASIC_PLANE_CODE_POINT2 ? 2 : 1;
5250
+ }
5251
+ return count;
5252
+ }
5253
+ function holdsRun(spelling) {
5254
+ if (spelling.length > 2 * PARTIAL_WORD_RUN_CODE_POINTS) return true;
5255
+ if (spelling.length <= PARTIAL_WORD_RUN_CODE_POINTS) return false;
5256
+ return codePointCountOf(spelling) > PARTIAL_WORD_RUN_CODE_POINTS;
5257
+ }
5278
5258
  function countSpelling(counts, token, spelling, change) {
5259
+ if (!holdsRun(spelling)) return;
5279
5260
  let byToken = counts.get(token);
5280
5261
  if (byToken === void 0) {
5281
5262
  byToken = /* @__PURE__ */ new Map();
@@ -5296,11 +5277,12 @@ function textsOf(value) {
5296
5277
  }
5297
5278
  function countDocumentSpellings(counts, document2, analysis, direction) {
5298
5279
  const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
5280
+ const countOccurrence2 = (token, _position, surface) => {
5281
+ countSpelling(counts, token, surface ?? token, direction);
5282
+ };
5299
5283
  for (const field of analysis.fields) {
5300
5284
  for (const text of textsOf(getNestedValue(document2, field))) {
5301
- for (const occurrence of tokenizeIterator(text, analysis.language, tokenizing)) {
5302
- countSpelling(counts, occurrence.token, occurrence.surface ?? occurrence.token, direction);
5303
- }
5285
+ visitTokens(text, analysis.language, tokenizing, countOccurrence2);
5304
5286
  }
5305
5287
  }
5306
5288
  }
@@ -5320,6 +5302,13 @@ function encodePartialSpellings(fields, counts) {
5320
5302
  }
5321
5303
  return encoded;
5322
5304
  }
5305
+ function sameFields(left, right) {
5306
+ if (left.length !== right.length) return false;
5307
+ for (let at = 0; at < left.length; at++) {
5308
+ if (left[at] !== right[at]) return false;
5309
+ }
5310
+ return true;
5311
+ }
5323
5312
 
5324
5313
  // src/core/partition/value-sizes.ts
5325
5314
  function keepsValueSize(base) {
@@ -5438,27 +5427,27 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
5438
5427
  const surfaceCounts = /* @__PURE__ */ new Map();
5439
5428
  if (state.trackPositions) {
5440
5429
  const tokenFreqs = /* @__PURE__ */ new Map();
5441
- for (const t of tokenizeIterator(text, language, opts)) {
5442
- const existing = tokenFreqs.get(t.token);
5430
+ visitTokens(text, language, opts, (token, position, surface) => {
5431
+ const existing = tokenFreqs.get(token);
5443
5432
  if (existing) {
5444
5433
  existing.count++;
5445
- existing.positions.push(t.position);
5434
+ existing.positions.push(position);
5446
5435
  } else {
5447
- tokenFreqs.set(t.token, { count: 1, positions: [t.position] });
5436
+ tokenFreqs.set(token, { count: 1, positions: [position] });
5448
5437
  }
5449
- fieldTokenList.push(t.token);
5450
- countOccurrence(surfaceCounts, spellings, t.surface, t.token);
5451
- }
5438
+ fieldTokenList.push(token);
5439
+ countOccurrence(surfaceCounts, spellings, surface, token);
5440
+ });
5452
5441
  for (const [token, freq] of tokenFreqs) {
5453
5442
  state.invertedIdx.insert(token, internalId, freq.count, fieldNameIndex2, freq.positions);
5454
5443
  }
5455
5444
  } else {
5456
5445
  const tokenCounts = /* @__PURE__ */ new Map();
5457
- for (const t of tokenizeIterator(text, language, opts)) {
5458
- tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
5459
- fieldTokenList.push(t.token);
5460
- countOccurrence(surfaceCounts, spellings, t.surface, t.token);
5461
- }
5446
+ visitTokens(text, language, opts, (token, _position, surface) => {
5447
+ tokenCounts.set(token, (tokenCounts.get(token) ?? 0) + 1);
5448
+ fieldTokenList.push(token);
5449
+ countOccurrence(surfaceCounts, spellings, surface, token);
5450
+ });
5462
5451
  for (const [token, count] of tokenCounts) {
5463
5452
  state.invertedIdx.insert(token, internalId, count, fieldNameIndex2, null);
5464
5453
  }
@@ -5475,20 +5464,22 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
5475
5464
  if (state.trackPositions) {
5476
5465
  const tokenFreqs = /* @__PURE__ */ new Map();
5477
5466
  let positionOffset = 0;
5478
- for (const item of arr) {
5479
- let itemTokenCount = 0;
5480
- for (const t of tokenizeIterator(item, language, opts)) {
5481
- const existing = tokenFreqs.get(t.token);
5482
- if (existing) {
5483
- existing.count++;
5484
- existing.positions.push(positionOffset + t.position);
5485
- } else {
5486
- tokenFreqs.set(t.token, { count: 1, positions: [positionOffset + t.position] });
5487
- }
5488
- fieldTokenList.push(t.token);
5489
- countOccurrence(surfaceCounts, spellings, t.surface, t.token);
5490
- itemTokenCount++;
5467
+ let itemTokenCount = 0;
5468
+ const addToken = (token, position, surface) => {
5469
+ const existing = tokenFreqs.get(token);
5470
+ if (existing) {
5471
+ existing.count++;
5472
+ existing.positions.push(positionOffset + position);
5473
+ } else {
5474
+ tokenFreqs.set(token, { count: 1, positions: [positionOffset + position] });
5491
5475
  }
5476
+ fieldTokenList.push(token);
5477
+ countOccurrence(surfaceCounts, spellings, surface, token);
5478
+ itemTokenCount++;
5479
+ };
5480
+ for (const item of arr) {
5481
+ itemTokenCount = 0;
5482
+ visitTokens(item, language, opts, addToken);
5492
5483
  positionOffset += itemTokenCount;
5493
5484
  }
5494
5485
  fieldLengths[fieldPath] = fieldTokenList.length;
@@ -5497,13 +5488,12 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
5497
5488
  }
5498
5489
  } else {
5499
5490
  const tokenCounts = /* @__PURE__ */ new Map();
5500
- for (const item of arr) {
5501
- for (const t of tokenizeIterator(item, language, opts)) {
5502
- tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
5503
- fieldTokenList.push(t.token);
5504
- countOccurrence(surfaceCounts, spellings, t.surface, t.token);
5505
- }
5506
- }
5491
+ const addToken = (token, _position, surface) => {
5492
+ tokenCounts.set(token, (tokenCounts.get(token) ?? 0) + 1);
5493
+ fieldTokenList.push(token);
5494
+ countOccurrence(surfaceCounts, spellings, surface, token);
5495
+ };
5496
+ for (const item of arr) visitTokens(item, language, opts, addToken);
5507
5497
  fieldLengths[fieldPath] = fieldTokenList.length;
5508
5498
  for (const [token, count] of tokenCounts) {
5509
5499
  state.invertedIdx.insert(token, internalId, count, fieldNameIndex2, null);
@@ -5736,6 +5726,7 @@ function updateFieldIndexOnly(state, docId, oldFields, newDoc, fields) {
5736
5726
  // src/filters/constants.ts
5737
5727
  var MAX_FILTER_DEPTH = 30;
5738
5728
  var MAX_FILTER_TESTS = 1024;
5729
+ var FILTER_OPERAND_COPY_DEPTH = 3;
5739
5730
  var MAX_FILTER_ARRAY_SIZE = 65536;
5740
5731
  var GEO_UNIT_METRES = { km: 1e3, mi: 1609.344, m: 1 };
5741
5732
 
@@ -6134,9 +6125,19 @@ function requireGeoOperands(fieldPath, filter) {
6134
6125
  }
6135
6126
 
6136
6127
  // src/filters/operands/lists.ts
6137
- var WHOLE_VALUE_OPERATORS = ["eq", "ne", "in", "nin", "startsWith", "endsWith", "matchesAny", "containsAll"];
6128
+ var WHOLE_VALUE_OPERATORS = [
6129
+ "eq",
6130
+ "ne",
6131
+ "in",
6132
+ "nin",
6133
+ "startsWith",
6134
+ "endsWith",
6135
+ "matchesAny",
6136
+ "containsAll"
6137
+ ];
6138
6138
  var ELEMENT_OPERATORS = ["matchesAny", "containsAll"];
6139
6139
  var PRESENCE_OPERATORS = ["exists", "notExists", "isEmpty", "isNotEmpty"];
6140
+ var SIZE_OPERAND_KEYS = ["size"];
6140
6141
  var SIZE_OPERATORS = ["eq", "ne", "gt", "gte", "lt", "lte", "between"];
6141
6142
  var SIZE_OPERATOR_SET = new Set(SIZE_OPERATORS);
6142
6143
  var ELEMENT_KIND_BY_BASE = {
@@ -6245,6 +6246,14 @@ function requirePatternText(fieldPath, operator, text, caseFold) {
6245
6246
  }
6246
6247
  }
6247
6248
  var ELEMENT_OPERATORS2 = ["matchesAny", "containsAll"];
6249
+ var PATTERN_OPERAND_KEYS = [
6250
+ "contains",
6251
+ "caseFold",
6252
+ "tolerance",
6253
+ ...PATTERN_EXPRESSION_OPERATORS,
6254
+ ...PATTERN_TEST_OPERATORS,
6255
+ ...ELEMENT_OPERATORS2
6256
+ ];
6248
6257
  function requireExpressions(fieldPath, filter, type) {
6249
6258
  for (const operator of PATTERN_EXPRESSION_OPERATORS) {
6250
6259
  const pattern = filter[operator];
@@ -6320,18 +6329,20 @@ function requireTolerance(fieldPath, filter) {
6320
6329
  { fieldPath, operator: "tolerance", requiredRuns, distinctRuns }
6321
6330
  );
6322
6331
  }
6323
- function requireWellFormedText(fieldPath, filter) {
6324
- for (const [operator, operand] of Object.entries(filter)) {
6325
- const texts = Array.isArray(operand) ? operand : [operand];
6326
- for (const text of texts) {
6327
- if (typeof text !== "string" || !holdsUnpairedSurrogate(text)) continue;
6328
- invalid(
6329
- `The text of "${operator}" on field "${fieldPath}" holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so no stored value can hold it`,
6330
- fieldPath,
6331
- operator
6332
- );
6333
- }
6332
+ function requireWellFormedOperand(fieldPath, operator, text) {
6333
+ if (typeof text !== "string" || !holdsUnpairedSurrogate(text)) return;
6334
+ invalid(
6335
+ `The text of "${operator}" on field "${fieldPath}" holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so no stored value can hold it`,
6336
+ fieldPath,
6337
+ operator
6338
+ );
6339
+ }
6340
+ function requireWellFormedText(fieldPath, operator, operand) {
6341
+ if (!Array.isArray(operand)) {
6342
+ requireWellFormedOperand(fieldPath, operator, operand);
6343
+ return;
6334
6344
  }
6345
+ for (const text of operand) requireWellFormedOperand(fieldPath, operator, text);
6335
6346
  }
6336
6347
 
6337
6348
  // src/filters/operands/index.ts
@@ -6341,9 +6352,16 @@ var EQUALITY_OPERATORS = ["eq", "ne"];
6341
6352
  var LIST_OPERATORS = ["in", "nin"];
6342
6353
  var GEO_OPERATORS = ["radius", "polygon"];
6343
6354
  var TEST_MODIFIERS = /* @__PURE__ */ new Set(["caseFold", "tolerance"]);
6355
+ var EXPRESSION_KEYS = new Set(FILTER_EXPRESSION_KEYS);
6356
+ var OPERATOR_KEYS = new Set(FIELD_FILTER_OPERATORS);
6357
+ var PROTOTYPE_KEY = "__proto__";
6358
+ var validatedFilters = /* @__PURE__ */ new WeakMap();
6344
6359
  function invalidShape(message) {
6345
6360
  throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message);
6346
6361
  }
6362
+ function invalidKey(message, key) {
6363
+ throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message, { key });
6364
+ }
6347
6365
  function undeclaredFieldOnStrictIndex(field, option) {
6348
6366
  return new NarsilError(
6349
6367
  ErrorCodes.SEARCH_INVALID_FIELD,
@@ -6420,24 +6438,47 @@ function requireValueOperands(fieldPath, filter, type) {
6420
6438
  if (filter[operator] !== void 0) requireFieldKind(fieldPath, operator, type, ["geopoint"]);
6421
6439
  }
6422
6440
  }
6423
- function requireValidOperands(fieldPath, filter, type) {
6424
- requireWellFormedText(fieldPath, filter);
6425
- requireRangeOperands(fieldPath, filter, type);
6426
- requireValueOperands(fieldPath, filter, type);
6427
- requireWholeValueIndex(fieldPath, filter, type);
6428
- requirePresenceOperands(fieldPath, filter);
6429
- requireElementOperands(fieldPath, filter, type);
6430
- requireSizeOperands(fieldPath, filter, type);
6431
- requirePatternOperands(fieldPath, filter, type);
6432
- requireGeoOperands(fieldPath, filter);
6441
+ var OPERAND_CHECKS = [
6442
+ [[...RANGE_OPERATORS, "between"], requireRangeOperands],
6443
+ [[...TEXT_OPERATORS, ...EQUALITY_OPERATORS, ...LIST_OPERATORS, ...GEO_OPERATORS], requireValueOperands],
6444
+ [WHOLE_VALUE_OPERATORS, requireWholeValueIndex],
6445
+ [PRESENCE_OPERATORS, requirePresenceOperands],
6446
+ [ELEMENT_OPERATORS, requireElementOperands],
6447
+ [SIZE_OPERAND_KEYS, requireSizeOperands],
6448
+ [PATTERN_OPERAND_KEYS, requirePatternOperands],
6449
+ [GEO_OPERATORS, requireGeoOperands]
6450
+ ];
6451
+ var CHECKS_BY_OPERATOR = checksByOperator();
6452
+ function checksByOperator() {
6453
+ const checks = /* @__PURE__ */ new Map();
6454
+ for (let at = 0; at < OPERAND_CHECKS.length; at++) {
6455
+ for (const operator of OPERAND_CHECKS[at][0]) checks.set(operator, (checks.get(operator) ?? 0) | 1 << at);
6456
+ }
6457
+ return checks;
6433
6458
  }
6434
- function testsIn(filter) {
6459
+ function testsOfOperator(operator, operand) {
6460
+ if (TEST_MODIFIERS.has(operator)) return 0;
6461
+ return operator === "size" && isObjectRecord(operand) ? Math.max(1, Object.keys(operand).length) : 1;
6462
+ }
6463
+ function requireValidFieldTests(fieldPath, filter, type, walk) {
6464
+ const operators = Object.keys(filter);
6465
+ let wanted = 0;
6435
6466
  let tests = 0;
6436
- for (const [operator, operand] of Object.entries(filter)) {
6437
- if (TEST_MODIFIERS.has(operator)) continue;
6438
- tests += operator === "size" && isObjectRecord(operand) ? Math.max(1, Object.keys(operand).length) : 1;
6467
+ for (const operator of operators) {
6468
+ if (walk.unknownKey === null && !OPERATOR_KEYS.has(operator)) {
6469
+ walk.unknownKey = {
6470
+ message: `Field "${fieldPath}" is tested with "${operator}", which is no filter operator`,
6471
+ key: operator
6472
+ };
6473
+ }
6474
+ tests += testsOfOperator(operator, filter[operator]);
6475
+ wanted |= CHECKS_BY_OPERATOR.get(operator) ?? 0;
6476
+ }
6477
+ countTests(walk, Math.max(1, tests));
6478
+ for (const operator of operators) requireWellFormedText(fieldPath, operator, filter[operator]);
6479
+ for (let at = 0; wanted !== 0; at++, wanted >>>= 1) {
6480
+ if ((wanted & 1) !== 0) OPERAND_CHECKS[at][1](fieldPath, filter, type);
6439
6481
  }
6440
- return Math.max(1, tests);
6441
6482
  }
6442
6483
  function countTests(walk, tests) {
6443
6484
  walk.tests += tests;
@@ -6445,51 +6486,92 @@ function countTests(walk, tests) {
6445
6486
  invalidShape(`A filter holds at most ${MAX_FILTER_TESTS} tests across all of its fields and nested expressions`);
6446
6487
  }
6447
6488
  }
6448
- function holdsNestedExpressions(expression) {
6449
- const { and, or, not } = expression;
6489
+ function holdsNestedExpressions(and, or, not) {
6450
6490
  return Array.isArray(and) && and.length > 0 || Array.isArray(or) && or.length > 0 || not !== void 0;
6451
6491
  }
6452
- function requireValidExpression(expression, walk, depth) {
6453
- if (depth > MAX_FILTER_DEPTH) invalidShape(`A filter expression nests at most ${MAX_FILTER_DEPTH} levels deep`);
6454
- if (!isObjectRecord(expression)) invalidShape("Each filter expression must be an object");
6455
- const { fields, and, or, not } = expression;
6456
- if (fields !== void 0 && !isObjectRecord(fields)) {
6457
- invalidShape('"fields" in a filter expression must be an object keyed by field path');
6492
+ function noteUnknownExpressionKey(expression, walk) {
6493
+ for (const key in expression) {
6494
+ if (!Object.hasOwn(expression, key) || EXPRESSION_KEYS.has(key)) continue;
6495
+ walk.unknownKey = {
6496
+ message: `A filter expression takes "fields", "and", "or", and "not", and "${key}" is none of them. A per-field test such as { ${key}: { eq: ... } } belongs under "fields"`,
6497
+ key
6498
+ };
6499
+ return;
6500
+ }
6501
+ }
6502
+ function setOwn(record, key, value) {
6503
+ if (key === PROTOTYPE_KEY) {
6504
+ Object.defineProperty(record, key, { value, enumerable: true, writable: true, configurable: true });
6505
+ } else {
6506
+ record[key] = value;
6507
+ }
6508
+ }
6509
+ function ownedOperandOf(value, fieldPath, operator, depth) {
6510
+ if (typeof value !== "object" || value === null || depth > FILTER_OPERAND_COPY_DEPTH) return value;
6511
+ if (Array.isArray(value)) {
6512
+ requireListLength(fieldPath, operator, value);
6513
+ const owned2 = [];
6514
+ for (let at = 0; at < value.length; at++) owned2.push(ownedOperandOf(value[at], fieldPath, operator, depth + 1));
6515
+ return owned2;
6458
6516
  }
6459
- const fieldFilters = Object.entries(fields ?? {});
6460
- if (fieldFilters.length === 0 && !holdsNestedExpressions(expression)) countTests(walk, 1);
6461
- for (const [fieldPath, filter] of fieldFilters) {
6517
+ const record = value;
6518
+ const owned = {};
6519
+ for (const key of Object.keys(record)) setOwn(owned, key, ownedOperandOf(record[key], fieldPath, operator, depth + 1));
6520
+ return owned;
6521
+ }
6522
+ function ownedFieldFilterOf(fieldPath, filter) {
6523
+ const owned = {};
6524
+ for (const operator of Object.keys(filter)) {
6525
+ setOwn(owned, operator, ownedOperandOf(filter[operator], fieldPath, operator, 1));
6526
+ }
6527
+ return owned;
6528
+ }
6529
+ function ownedFieldFiltersOf(fields, walk) {
6530
+ const owned = {};
6531
+ for (const fieldPath of Object.keys(fields)) {
6462
6532
  const type = walk.fieldTypes[fieldPath];
6463
6533
  if (!Object.hasOwn(walk.fieldTypes, fieldPath) || type === void 0) throw undeclaredFilterField(fieldPath);
6534
+ const filter = fields[fieldPath];
6464
6535
  if (!isObjectRecord(filter)) invalidShape(`The filter on field "${fieldPath}" must be an object of tests`);
6465
- countTests(walk, testsIn(filter));
6466
- requireValidOperands(fieldPath, filter, type);
6536
+ const ownedFilter = ownedFieldFilterOf(fieldPath, filter);
6537
+ requireValidFieldTests(fieldPath, ownedFilter, type, walk);
6538
+ setOwn(owned, fieldPath, ownedFilter);
6467
6539
  }
6468
- for (const [clause, list] of [
6469
- ["and", and],
6470
- ["or", or]
6471
- ]) {
6472
- if (list === void 0) continue;
6473
- if (!Array.isArray(list)) invalidShape(`"${clause}" in a filter expression must be a list of filter expressions`);
6474
- for (const nested of list) requireValidExpression(nested, walk, depth + 1);
6475
- }
6476
- if (not !== void 0) requireValidExpression(not, walk, depth + 1);
6540
+ return owned;
6477
6541
  }
6478
- function requireValidFilter(expression, fieldTypes) {
6479
- if (expression === void 0 || expression === null) return;
6480
- requireValidExpression(expression, { fieldTypes, tests: 0 }, 0);
6542
+ function ownedClauseOf(clause, list, walk, depth) {
6543
+ if (!Array.isArray(list)) invalidShape(`"${clause}" in a filter expression must be a list of filter expressions`);
6544
+ const owned = [];
6545
+ for (let at = 0; at < list.length; at++) owned.push(ownedExpressionOf(list[at], walk, depth + 1));
6546
+ return owned;
6481
6547
  }
6482
-
6483
- // src/filters/evaluator.ts
6484
- var EXPRESSION_KEYS = new Set(FILTER_EXPRESSION_KEYS);
6485
- var OPERATOR_KEYS = new Set(FIELD_FILTER_OPERATORS);
6486
- function requireKnownKeys2(value, known, describe) {
6487
- for (const key of Object.keys(value)) {
6488
- if (!known.has(key)) {
6489
- throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, describe(key), { key });
6490
- }
6548
+ function ownedExpressionOf(expression, walk, depth) {
6549
+ if (depth > MAX_FILTER_DEPTH) invalidShape(`A filter expression nests at most ${MAX_FILTER_DEPTH} levels deep`);
6550
+ if (!isObjectRecord(expression)) invalidShape("Each filter expression must be an object");
6551
+ if (walk.unknownKey === null) noteUnknownExpressionKey(expression, walk);
6552
+ const { fields, and, or, not } = expression;
6553
+ if (fields !== void 0 && !isObjectRecord(fields)) {
6554
+ invalidShape('"fields" in a filter expression must be an object keyed by field path');
6491
6555
  }
6556
+ const owned = {};
6557
+ const fieldCount = fields === void 0 ? 0 : Object.keys(fields).length;
6558
+ if (fieldCount === 0 && !holdsNestedExpressions(and, or, not)) countTests(walk, 1);
6559
+ if (fields !== void 0) owned.fields = ownedFieldFiltersOf(fields, walk);
6560
+ if (and !== void 0) owned.and = ownedClauseOf("and", and, walk, depth);
6561
+ if (or !== void 0) owned.or = ownedClauseOf("or", or, walk, depth);
6562
+ if (not !== void 0) owned.not = ownedExpressionOf(not, walk, depth + 1);
6563
+ return owned;
6564
+ }
6565
+ function validatedFilterOf(expression, fieldTypes) {
6566
+ if (validatedFilters.get(expression) === fieldTypes) return expression;
6567
+ const walk = { fieldTypes, tests: 0, unknownKey: null };
6568
+ const validated = ownedExpressionOf(expression, walk, 0);
6569
+ if (walk.unknownKey !== null) invalidKey(walk.unknownKey.message, walk.unknownKey.key);
6570
+ validatedFilters.set(validated, fieldTypes);
6571
+ return validated;
6492
6572
  }
6573
+
6574
+ // src/filters/evaluator.ts
6493
6575
  var Narrowing = class {
6494
6576
  bits = null;
6495
6577
  narrow = (matched) => {
@@ -6501,24 +6583,11 @@ var Narrowing = class {
6501
6583
  }
6502
6584
  };
6503
6585
  function narrowByField(fieldPath, filter, context, narrow) {
6504
- requireKnownKeys2(
6505
- filter,
6506
- OPERATOR_KEYS,
6507
- (key) => `Field "${fieldPath}" is tested with "${key}", which is no filter operator`
6508
- );
6509
- const type = context.fieldTypes[fieldPath];
6510
- const parsed = parsedTypeOf(type);
6511
- if (type === void 0 || parsed === void 0) throw undeclaredFilterField(fieldPath);
6512
- const f = filter;
6513
- requireValidOperands(fieldPath, f, type);
6514
- narrowByFieldTests(fieldPath, f, parsed, context, narrow.narrow);
6586
+ const parsed = parsedTypeOf(context.fieldTypes[fieldPath]);
6587
+ if (parsed === void 0) throw undeclaredFilterField(fieldPath);
6588
+ narrowByFieldTests(fieldPath, filter, parsed, context, narrow.narrow);
6515
6589
  }
6516
- function evaluateFilters(expression, context) {
6517
- requireKnownKeys2(
6518
- expression,
6519
- EXPRESSION_KEYS,
6520
- (key) => `A filter expression takes "fields", "and", "or", and "not", and "${key}" is none of them. A per-field test such as { ${key}: { eq: ... } } belongs under "fields"`
6521
- );
6590
+ function evaluateExpression(expression, context) {
6522
6591
  const narrowing = new Narrowing();
6523
6592
  if (expression.fields) {
6524
6593
  for (const [fieldPath, filter] of Object.entries(expression.fields)) {
@@ -6526,21 +6595,24 @@ function evaluateFilters(expression, context) {
6526
6595
  }
6527
6596
  }
6528
6597
  if (expression.and?.length) {
6529
- for (const nested of expression.and) narrowing.narrow(evaluateFilters(nested, context));
6598
+ for (const nested of expression.and) narrowing.narrow(evaluateExpression(nested, context));
6530
6599
  }
6531
6600
  if (expression.or?.length) {
6532
6601
  const union = createBitSet(context.capacity);
6533
- for (const nested of expression.or) bitsetOrInto(union, evaluateFilters(nested, context));
6602
+ for (const nested of expression.or) bitsetOrInto(union, evaluateExpression(nested, context));
6534
6603
  narrowing.narrow(union);
6535
6604
  }
6536
6605
  if (expression.not) {
6537
- const excluded = evaluateFilters(expression.not, context);
6606
+ const excluded = evaluateExpression(expression.not, context);
6538
6607
  const kept = context.allDocIdsBitset.slice();
6539
6608
  bitsetAndNotInto(kept, excluded);
6540
6609
  narrowing.narrow(kept);
6541
6610
  }
6542
6611
  return narrowing.value(() => context.allDocIdsBitset);
6543
6612
  }
6613
+ function evaluateFilters(expression, context) {
6614
+ return evaluateExpression(validatedFilterOf(expression, context.fieldTypes), context);
6615
+ }
6544
6616
 
6545
6617
  // src/core/partition/constants.ts
6546
6618
  var TARGET_BLOCK_ENTRIES = 128;
@@ -6550,6 +6622,7 @@ var SEGMENT_TOKEN_TABLE_START = 1 << 12;
6550
6622
  var SEGMENT_POSTING_LOG_START = 1 << 14;
6551
6623
  var SEGMENT_POSITION_POOL_START = 1 << 16;
6552
6624
  var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
6625
+ var LOOKUP_KEY_SCRATCH_BYTES = 4096;
6553
6626
  var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
6554
6627
  var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
6555
6628
 
@@ -7130,7 +7203,7 @@ function ensureColumns(state) {
7130
7203
  function textRangeBitsetOf(state, fieldPath, fieldType, bounds, capacity) {
7131
7204
  const column = ensureColumns(state).column(fieldPath, fieldType, "min");
7132
7205
  const resolver = state.docStore.resolver();
7133
- return selectTextRange(column, bounds, capacity, (internalId) => resolver.toExternal(internalId) !== void 0);
7206
+ return selectTextRange(column, bounds, capacity, (internalId) => resolver.isLive(internalId));
7134
7207
  }
7135
7208
  function sortedPageOf(state, request) {
7136
7209
  ensureColumns(state);
@@ -7267,7 +7340,7 @@ function partitionFilterMatches(state, filters, schema, meter) {
7267
7340
  const base = wordIndex << 5;
7268
7341
  while (word !== 0) {
7269
7342
  const trailingZeros = Math.clz32(word & -word) ^ 31;
7270
- if (resolver.toExternal(base + trailingZeros) !== void 0) count++;
7343
+ if (resolver.isLive(base + trailingZeros)) count++;
7271
7344
  word &= word - 1;
7272
7345
  }
7273
7346
  }
@@ -7536,7 +7609,7 @@ function searchFulltextMatches(state, params) {
7536
7609
  const base = wordIndex << 5;
7537
7610
  while (word !== 0) {
7538
7611
  const trailingZeros = Math.clz32(word & -word) ^ 31;
7539
- if (resolver.toExternal(base + trailingZeros) !== void 0) count++;
7612
+ if (resolver.isLive(base + trailingZeros)) count++;
7540
7613
  word &= word - 1;
7541
7614
  }
7542
7615
  }
@@ -7990,7 +8063,7 @@ function kthBestScore(buffer, selection, resolver) {
7990
8063
  const internalId = touched[index];
7991
8064
  const score = scores[internalId];
7992
8065
  if (size === wanted && score <= selection[0]) continue;
7993
- if (resolver.toExternal(internalId) === void 0) continue;
8066
+ if (!resolver.isLive(internalId)) continue;
7994
8067
  let at = 0;
7995
8068
  if (size < wanted) {
7996
8069
  at = size++;
@@ -11053,7 +11126,7 @@ function compositeFilterMatches(subs, layout, filters, schema, meter) {
11053
11126
  const base = wordIndex << 5;
11054
11127
  while (word !== 0) {
11055
11128
  const trailingZeros = Math.clz32(word & -word) ^ 31;
11056
- if (resolver.toExternal(base + trailingZeros) !== void 0) count++;
11129
+ if (resolver.isLive(base + trailingZeros)) count++;
11057
11130
  word &= word - 1;
11058
11131
  }
11059
11132
  }
@@ -11359,6 +11432,13 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
11359
11432
  }
11360
11433
  return void 0;
11361
11434
  }
11435
+ function frozenStored(docId) {
11436
+ for (const segment of frozen) {
11437
+ const stored = segment.docStore.get(docId);
11438
+ if (stored !== void 0) return stored;
11439
+ }
11440
+ return void 0;
11441
+ }
11362
11442
  return {
11363
11443
  get partitionId() {
11364
11444
  return partitionId;
@@ -11531,18 +11611,13 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
11531
11611
  expandPartialWords(subs(), analysis, []);
11532
11612
  },
11533
11613
  get(docId, projection) {
11534
- const owner = frozenOwner(docId);
11535
- if (owner !== void 0) {
11536
- const stored = owner.docStore.get(docId);
11537
- return stored === void 0 ? void 0 : cloneProjected(stored.fields, projection);
11538
- }
11614
+ const stored = frozenStored(docId);
11615
+ if (stored !== void 0) return cloneProjected(stored.fields, projection);
11539
11616
  return live.get(docId, projection);
11540
11617
  },
11541
11618
  getRef(docId) {
11542
- const owner = frozenOwner(docId);
11543
- if (owner !== void 0) {
11544
- return owner.docStore.get(docId)?.fields;
11545
- }
11619
+ const stored = frozenStored(docId);
11620
+ if (stored !== void 0) return stored.fields;
11546
11621
  return live.getRef(docId);
11547
11622
  },
11548
11623
  has(docId) {
@@ -18320,6 +18395,9 @@ function createFrozenDocTable(idTable, lengthColumns, documents, tombstones) {
18320
18395
  },
18321
18396
  toInternal(externalId) {
18322
18397
  return liveOrdinal(externalId);
18398
+ },
18399
+ isLive(internalId) {
18400
+ return internalId >= 0 && internalId < capacity && !tombstones.has(internalId);
18323
18401
  }
18324
18402
  };
18325
18403
  return {
@@ -18481,7 +18559,14 @@ function wrapDocumentArray(documents) {
18481
18559
 
18482
18560
  // src/core/partition/frozen/string-blob.ts
18483
18561
  var SURROGATE = /[\ud800-\udfff]/;
18562
+ var UTF8_BYTES_PER_UTF16_UNIT = 3;
18484
18563
  var encoder3 = new TextEncoder();
18564
+ var lookupKeyBytes = new Uint8Array(LOOKUP_KEY_SCRATCH_BYTES);
18565
+ function encodeLookupKey(text) {
18566
+ if (text.length * UTF8_BYTES_PER_UTF16_UNIT > LOOKUP_KEY_SCRATCH_BYTES) return encoder3.encode(text);
18567
+ const { written } = encoder3.encodeInto(text, lookupKeyBytes);
18568
+ return lookupKeyBytes.subarray(0, written);
18569
+ }
18485
18570
  function encodeOneByOne(strings) {
18486
18571
  const encoded = new Array(strings.length);
18487
18572
  let blobLength = 0;
@@ -18543,7 +18628,6 @@ function codePointOrder(strings) {
18543
18628
  }
18544
18629
 
18545
18630
  // src/core/partition/frozen/external-ids.ts
18546
- var encoder4 = new TextEncoder();
18547
18631
  var decoder2 = new TextDecoder();
18548
18632
  function encodeExternalIdTableData(docIds) {
18549
18633
  const { blob, offsets } = encodeStringBlob(docIds);
@@ -18562,11 +18646,13 @@ function wrapExternalIdTable(data) {
18562
18646
  const { blob, offsets, sortedOrdinals } = data;
18563
18647
  const count = sortedOrdinals.length;
18564
18648
  const decoded = new Array(count);
18649
+ const decodedOrdinals = /* @__PURE__ */ new Map();
18565
18650
  function idAt2(ordinal) {
18566
18651
  let id = decoded[ordinal];
18567
18652
  if (id === void 0) {
18568
18653
  id = decoder2.decode(blob.subarray(offsets[ordinal], offsets[ordinal + 1]));
18569
18654
  decoded[ordinal] = id;
18655
+ decodedOrdinals.set(id, ordinal);
18570
18656
  }
18571
18657
  return id;
18572
18658
  }
@@ -18575,7 +18661,9 @@ function wrapExternalIdTable(data) {
18575
18661
  bytes: blob.byteLength + offsets.byteLength + sortedOrdinals.byteLength,
18576
18662
  idAt: idAt2,
18577
18663
  ordinalOf(docId) {
18578
- const query = encoder4.encode(docId);
18664
+ const known = decodedOrdinals.get(docId);
18665
+ if (known !== void 0) return known;
18666
+ const query = encodeLookupKey(docId);
18579
18667
  let lo = 0;
18580
18668
  let hi = count;
18581
18669
  while (lo < hi) {
@@ -19019,7 +19107,7 @@ function createLazySurfaceReader(load) {
19019
19107
  }
19020
19108
 
19021
19109
  // src/core/partition/frozen/token-table.ts
19022
- var encoder5 = new TextEncoder();
19110
+ var encoder4 = new TextEncoder();
19023
19111
  var decoder4 = new TextDecoder();
19024
19112
  function compareBytes2(blob, start, end, query) {
19025
19113
  const length = end - start;
@@ -19083,7 +19171,7 @@ function wrapFrozenTokenTable(data) {
19083
19171
  size,
19084
19172
  bytes: blob.byteLength + offsets.byteLength + payloadSlots.byteLength + documentFrequencies.byteLength,
19085
19173
  find(token) {
19086
- const query = encoder5.encode(token);
19174
+ const query = encodeLookupKey(token);
19087
19175
  const at = lowerBound3(query);
19088
19176
  if (at >= size) return -1;
19089
19177
  return compareBytes2(blob, offsets[at], offsets[at + 1], query) === 0 ? at : -1;
@@ -19106,7 +19194,7 @@ function wrapFrozenTokenTable(data) {
19106
19194
  return documentFrequencies[sortedIndex];
19107
19195
  },
19108
19196
  prefixRange(prefix) {
19109
- const bytes = encoder5.encode(prefix);
19197
+ const bytes = encoder4.encode(prefix);
19110
19198
  const start = lowerBound3(bytes);
19111
19199
  return { start, end: prefixEnd(start, bytes) };
19112
19200
  },
@@ -19124,8 +19212,8 @@ function wrapFrozenTokenTable(data) {
19124
19212
  fromCodePoint = unit;
19125
19213
  toCodePoint = nextCodePoint(unit);
19126
19214
  }
19127
- const from = encoder5.encode(String.fromCodePoint(fromCodePoint));
19128
- const to = encoder5.encode(String.fromCodePoint(toCodePoint));
19215
+ const from = encoder4.encode(String.fromCodePoint(fromCodePoint));
19216
+ const to = encoder4.encode(String.fromCodePoint(toCodePoint));
19129
19217
  return { start: lowerBound3(from), end: lowerBound3(to) };
19130
19218
  }
19131
19219
  };
@@ -19146,7 +19234,7 @@ function createFrozenTombstones() {
19146
19234
  return revision;
19147
19235
  },
19148
19236
  has(internalId) {
19149
- return removed.has(internalId);
19237
+ return removed.size > 0 && removed.has(internalId);
19150
19238
  },
19151
19239
  add(internalId) {
19152
19240
  if (removed.has(internalId)) return false;
@@ -19188,7 +19276,7 @@ function sharedSurfaceTable(payload) {
19188
19276
  return { blob: sharedUint8(table.blob), offsets: sharedUint32(table.offsets), counts: sharedUint32(table.counts) };
19189
19277
  }
19190
19278
  function sharePartialWordTable(table) {
19191
- if (typeof SharedArrayBuffer !== "function") return table;
19279
+ if (typeof SharedArrayBuffer !== "function" || table.runEntries.buffer instanceof SharedArrayBuffer) return table;
19192
19280
  return {
19193
19281
  fields: [...table.fields],
19194
19282
  spellings: sharedUint8(table.spellings),
@@ -19209,17 +19297,19 @@ function sharedPartialWordsOf(tokenData, spellings, documentData, analysis) {
19209
19297
  const table = buildPartialWordTableFrom(source, wrapFrozenTokenTable(tokenData));
19210
19298
  return table === null ? null : sharePartialWordTable(table);
19211
19299
  }
19300
+ function partialWordsMatch(table, analysis) {
19301
+ if (table === null) return analysis === null || analysis.fields.length === 0;
19302
+ return analysis === null || sameFields(table.fields, analysis.fields);
19303
+ }
19212
19304
  function shareSegmentArrays(arrays, analysis, segmentId) {
19213
- return {
19214
- ...arrays,
19215
- segmentId: segmentId ?? generateId(),
19216
- partialWords: sharedPartialWordsOf(arrays.tokenTable, null, arrays.documentTable, analysis)
19217
- };
19305
+ const partialWords = partialWordsMatch(arrays.partialWords, analysis) ? arrays.partialWords : sharedPartialWordsOf(arrays.tokenTable, null, arrays.documentTable, analysis);
19306
+ return { ...arrays, segmentId: segmentId ?? generateId(), partialWords };
19218
19307
  }
19219
- function segmentArraysOf(payload, documentData) {
19308
+ function segmentArraysOf(payload, documentData, reusedPartialWords = null) {
19220
19309
  const tokenTable = encodeFrozenTokenTableData(payload.tokens, payload.docFrequencies);
19221
19310
  const idTable = encodeExternalIdTableData(payload.docIds);
19222
19311
  const surfaceTable = payload.surfaceForms === null ? null : encodeSurfaceTable(payload.surfaceForms);
19312
+ const partialWords = reusedPartialWords ?? (payload.partialSpellings === null ? null : buildPartialWordTableFrom(payload.partialSpellings, wrapFrozenTokenTable(tokenTable)));
19223
19313
  return {
19224
19314
  documentCount: payload.documentCount,
19225
19315
  fieldNames: payload.fieldNames,
@@ -19241,14 +19331,15 @@ function segmentArraysOf(payload, documentData) {
19241
19331
  enums: payload.enums,
19242
19332
  geo: payload.geo,
19243
19333
  pattern: payload.pattern,
19244
- surfaceTable: surfaceTable === null || surfaceTable.counts.length === 0 ? null : surfaceTable
19334
+ surfaceTable: surfaceTable === null || surfaceTable.counts.length === 0 ? null : surfaceTable,
19335
+ partialWords
19245
19336
  };
19246
19337
  }
19247
19338
  function freezeSegmentShared(payload, documents, segmentId, analysis = null) {
19248
19339
  if (typeof SharedArrayBuffer !== "function") return null;
19249
19340
  return freezeEncodedSegmentShared(payload, encodeDocumentTableData(documents), segmentId, analysis);
19250
19341
  }
19251
- function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis = null) {
19342
+ function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis = null, reusedPartialWords = null) {
19252
19343
  if (typeof SharedArrayBuffer !== "function") return null;
19253
19344
  const tokenData = encodeFrozenTokenTableData(payload.tokens, payload.docFrequencies);
19254
19345
  const idData = encodeExternalIdTableData(payload.docIds);
@@ -19316,7 +19407,7 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis =
19316
19407
  postings: sharedUint32(entry.postings)
19317
19408
  })),
19318
19409
  surfaceTable: sharedSurfaceTable(payload),
19319
- partialWords: sharedPartialWordsOf(tokenData, payload.partialSpellings, documentData, analysis)
19410
+ partialWords: reusedPartialWords === null ? sharedPartialWordsOf(tokenData, payload.partialSpellings, documentData, analysis) : sharePartialWordTable(reusedPartialWords)
19320
19411
  };
19321
19412
  }
19322
19413
 
@@ -19352,11 +19443,6 @@ function buildGeoReaders(entries) {
19352
19443
  }
19353
19444
  return readers;
19354
19445
  }
19355
- function* survivingDocuments(documentSource, tombstones) {
19356
- for (let ordinal = 0; ordinal < documentSource.count; ordinal++) {
19357
- if (!tombstones.has(ordinal)) yield documentSource.readAt(ordinal);
19358
- }
19359
- }
19360
19446
  function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable) {
19361
19447
  const tombstones = createFrozenTombstones();
19362
19448
  const postingViews = createFrozenPostingViews(source, tombstones);
@@ -19366,8 +19452,9 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
19366
19452
  }
19367
19453
  const docStore = createFrozenDocTable(source.idTable, lengthColumns, documentSource, tombstones);
19368
19454
  const partialWords = createFrozenPartialWords(partialWordTable, source.tokenTable, {
19455
+ count: documentSource.count,
19369
19456
  documentAt: (ordinal) => documentSource.readAt(ordinal),
19370
- survivors: () => survivingDocuments(documentSource, tombstones)
19457
+ isTombstoned: (ordinal) => tombstones.has(ordinal)
19371
19458
  });
19372
19459
  const numericIndexes = /* @__PURE__ */ new Map();
19373
19460
  for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
@@ -19819,9 +19906,9 @@ function uniqueSurvivors(range) {
19819
19906
  }
19820
19907
  return seen.size;
19821
19908
  }
19822
- function countRange(range, counts, keepsEveryDocument) {
19909
+ function countRange(range, counts, keepsEveryDocument2) {
19823
19910
  const { arrays } = range.input.segment;
19824
- if (keepsEveryDocument) {
19911
+ if (keepsEveryDocument2) {
19825
19912
  counts.postings += range.end - range.start;
19826
19913
  if (arrays.positionOffsets !== null) {
19827
19914
  counts.positions += arrays.positionOffsets[range.end] - arrays.positionOffsets[range.start];
@@ -19846,15 +19933,15 @@ function mergePostings2(inputs) {
19846
19933
  for (const input of inputs) {
19847
19934
  if (input.segment.arrays.positionOffsets !== null) hasPositions = true;
19848
19935
  }
19849
- const keepsEveryDocument = /* @__PURE__ */ new Map();
19936
+ const keepsEveryDocument2 = /* @__PURE__ */ new Map();
19850
19937
  for (const input of inputs)
19851
- keepsEveryDocument.set(
19938
+ keepsEveryDocument2.set(
19852
19939
  input,
19853
19940
  input.remap.every((target) => target >= 0)
19854
19941
  );
19855
19942
  walkTokens(inputs, (_token, ranges) => {
19856
19943
  const counts = { postings: 0, positions: 0 };
19857
- for (const range of ranges) countRange(range, counts, keepsEveryDocument.get(range.input) === true);
19944
+ for (const range of ranges) countRange(range, counts, keepsEveryDocument2.get(range.input) === true);
19858
19945
  if (counts.postings === 0) return;
19859
19946
  tokenCount += 1;
19860
19947
  totalPostings += counts.postings;
@@ -19878,7 +19965,7 @@ function mergePostings2(inputs) {
19878
19965
  const remap = range.input.remap;
19879
19966
  const fieldIndexRemap = range.input.fieldIndexRemap;
19880
19967
  docs += uniqueSurvivors(range);
19881
- if (keepsEveryDocument.get(range.input) === true) {
19968
+ if (keepsEveryDocument2.get(range.input) === true) {
19882
19969
  const count = range.end - range.start;
19883
19970
  for (let k = 0; k < count; k++) {
19884
19971
  postingDocIds[postingCursor + k] = remap[arrays.postingDocIds[range.start + k]];
@@ -20055,12 +20142,33 @@ function mergeDocuments2(inputs, documentCount) {
20055
20142
  }
20056
20143
  return { blob, offsets };
20057
20144
  }
20145
+ function keepsEveryDocument(remap) {
20146
+ for (let ordinal = 0; ordinal < remap.length; ordinal++) {
20147
+ if (remap[ordinal] < 0) return false;
20148
+ }
20149
+ return true;
20150
+ }
20151
+ function mergePartialWords(inputs, mergedTokenCount, analysis) {
20152
+ if (analysis === null || analysis.fields.length === 0) return { spellings: null, reused: null };
20153
+ if (inputs.length === 1) {
20154
+ const [only] = inputs;
20155
+ const table = only.segment.partialWords.tableOfEveryDocument(analysis);
20156
+ const keepsTokens = mergedTokenCount === only.segment.arrays.tokenTable.size;
20157
+ if (table !== null && keepsTokens && keepsEveryDocument(only.remap)) return { spellings: null, reused: table };
20158
+ }
20159
+ const counts = /* @__PURE__ */ new Map();
20160
+ for (const input of inputs) input.segment.partialWords.countKeptSpellings(counts, analysis, input.remap);
20161
+ return { spellings: encodePartialSpellings(analysis.fields, counts), reused: null };
20162
+ }
20058
20163
  function mergeFrozenSegments(segments, analysis) {
20059
20164
  if (typeof SharedArrayBuffer !== "function") return null;
20060
- const merged = mergeSegmentContents(segments.map((segment) => ({ segment })));
20061
- return freezeEncodedSegmentShared(merged.payload, merged.documents, void 0, analysis);
20165
+ const merged = mergeSegmentContents(
20166
+ segments.map((segment) => ({ segment })),
20167
+ analysis
20168
+ );
20169
+ return freezeEncodedSegmentShared(merged.payload, merged.documents, void 0, analysis, merged.reusedPartialWords);
20062
20170
  }
20063
- function mergeSegmentContents(selections) {
20171
+ function mergeSegmentContents(selections, analysis = null) {
20064
20172
  const docIds = [];
20065
20173
  const fieldNames = [];
20066
20174
  const inputs = selections.map((selection) => ({
@@ -20071,6 +20179,7 @@ function mergeSegmentContents(selections) {
20071
20179
  const documentCount = docIds.length;
20072
20180
  const postings = mergePostings2(inputs);
20073
20181
  const lengths = mergeFieldLengths(inputs, documentCount);
20182
+ const partialWords = mergePartialWords(inputs, postings.tokens.length, analysis);
20074
20183
  const payload = {
20075
20184
  documentCount,
20076
20185
  docIds,
@@ -20087,10 +20196,10 @@ function mergeSegmentContents(selections) {
20087
20196
  totalFieldLengths: lengths.totalFieldLengths,
20088
20197
  docFrequencies: postings.docFrequencies,
20089
20198
  surfaceForms: mergeSurfaceForms2(inputs),
20090
- partialSpellings: null,
20199
+ partialSpellings: partialWords.spellings,
20091
20200
  ...mergeFieldIndexes2(inputs)
20092
20201
  };
20093
- return { payload, documents: mergeDocuments2(inputs, documentCount) };
20202
+ return { payload, documents: mergeDocuments2(inputs, documentCount), reusedPartialWords: partialWords.reused };
20094
20203
  }
20095
20204
 
20096
20205
  // src/core/partition/growable-uint32.ts
@@ -23889,7 +23998,7 @@ function selectionOfMask(segment, keep) {
23889
23998
  }
23890
23999
  function frozenLiveTail(live, shared2) {
23891
24000
  if (live.count() === 0) return null;
23892
- const payload = { ...live.encodeSegment(), partialSpellings: null };
24001
+ const payload = live.encodeSegment();
23893
24002
  const documents = survivorDocumentsOf(partitionStateOf(live), payload);
23894
24003
  if (shared2) {
23895
24004
  const snapshot = freezeSegmentShared(payload, documents);
@@ -24709,6 +24818,23 @@ function requireEntriesAscend(blob, offsets, order, name) {
24709
24818
  }
24710
24819
  }
24711
24820
  }
24821
+ function requireSpellingEntriesAscend(blob, offsets, tokens, name) {
24822
+ for (let at = 1; at < tokens.length; at++) {
24823
+ const order = compareEntries2(blob, offsets, at - 1, at) || tokens[at - 1] - tokens[at];
24824
+ if (order >= 0) corrupt2(`${name} fails to ascend strictly at entry ${at}`, { field: name, at });
24825
+ }
24826
+ }
24827
+ function requireBelow(values, limit, name) {
24828
+ for (let at = 0; at < values.length; at++) {
24829
+ if (values[at] >= limit)
24830
+ corrupt2(`${name} holds ${values[at]} at entry ${at}, at or above ${limit}`, { field: name, at });
24831
+ }
24832
+ }
24833
+ function requireNonZero(values, name) {
24834
+ for (let at = 0; at < values.length; at++) {
24835
+ if (values[at] === 0) corrupt2(`${name} holds 0 at entry ${at}`, { field: name, at });
24836
+ }
24837
+ }
24712
24838
  function requirePermutation(order, name) {
24713
24839
  const seen = new Uint8Array(order.length);
24714
24840
  for (let at = 0; at < order.length; at++) {
@@ -24848,6 +24974,34 @@ function decodeSurfaceForms(raw, context) {
24848
24974
  requireUtf8Entries(blob, offsets, "surfaceForms.text");
24849
24975
  return counts.length === 0 ? null : { blob, offsets, counts };
24850
24976
  }
24977
+ function decodePartialWords(raw, tokenCount, context) {
24978
+ if (raw === null || raw === void 0) return null;
24979
+ const column = requireRecord(raw, "partialWords");
24980
+ const fields = requireStrings(column.fields, "partialWords.fields");
24981
+ const spellings = bytesFrom(requireBytes(column.text, "partialWords.text", 1), context.shared);
24982
+ const spellingOffsets = uint32Column(column, "offsets", "partialWords", context);
24983
+ const spellingTokens = uint32Column(column, "tokens", "partialWords", context);
24984
+ const spellingCounts = uint32Column(column, "counts", "partialWords", context);
24985
+ const entries = spellingTokens.length;
24986
+ requireCount(spellingOffsets.length, entries + 1, "partialWords.offsets");
24987
+ requireCount(spellingCounts.length, entries, "partialWords.counts");
24988
+ requireOffsets(spellingOffsets, spellings.length, "partialWords.offsets");
24989
+ requireUtf8Entries(spellings, spellingOffsets, "partialWords.text");
24990
+ requireBelow(spellingTokens, tokenCount, "partialWords.tokens");
24991
+ requireNonZero(spellingCounts, "partialWords.counts");
24992
+ requireSpellingEntriesAscend(spellings, spellingOffsets, spellingTokens, "partialWords");
24993
+ const runs = uint32sFrom(requireBytes(column.runs, "partialWords.runs", 4 * CODE_POINTS_PER_RUN), context.shared);
24994
+ requireRunsAscend(runs, "partialWords.runs");
24995
+ const runOffsets = uint32Column(column, "runOffsets", "partialWords", context);
24996
+ const runEntries = uint32Column(column, "runEntries", "partialWords", context);
24997
+ requireCount(runOffsets.length, runs.length / CODE_POINTS_PER_RUN + 1, "partialWords.runOffsets");
24998
+ requireOffsets(runOffsets, runEntries.length, "partialWords.runOffsets");
24999
+ requireBelow(runEntries, entries, "partialWords.runEntries");
25000
+ for (let run = 0; run + 1 < runOffsets.length; run++) {
25001
+ requireStrictlyAscending(runEntries, runOffsets[run], runOffsets[run + 1], "partialWords.runEntries");
25002
+ }
25003
+ return { fields, spellings, spellingOffsets, spellingTokens, spellingCounts, runs, runOffsets, runEntries };
25004
+ }
24851
25005
 
24852
25006
  // src/serialization/keyword-segment/types.ts
24853
25007
  var KEYWORD_SEGMENT_PAYLOAD_VERSION = 1;
@@ -24968,12 +25122,13 @@ function decodeKeywordSegment(payload, shared2) {
24968
25122
  corrupt2(`fieldNames holds ${fieldNames.length} names, above the limit of ${MAX_SEGMENT_FIELD_NAMES}`);
24969
25123
  }
24970
25124
  const context = { documentCount, shared: shared2 };
25125
+ const postings = decodePostings(raw, fieldNames, context);
24971
25126
  const arrays = {
24972
25127
  documentCount,
24973
25128
  fieldNames,
24974
25129
  idTable: decodeIds(raw, context),
24975
25130
  documentTable: decodeDocuments(raw, schema, context),
24976
- ...decodePostings(raw, fieldNames, context),
25131
+ ...postings,
24977
25132
  ...decodeFieldLengths(raw.fieldLengths, context),
24978
25133
  numeric: decodeValueColumns(raw.numeric, "numeric", context),
24979
25134
  sizes: decodeValueColumns(raw.sizes, "sizes", context),
@@ -24981,7 +25136,8 @@ function decodeKeywordSegment(payload, shared2) {
24981
25136
  enums: decodeEnumColumns(raw.enum, context),
24982
25137
  geo: decodeGeopointColumns(raw.geopoint, context),
24983
25138
  pattern: decodePatternColumns(raw.pattern, context),
24984
- surfaceTable: decodeSurfaceForms(raw.surfaceForms, context)
25139
+ surfaceTable: decodeSurfaceForms(raw.surfaceForms, context),
25140
+ partialWords: decodePartialWords(raw.partialWords, postings.tokenTable.offsets.length - 1, context)
24985
25141
  };
24986
25142
  return { schema, tombstones, arrays };
24987
25143
  }
@@ -25105,7 +25261,7 @@ function postingsInTokenOrder(arrays) {
25105
25261
  }
25106
25262
 
25107
25263
  // src/serialization/keyword-segment/encode.ts
25108
- var TOP_LEVEL_ENTRIES = 27;
25264
+ var TOP_LEVEL_ENTRIES = 28;
25109
25265
  function writeFieldLengths(out, arrays) {
25110
25266
  out.array(arrays.fieldLengthNames.length);
25111
25267
  for (let f = 0; f < arrays.fieldLengthNames.length; f++) {
@@ -25167,6 +25323,21 @@ function writeSurfaceForms(out, arrays) {
25167
25323
  out.key("offsets").binary(littleEndianBytes(table.offsets));
25168
25324
  out.key("counts").binary(littleEndianBytes(table.counts));
25169
25325
  }
25326
+ function writePartialWords(out, arrays) {
25327
+ const table = arrays.partialWords;
25328
+ if (table === null) {
25329
+ out.value(null);
25330
+ return;
25331
+ }
25332
+ out.map(8).key("fields").value(table.fields);
25333
+ out.key("text").binary(table.spellings);
25334
+ out.key("offsets").binary(littleEndianBytes(table.spellingOffsets));
25335
+ out.key("tokens").binary(littleEndianBytes(table.spellingTokens));
25336
+ out.key("counts").binary(littleEndianBytes(table.spellingCounts));
25337
+ out.key("runs").binary(littleEndianBytes(table.runs));
25338
+ out.key("runOffsets").binary(littleEndianBytes(table.runOffsets));
25339
+ out.key("runEntries").binary(littleEndianBytes(table.runEntries));
25340
+ }
25170
25341
  function encodeKeywordSegment(contents) {
25171
25342
  const { arrays } = contents;
25172
25343
  const postings = postingsInTokenOrder(arrays);
@@ -25207,6 +25378,8 @@ function encodeKeywordSegment(contents) {
25207
25378
  writePatternColumns(out, arrays.pattern);
25208
25379
  out.key("surfaceForms");
25209
25380
  writeSurfaceForms(out, arrays);
25381
+ out.key("partialWords");
25382
+ writePartialWords(out, arrays);
25210
25383
  return out;
25211
25384
  }
25212
25385
 
@@ -25230,7 +25403,7 @@ var VALUE_ENTRY_BYTES = 12;
25230
25403
  var GEOPOINT_ENTRY_BYTES = 20;
25231
25404
  var TOKEN_ENTRY_BYTES = 12;
25232
25405
  var STRING_HEADER_BYTES3 = 5;
25233
- var encoder6 = new TextEncoder();
25406
+ var encoder5 = new TextEncoder();
25234
25407
  function documentBytes(segment, ordinal) {
25235
25408
  const table = segment.arrays.documentTable;
25236
25409
  if (table !== null) return table.offsets[ordinal + 1] - table.offsets[ordinal];
@@ -25273,7 +25446,7 @@ function chargeColumns(segment, costs) {
25273
25446
  }
25274
25447
  function stringListBytes(values) {
25275
25448
  let bytes = STRING_HEADER_BYTES3;
25276
- for (const value of values) bytes += STRING_HEADER_BYTES3 + encoder6.encode(value).byteLength;
25449
+ for (const value of values) bytes += STRING_HEADER_BYTES3 + encoder5.encode(value).byteLength;
25277
25450
  return bytes;
25278
25451
  }
25279
25452
  function fixedBytesOf(segment, chargesTokens) {
@@ -25285,7 +25458,7 @@ function fixedBytesOf(segment, chargesTokens) {
25285
25458
  const surfaces = segment.sharedSnapshot?.surfaceTable ?? null;
25286
25459
  if (surfaces !== null) bytes += surfaces.blob.byteLength + surfaces.offsets.byteLength + surfaces.counts.byteLength;
25287
25460
  else bytes += messagePackUpperBoundBytes(arrays.readSurfaceForms());
25288
- return bytes;
25461
+ return bytes + segment.partialWords.bytes();
25289
25462
  }
25290
25463
  function keptDocumentsOf(selection, chargesTokens) {
25291
25464
  const { segment } = selection;
@@ -25293,7 +25466,7 @@ function keptDocumentsOf(selection, chargesTokens) {
25293
25466
  const costs = new Float64Array(count);
25294
25467
  const perDocument = ORDINAL_BYTES * (3 + segment.arrays.fieldLengthColumns.length);
25295
25468
  for (let ordinal = 0; ordinal < count; ordinal++) {
25296
- costs[ordinal] = perDocument + encoder6.encode(segment.arrays.idTable.idAt(ordinal)).byteLength;
25469
+ costs[ordinal] = perDocument + encoder5.encode(segment.arrays.idTable.idAt(ordinal)).byteLength;
25297
25470
  }
25298
25471
  chargePostings(segment, costs, chargesTokens);
25299
25472
  chargeColumns(segment, costs);
@@ -25414,16 +25587,26 @@ function emptyPayload() {
25414
25587
  sizes: []
25415
25588
  };
25416
25589
  }
25417
- function contentsOfGroup(group, schema, tombstones) {
25590
+ function noSpellingsOf(analysis) {
25591
+ if (analysis === null || analysis.fields.length === 0) return null;
25592
+ return { fields: [...analysis.fields], tokens: [], spellings: [], counts: [] };
25593
+ }
25594
+ function contentsOfGroup(group, request, tombstones) {
25595
+ const { schema, partialWordAnalysis } = request;
25418
25596
  if (group === null) {
25597
+ const payload = { ...emptyPayload(), partialSpellings: noSpellingsOf(partialWordAnalysis) };
25419
25598
  return {
25420
25599
  schema,
25421
25600
  tombstones,
25422
- arrays: segmentArraysOf(emptyPayload(), { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
25601
+ arrays: segmentArraysOf(payload, { blob: new Uint8Array(0), offsets: new Uint32Array(1) })
25423
25602
  };
25424
25603
  }
25425
- const merged = mergeSegmentContents(group.selections);
25426
- return { schema, tombstones, arrays: segmentArraysOf(merged.payload, merged.documents) };
25604
+ const merged = mergeSegmentContents(group.selections, partialWordAnalysis);
25605
+ return {
25606
+ schema,
25607
+ tombstones,
25608
+ arrays: segmentArraysOf(merged.payload, merged.documents, merged.reusedPartialWords)
25609
+ };
25427
25610
  }
25428
25611
  function requireRoomForAnotherSegment(state) {
25429
25612
  if (state.segments.length < MAX_SEGMENTS_PER_PARTITION) return;
@@ -25436,7 +25619,7 @@ function requireRoomForAnotherSegment(state) {
25436
25619
  async function writeGroup(state, group) {
25437
25620
  const { request } = state;
25438
25621
  const tombstones = state.tombstonesWritten ? [] : [...request.tombstones];
25439
- const contents = contentsOfGroup(group, request.schema, tombstones);
25622
+ const contents = contentsOfGroup(group, request, tombstones);
25440
25623
  const encoded = encodeKeywordSegment(contents);
25441
25624
  if (HEADER_SIZE + encoded.byteLength > MAX_FILE_BYTES && group !== null && group.documentCount > 1) {
25442
25625
  const [first, second] = halvesOf(group);
@@ -25552,6 +25735,7 @@ async function compactPartitionSegments(input) {
25552
25735
  partitionId: input.partitionId,
25553
25736
  nextSegmentId: input.nextSegmentId,
25554
25737
  schema: input.schema,
25738
+ partialWordAnalysis: input.partialWordAnalysis,
25555
25739
  selections: frozen.map((segment) => ({ segment })),
25556
25740
  tombstones: retainedTombstones(frozen, contents),
25557
25741
  mergesIntoOneFile: true
@@ -25567,6 +25751,27 @@ async function compactPartitionSegments(input) {
25567
25751
  }
25568
25752
 
25569
25753
  // src/persistence/durability/segment/write.ts
25754
+ function partialWordAnalysisOf2(config) {
25755
+ const fields = partialFieldsOf(config.schema);
25756
+ if (fields.length === 0) return null;
25757
+ let resolved = null;
25758
+ const resolve = () => {
25759
+ resolved ??= {
25760
+ language: getLanguage(config.language ?? "english"),
25761
+ options: resolvePartitionInsertOptions(config, resolveIndexAnalysis(config))
25762
+ };
25763
+ return resolved;
25764
+ };
25765
+ return {
25766
+ fields,
25767
+ get language() {
25768
+ return resolve().language;
25769
+ },
25770
+ get options() {
25771
+ return resolve().options;
25772
+ }
25773
+ };
25774
+ }
25570
25775
  function resolveCompactionThreshold(configured) {
25571
25776
  if (!Number.isInteger(configured) || configured <= 0 || configured > MAX_SEGMENTS_PER_PARTITION) {
25572
25777
  return DEFAULT_COMPACTION_THRESHOLD;
@@ -25586,6 +25791,7 @@ async function writeWholePartition(context, partitionId, priorPartition, whole)
25586
25791
  partitionId,
25587
25792
  nextSegmentId: priorPartition?.nextSegmentId ?? 0,
25588
25793
  schema: context.input.metadata.schema,
25794
+ partialWordAnalysis: context.partialWordAnalysis,
25589
25795
  selections: whole.selections,
25590
25796
  tombstones: []
25591
25797
  });
@@ -25608,6 +25814,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
25608
25814
  partitionId,
25609
25815
  nextSegmentId,
25610
25816
  schema: context.input.metadata.schema,
25817
+ partialWordAnalysis: context.partialWordAnalysis,
25611
25818
  selections: built.selections,
25612
25819
  tombstones: built.tombstones
25613
25820
  });
@@ -25626,6 +25833,7 @@ async function writeLoggedChanges(context, partitionId, priorPartition, entries)
25626
25833
  indexName: context.indexName,
25627
25834
  partitionId,
25628
25835
  schema: context.input.metadata.schema,
25836
+ partialWordAnalysis: context.partialWordAnalysis,
25629
25837
  segments,
25630
25838
  nextSegmentId,
25631
25839
  compactionThreshold: context.compactionThreshold
@@ -25652,6 +25860,7 @@ async function writeCheckpointSegments(input) {
25652
25860
  input,
25653
25861
  indexName: input.metadata.indexName,
25654
25862
  config,
25863
+ partialWordAnalysis: partialWordAnalysisOf2(config),
25655
25864
  vectorFieldPaths: new Set(extractVectorFieldsFromSchema(config.schema).keys()),
25656
25865
  compactionThreshold: resolveCompactionThreshold(input.compactionThreshold)
25657
25866
  };
@@ -34313,7 +34522,7 @@ async function* exportLines2(header2, checkpoint) {
34313
34522
  }
34314
34523
  function documentExportStream(header2, checkpoint) {
34315
34524
  const lines = exportLines2(header2, checkpoint);
34316
- const encoder9 = new TextEncoder();
34525
+ const encoder8 = new TextEncoder();
34317
34526
  let released = false;
34318
34527
  const release = () => {
34319
34528
  if (released) return;
@@ -34327,14 +34536,14 @@ function documentExportStream(header2, checkpoint) {
34327
34536
  while (text.length < EXPORT_CHUNK_BYTES) {
34328
34537
  const step = await lines.next();
34329
34538
  if (step.done) {
34330
- if (text.length > 0) controller.enqueue(encoder9.encode(text));
34539
+ if (text.length > 0) controller.enqueue(encoder8.encode(text));
34331
34540
  release();
34332
34541
  controller.close();
34333
34542
  return;
34334
34543
  }
34335
34544
  text += step.value;
34336
34545
  }
34337
- controller.enqueue(encoder9.encode(text));
34546
+ controller.enqueue(encoder8.encode(text));
34338
34547
  } catch (error) {
34339
34548
  release();
34340
34549
  controller.error(error);
@@ -34664,10 +34873,10 @@ var REGULAR_FILE = 48;
34664
34873
  var OLD_REGULAR_FILE = 0;
34665
34874
  var PAX_EXTENDED_HEADER = 120;
34666
34875
  var PAX_HEADER_NAME = "PaxHeader";
34667
- var encoder7 = new TextEncoder();
34876
+ var encoder6 = new TextEncoder();
34668
34877
  var decoder7 = new TextDecoder("utf-8", { fatal: true });
34669
34878
  function writeString(block, offset, length, value) {
34670
- const bytes = encoder7.encode(value);
34879
+ const bytes = encoder6.encode(value);
34671
34880
  block.set(bytes.subarray(0, length), offset);
34672
34881
  }
34673
34882
  function writeOctal(block, offset, digits, value) {
@@ -34700,11 +34909,11 @@ function ustarBlock(name, prefix, size, typeflag) {
34700
34909
  return block;
34701
34910
  }
34702
34911
  function splitForUstar(name) {
34703
- if (encoder7.encode(name).byteLength <= NAME_BYTES) return { name, prefix: "" };
34912
+ if (encoder6.encode(name).byteLength <= NAME_BYTES) return { name, prefix: "" };
34704
34913
  for (let slash = name.indexOf("/"); slash >= 0; slash = name.indexOf("/", slash + 1)) {
34705
34914
  const prefix = name.slice(0, slash);
34706
34915
  const rest = name.slice(slash + 1);
34707
- if (encoder7.encode(prefix).byteLength <= PREFIX_BYTES && encoder7.encode(rest).byteLength <= NAME_BYTES) {
34916
+ if (encoder6.encode(prefix).byteLength <= PREFIX_BYTES && encoder6.encode(rest).byteLength <= NAME_BYTES) {
34708
34917
  return { name: rest, prefix };
34709
34918
  }
34710
34919
  }
@@ -34713,7 +34922,7 @@ function splitForUstar(name) {
34713
34922
  function paxRecord(key, value) {
34714
34923
  const body = ` ${key}=${value}
34715
34924
  `;
34716
- const bodyBytes = encoder7.encode(body).byteLength;
34925
+ const bodyBytes = encoder6.encode(body).byteLength;
34717
34926
  let digits = 1;
34718
34927
  while (String(bodyBytes + digits).length !== digits) digits += 1;
34719
34928
  return `${bodyBytes + digits}${body}`;
@@ -34725,7 +34934,7 @@ function tarPadding(size) {
34725
34934
  function tarEntryHeaders(name, size) {
34726
34935
  const split = splitForUstar(name);
34727
34936
  if (split !== null) return [ustarBlock(split.name, split.prefix, size, REGULAR_FILE)];
34728
- const record = encoder7.encode(paxRecord("path", name));
34937
+ const record = encoder6.encode(paxRecord("path", name));
34729
34938
  return [
34730
34939
  ustarBlock(PAX_HEADER_NAME, "", record.byteLength, PAX_EXTENDED_HEADER),
34731
34940
  record,
@@ -35436,7 +35645,9 @@ var NUMBER_TAG2 = 4;
35436
35645
  var STRING_TAG = 5;
35437
35646
  var LIST_TAG = 6;
35438
35647
  var MAP_TAG = 7;
35439
- var encoder8 = new TextEncoder();
35648
+ var LENGTH_BYTES = 4;
35649
+ var LAST_ASCII_UNIT = 127;
35650
+ var encoder7 = new TextEncoder();
35440
35651
  function ensureCapacity2(stream, extra) {
35441
35652
  if (stream.length + extra <= stream.bytes.length) return;
35442
35653
  const grown = new Uint8Array(Math.max(stream.bytes.length * 2, stream.length + extra));
@@ -35465,13 +35676,28 @@ function writeNumber(stream, value) {
35465
35676
  }
35466
35677
  stream.length += 8;
35467
35678
  }
35468
- function writeString2(stream, value) {
35469
- const encoded = encoder8.encode(value);
35679
+ function writeEncodedString(stream, value) {
35680
+ const encoded = encoder7.encode(value);
35470
35681
  writeUint32(stream, encoded.length);
35471
35682
  ensureCapacity2(stream, encoded.length);
35472
35683
  stream.bytes.set(encoded, stream.length);
35473
35684
  stream.length += encoded.length;
35474
35685
  }
35686
+ function writeString2(stream, value) {
35687
+ ensureCapacity2(stream, LENGTH_BYTES + value.length);
35688
+ const start = stream.length + LENGTH_BYTES;
35689
+ const bytes = stream.bytes;
35690
+ for (let at = 0; at < value.length; at++) {
35691
+ const unit = value.charCodeAt(at);
35692
+ if (unit > LAST_ASCII_UNIT) {
35693
+ writeEncodedString(stream, value);
35694
+ return;
35695
+ }
35696
+ bytes[start + at] = unit;
35697
+ }
35698
+ writeUint32(stream, value.length);
35699
+ stream.length += value.length;
35700
+ }
35475
35701
  var NumberList = class {
35476
35702
  constructor(members) {
35477
35703
  this.members = members;
@@ -35748,16 +35974,18 @@ function requireBooleanPartialWords(params) {
35748
35974
  received: setting === null ? "null" : typeof setting
35749
35975
  });
35750
35976
  }
35751
- function requireValidQueryOptions(params, config) {
35977
+ function validQueryParams(params, config) {
35752
35978
  const strict = config.strict === true;
35753
35979
  requireValidTypoTolerance(params);
35754
35980
  requireBooleanPartialWords(params);
35755
35981
  requireSortableFields(params.sort, config.schema, strict);
35756
- requireValidQueryFilter(params.filters, config.schema);
35982
+ const filters = validQueryFilter(params.filters, config.schema);
35757
35983
  requireUsableQueryFields(params, config.schema, strict);
35984
+ return filters === params.filters ? params : { ...params, filters };
35758
35985
  }
35759
- function requireValidQueryFilter(filters, schema) {
35760
- if (filters !== void 0) requireValidFilter(filters, flattenSchema(schema));
35986
+ function validQueryFilter(filters, schema) {
35987
+ if (filters === void 0 || filters === null) return filters;
35988
+ return validatedFilterOf(filters, flattenSchema(schema));
35761
35989
  }
35762
35990
  function requireVectorSearchable(params, context, needsVector) {
35763
35991
  if (!needsVector) return;
@@ -35873,7 +36101,7 @@ function executeListDocuments(params, context) {
35873
36101
  const limit = clampListLimit(params.limit);
35874
36102
  const signature = sortSignatureOf(params.sort);
35875
36103
  requireSortableFields(params.sort, schema, context.strict === true);
35876
- requireValidQueryFilter(params.filters, schema);
36104
+ const filters = validQueryFilter(params.filters, schema);
35877
36105
  const binding = listBindingOf(params.filters);
35878
36106
  let cursor = null;
35879
36107
  if (params.cursor !== void 0) {
@@ -35881,7 +36109,7 @@ function executeListDocuments(params, context) {
35881
36109
  requireMatchingCursor(cursor, params.cursor, signature, false, binding);
35882
36110
  }
35883
36111
  const partitions = context.partitionIds === void 0 ? manager.getAllPartitions() : context.partitionIds.map((partitionId) => manager.partitionAt(partitionId)).filter((partition) => partition !== void 0);
35884
- const filtered = params.filters === void 0 ? null : collectFilterMatches(partitions, params.filters, schema, context.patternWorkCap);
36112
+ const filtered = filters === void 0 ? null : collectFilterMatches(partitions, filters, schema, context.patternWorkCap);
35885
36113
  const total = filtered === null ? countAcrossPartitions(manager, context.partitionIds, partitions) : filtered.total;
35886
36114
  const page = params.sort === void 0 || signature === null ? pageInIdOrder(partitions, cursor, filtered, limit, binding) : pageInSortOrder(partitions, cursor, filtered, limit, params.sort, signature, schema, binding);
35887
36115
  const projection = resolveProjection(params.document);
@@ -36618,28 +36846,28 @@ function requireCursorDepth(cursor, signature, binding) {
36618
36846
  requireMatchingCursor(decoded, cursor, signature, true, binding);
36619
36847
  return decoded.depth ?? 0;
36620
36848
  }
36621
- async function executeQuery(params, callerContext) {
36849
+ async function executeQuery(request, callerContext) {
36622
36850
  const context = withPatternWork(callerContext);
36623
36851
  const { manager, language, config, workerSearch, indexName } = context;
36624
36852
  const startTime = now();
36625
- const limit = clampLimit(params.limit);
36626
- const offset = clampOffset(params.offset);
36853
+ const limit = clampLimit(request.limit);
36854
+ const offset = clampOffset(request.offset);
36627
36855
  requireWithinResultWindow(limit, offset);
36628
- const sortFields = normalizeSort(params.sort);
36629
- requireKnownMode(params);
36630
- const sortSignature = sortSignatureOf(params.sort);
36631
- const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
36632
- const hasVector = params.vector !== void 0 && params.vector.value !== void 0;
36633
- const isHybridMode = params.mode === "hybrid" || hasTerm && hasVector;
36634
- const isVectorOnly = (params.mode === "vector" || hasVector && !hasTerm) && !isHybridMode;
36635
- if (sortSignature !== null && (isHybridMode || params.hybrid !== void 0)) {
36856
+ const sortFields = normalizeSort(request.sort);
36857
+ requireKnownMode(request);
36858
+ const sortSignature = sortSignatureOf(request.sort);
36859
+ const hasTerm = request.term !== void 0 && request.term.trim().length > 0;
36860
+ const hasVector = request.vector !== void 0 && request.vector.value !== void 0;
36861
+ const isHybridMode = request.mode === "hybrid" || hasTerm && hasVector;
36862
+ const isVectorOnly = (request.mode === "vector" || hasVector && !hasTerm) && !isHybridMode;
36863
+ if (sortSignature !== null && (isHybridMode || request.hybrid !== void 0)) {
36636
36864
  throw new NarsilError(
36637
36865
  ErrorCodes.SEARCH_INVALID_MODE,
36638
36866
  "A hybrid query cannot carry a sort, because fusion defines the order of hybrid results",
36639
36867
  { sort: sortFields.map((entry) => entry.field) }
36640
36868
  );
36641
36869
  }
36642
- requireValidQueryOptions(params, config);
36870
+ const params = validQueryParams(request, config);
36643
36871
  requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
36644
36872
  let paginated;
36645
36873
  let nextCursor;
@@ -36802,12 +37030,12 @@ async function executeQuery(params, callerContext) {
36802
37030
  coverage: coverageFor(manager, context.partitionIds)
36803
37031
  };
36804
37032
  }
36805
- async function executePreflight(params, callerContext) {
37033
+ async function executePreflight(request, callerContext) {
36806
37034
  const context = withPatternWork(callerContext);
36807
37035
  const { manager, language, config, workerSearch, indexName } = context;
36808
37036
  const startTime = now();
36809
- requireKnownMode(params);
36810
- requireValidQueryOptions(params, config);
37037
+ requireKnownMode(request);
37038
+ const params = validQueryParams(request, config);
36811
37039
  const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
36812
37040
  const hasVector = params.vector !== void 0 && params.vector.value !== void 0;
36813
37041
  const isHybridMode = params.mode === "hybrid" || hasTerm && hasVector;