@delali/narsil 0.3.1-next.78 → 0.3.1-next.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +3 -3
  2. package/dist/{capabilities-1lEYL3WK.d.ts → capabilities-CcTDWh_P.d.ts} +2 -2
  3. package/dist/{chunk-AMGDG4ZJ.mjs → chunk-23HJ5QXQ.mjs} +1 -1
  4. package/dist/{chunk-R6WBLBTC.mjs → chunk-4QDQEJDV.mjs} +9 -45
  5. package/dist/chunk-DQABW5UQ.mjs +140 -0
  6. package/dist/chunk-GG2FBGVZ.mjs +30 -0
  7. package/dist/{chunk-XSZTF4ZQ.mjs → chunk-HJASKX4F.mjs} +1 -1
  8. package/dist/{chunk-DFW3U5UW.mjs → chunk-K4FZ55Q2.mjs} +21 -192
  9. package/dist/{chunk-PJWG547Z.mjs → chunk-MEOOYKWL.mjs} +3 -3
  10. package/dist/{chunk-55WEJKWT.mjs → chunk-NSXQCE4K.mjs} +17 -63
  11. package/dist/{chunk-PXDQFYD5.mjs → chunk-OSF3SGJV.mjs} +23 -139
  12. package/dist/{chunk-RTR44V2A.mjs → chunk-P2PYEZZE.mjs} +586 -4
  13. package/dist/{chunk-Z3BXCZXS.mjs → chunk-QPYV22XH.mjs} +2 -1
  14. package/dist/{chunk-4344I4MR.mjs → chunk-R2XCJIO7.mjs} +335 -12
  15. package/dist/{chunk-IBR6634K.mjs → chunk-R6A773WK.mjs} +2 -69
  16. package/dist/chunk-WQBCBKY3.mjs +8 -0
  17. package/dist/{chunk-P2BEMMOD.mjs → chunk-YGLSR7BC.mjs} +150 -203
  18. package/dist/chunk-YTJCGKM4.mjs +56 -0
  19. package/dist/{client-ZXcbSViH.d.ts → client-CsybN673.d.ts} +2 -2
  20. package/dist/client.d.ts +6 -6
  21. package/dist/{config-DTe5EbWW.d.ts → config-BCNQtszf.d.ts} +1 -1
  22. package/dist/{config-DHL2KsfD.d.ts → config-CFonR_gY.d.ts} +1 -1
  23. package/dist/direct-executor-VNMQ6NIH.mjs +19 -0
  24. package/dist/distribution/coordinator/etcd.mjs +2 -1
  25. package/dist/distribution/coordinator/in-memory.mjs +2 -1
  26. package/dist/distribution.d.ts +5 -5
  27. package/dist/distribution.mjs +15 -13
  28. package/dist/{engine-BjPBzWkA.d.ts → engine-B0oIF1_r.d.ts} +1 -1
  29. package/dist/index.browser.mjs +113 -516
  30. package/dist/index.d.ts +5 -5
  31. package/dist/index.mjs +14 -12
  32. package/dist/invalidation/filesystem.mjs +2 -1
  33. package/dist/{memory-DW3OJ0im.d.ts → memory-B6SBOKlS.d.ts} +1287 -1287
  34. package/dist/partial-words.mjs +5 -4
  35. package/dist/pattern.mjs +4 -1
  36. package/dist/persistence/durability/checkpoint-worker.d.ts +7 -7
  37. package/dist/persistence/durability/checkpoint-worker.mjs +14 -12
  38. package/dist/react.d.ts +6 -6
  39. package/dist/server.d.ts +8 -8
  40. package/dist/server.mjs +10 -7
  41. package/dist/{tasks-BTbwBqar.d.ts → tasks-CYAQe6qd.d.ts} +1 -1
  42. package/dist/{thread-XBDVXKUE.mjs → thread-P6QJ2Q5O.mjs} +11 -8
  43. package/dist/vector/search-worker.mjs +3 -2
  44. package/dist/workers/entry.mjs +3 -3
  45. package/package.json +9 -7
  46. package/dist/chunk-IOZPBNSG.mjs +0 -55
  47. package/dist/chunk-WFVOBZS4.mjs +0 -485
  48. package/dist/direct-executor-WB2HKU4N.mjs +0 -17
package/README.md CHANGED
@@ -100,7 +100,7 @@ Every hit holds the document, its id, and its BM25 score. Beside the hits, `resu
100
100
 
101
101
  ## Features
102
102
 
103
- **Search.** For [full-text search](https://github.com/assetcorp/narsil/blob/main/docs/full-text-search.md), the engine scores with BM25 and supports field boosting, fuzzy matching through bounded Levenshtein distance, search as you type through last-word prefix matching, and term-coverage and score thresholds. You can combine a query with [filters, facets, sorting, grouping, cursor pagination, and pinned results](https://github.com/assetcorp/narsil/blob/main/docs/filters-facets-and-pagination.md).
103
+ **Search.** For [full-text search](https://github.com/assetcorp/narsil/blob/main/docs/full-text-search.md), the engine scores with BM25 and supports field boosting, fuzzy matching through bounded Levenshtein distance, search as you type through last-word prefix matching, and term-coverage and score thresholds. You can combine a query with [filters, facets, sorting, grouping, cursor pagination, and pinned results](https://github.com/assetcorp/narsil/blob/main/docs/filters-facets-and-pagination.md). On a field that you declare for [pattern search](https://github.com/assetcorp/narsil/blob/main/docs/filters-facets-and-pagination.md#pattern-tests), a filter can find text anywhere inside a value with `contains`, test the whole value against a wildcard pattern, look for a match of a regular expression, and accept a typo or two in an `eq` test. With [partial-word search](https://github.com/assetcorp/narsil/blob/main/docs/full-text-search.md#matching-inside-words), a query word also matches inside longer words, so a search for `proof` finds `waterproof`.
104
104
 
105
105
  **Vector and hybrid retrieval.** For [vector search](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md), the engine compares vectors by cosine similarity, dot product, or Euclidean distance. It scans a field exactly until the field holds 1,024 vectors, a count that you can change, after which it builds an HNSW graph with optimised scalar quantization on by default. Outside a browser, on an arm64 or x64 machine under macOS, Linux, or Windows, the engine searches that graph through a [native core in C](https://github.com/assetcorp/narsil/blob/main/docs/vector-search.md#native-search-core), which returns the same results as its WebAssembly search. For [hybrid search](https://github.com/assetcorp/narsil/blob/main/docs/hybrid-search.md), the engine fuses BM25 and vector rankings through reciprocal rank fusion or linear blending. [Embedding adapters](https://github.com/assetcorp/narsil/blob/main/docs/embedding-adapters.md) turn text into vectors on insert and query, through OpenAI, local Transformers.js models, or an adapter of your own.
106
106
 
@@ -110,7 +110,7 @@ Every hit holds the document, its id, and its BM25 score. Beside the hits, `resu
110
110
 
111
111
  **Scale.** The engine routes each document to a [partition](https://github.com/assetcorp/narsil/blob/main/docs/partitions-and-workers.md) by a deterministic hash. When you call `rebalance()`, the engine reshapes the partitions online and buffers incoming writes in a write-ahead queue. Once an index holds 1,000 documents, the engine serves keyword queries from worker copies on half its worker threads, while the HTTP server receives requests on the threads that hold those copies.
112
112
 
113
- **Operations.** The [HTTP server](https://github.com/assetcorp/narsil/blob/main/docs/http-server.md) subpath wraps an engine in a REST API with health probes, bulk NDJSON import, snapshot and restore endpoints, and task-based long operations. The [client](https://github.com/assetcorp/narsil/blob/main/docs/client.md) subpath calls every one of those routes from a browser or from Node under the engine's own method names. The [React](https://github.com/assetcorp/narsil/blob/main/docs/react.md) subpath exposes those methods to components as hooks. For observability, you have [events, plugins, and memory reporting](https://github.com/assetcorp/narsil/blob/main/docs/observability.md). [Language modules](https://github.com/assetcorp/narsil/blob/main/docs/language-support.md) cover 107 languages as separate entry points, including 20 African languages.
113
+ **Operations.** The [HTTP server](https://github.com/assetcorp/narsil/blob/main/docs/http-server.md) subpath wraps an engine in a REST API with health probes, bulk NDJSON import, snapshot and restore endpoints, and task-based long operations. The [client](https://github.com/assetcorp/narsil/blob/main/docs/client.md) subpath calls every one of those routes from a browser or from Node under the engine's own method names. The [React](https://github.com/assetcorp/narsil/blob/main/docs/react.md) subpath turns the read methods into hooks that call either a client or an engine in the same page. For observability, you have [events, plugins, and memory reporting](https://github.com/assetcorp/narsil/blob/main/docs/observability.md). [Language modules](https://github.com/assetcorp/narsil/blob/main/docs/language-support.md) cover 107 languages as separate entry points, including 20 African languages.
114
114
 
115
115
  ## Documentation
116
116
 
@@ -130,7 +130,7 @@ Every hit holds the document, its id, and its BM25 score. Beside the hits, `resu
130
130
  | [HTTP server](https://github.com/assetcorp/narsil/blob/main/docs/http-server.md) | Wrapping an engine in a REST API, every route it serves, and long-running tasks |
131
131
  | [Cluster mode](https://github.com/assetcorp/narsil/blob/main/docs/cluster.md) | Multi-node indexes: nodes and roles, replication, routed writes, distributed searches and reads, and the calls that fail in a cluster |
132
132
  | [Client](https://github.com/assetcorp/narsil/blob/main/docs/client.md) | Reaching a server from a browser or Node, following a task, and the codes it raises |
133
- | [React](https://github.com/assetcorp/narsil/blob/main/docs/react.md) | The hooks over the client, one shared request per key, and loading a corpus from a component |
133
+ | [React](https://github.com/assetcorp/narsil/blob/main/docs/react.md) | The hooks over a client or an engine in the page, one shared request per key, and loading a corpus from a component |
134
134
  | [Observability](https://github.com/assetcorp/narsil/blob/main/docs/observability.md) | Plugin hooks, engine events, and memory reporting |
135
135
  | [Errors](https://github.com/assetcorp/narsil/blob/main/docs/errors.md) | The codes you handle most often, and the three sets that `NarsilError.code` covers |
136
136
 
@@ -1,6 +1,6 @@
1
1
  import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
2
- import { C as ClusterNamespace } from './config-DHL2KsfD.js';
3
- import { h as TaskStore } from './tasks-BTbwBqar.js';
2
+ import { C as ClusterNamespace } from './config-CFonR_gY.js';
3
+ import { h as TaskStore } from './tasks-CYAQe6qd.js';
4
4
 
5
5
  /**
6
6
  * Context handed to {@link OnRequestHook} for every inbound request, captured
@@ -1,7 +1,7 @@
1
1
  import { OSQ_STEPS, OSQ_GRID, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ4_NARROW_DIMENSION_LIMIT, DEFAULT_OSQ4_NARROW_OVERSAMPLE, DEFAULT_OVERSAMPLE_BY_BITS, OSQ_QUERY_BITS, OSQ_LAMBDA } from './chunk-ZJ4CSAB2.mjs';
2
2
  import { nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback } from './chunk-2E35UM3G.mjs';
3
3
  import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-KEHK2O6P.mjs';
4
- import { compareCodePoints } from './chunk-IBR6634K.mjs';
4
+ import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
5
5
  import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
6
6
  import { readSync, closeSync, openSync } from 'fs';
7
7
 
@@ -1,10 +1,12 @@
1
- import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, fnv1aBytes, validatedFilterOf, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, undeclaredFieldOnStrictIndex } from './chunk-P2BEMMOD.mjs';
2
- import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_MAX_PER_GROUP, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_OFFSET, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-IOZPBNSG.mjs';
1
+ import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, fnv1aBytes, validatedFilterOf, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, partialWordsToHighlight, partialWordHighlighter, undeclaredFieldOnStrictIndex } from './chunk-YGLSR7BC.mjs';
2
+ import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-GG2FBGVZ.mjs';
3
3
  import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
4
- import { tokenize } from './chunk-WFVOBZS4.mjs';
5
- import { partialFieldsOf, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, flattenSchema, isWordIndexedFieldType, isGeopointOrVectorType } from './chunk-PXDQFYD5.mjs';
4
+ import { tokenize } from './chunk-P2PYEZZE.mjs';
5
+ import { partialFieldsOf, DEFAULT_MAX_PER_GROUP, flattenSchema, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-OSF3SGJV.mjs';
6
6
  import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
7
- import { holdsUnpairedSurrogate, compareCodePoints, compareComparableKeys, codePointCount } from './chunk-IBR6634K.mjs';
7
+ import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isWordIndexedFieldType, isGeopointOrVectorType } from './chunk-DQABW5UQ.mjs';
8
+ import { compareComparableKeys } from './chunk-R6A773WK.mjs';
9
+ import { holdsUnpairedSurrogate, compareCodePoints } from './chunk-YTJCGKM4.mjs';
8
10
  import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
9
11
 
10
12
  // src/search/cursor-binding.ts
@@ -855,31 +857,6 @@ function applySnippetTruncation(taggedText, originalText, positions, maxLen, pre
855
857
  if (needsTrailingEllipsis) snippet = `${snippet}...`;
856
858
  return { snippet, visiblePositions };
857
859
  }
858
- function containsPartialWord(spelling, words) {
859
- for (const word of words) {
860
- if (spelling.length > word.length && spelling.includes(word)) return true;
861
- }
862
- return false;
863
- }
864
- function partialWordMatcher(text, language, partial) {
865
- const indexed = tokenize(text, language, {
866
- stem: true,
867
- removeStopWords: true,
868
- collectSurfaces: true,
869
- customTokenizer: partial.customTokenizer,
870
- stopWordOverride: partial.stopWordOverride
871
- });
872
- const positions = /* @__PURE__ */ new Set();
873
- const spellings = /* @__PURE__ */ new Set();
874
- for (let at = 0; at < indexed.tokens.length; at++) {
875
- const spelling = indexed.surfaces?.[at] ?? indexed.tokens[at].token;
876
- if (!containsPartialWord(spelling, partial.words)) continue;
877
- positions.add(indexed.tokens[at].position);
878
- spellings.add(spelling);
879
- }
880
- if (partial.customTokenizer !== void 0) return (_position, fieldToken) => spellings.has(fieldToken);
881
- return (position) => positions.has(position);
882
- }
883
860
  function highlightField(text, queryTokens, language, options) {
884
861
  const preTag = options?.preTag ?? "<mark>";
885
862
  const postTag = options?.postTag ?? "</mark>";
@@ -916,7 +893,7 @@ function highlightField(text, queryTokens, language, options) {
916
893
  matchByFieldToken.set(fieldToken, matches);
917
894
  return matches;
918
895
  };
919
- const matchesInside = options?.partialWords === void 0 ? null : partialWordMatcher(text, language, options.partialWords);
896
+ const matchesInside = options?.partialWords === void 0 ? null : partialWordHighlighter(text, language, options.partialWords);
920
897
  for (let i = 0; i < fieldResult.tokens.length; i++) {
921
898
  if (i >= charOffsets.length) break;
922
899
  const fieldToken = fieldResult.tokens[i].token;
@@ -966,20 +943,7 @@ function partialWordTagging(params, schema, queryTokens, written, analysis) {
966
943
  if (params.partialWords !== true || params.exact === true) return null;
967
944
  const fields = partialFieldsOf(schema).filter((field) => params.fields === void 0 || params.fields.includes(field));
968
945
  if (fields.length === 0) return null;
969
- const writtenAt = /* @__PURE__ */ new Map();
970
- for (const word of written) {
971
- if (!writtenAt.has(word.position)) writtenAt.set(word.position, word.token);
972
- }
973
- const seenTokens = /* @__PURE__ */ new Set();
974
- const words = [];
975
- for (const queryToken of queryTokens) {
976
- if (seenTokens.has(queryToken.token)) continue;
977
- seenTokens.add(queryToken.token);
978
- const word = writtenAt.get(queryToken.position);
979
- if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
980
- words.push(word);
981
- if (words.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
982
- }
946
+ const words = partialWordsToHighlight(queryTokens, written);
983
947
  if (words.length === 0) return null;
984
948
  return {
985
949
  fields: new Set(fields),
@@ -0,0 +1,140 @@
1
+ import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
2
+
3
+ // src/schema/constants.ts
4
+ var MAX_NESTING_DEPTH = 4;
5
+ var MAX_DOCUMENT_NESTING_DEPTH = 32;
6
+ var DEFAULT_PATTERN_VALUE_LIMIT = 8192;
7
+ var MIN_PATTERN_VALUE_LIMIT = 1;
8
+ var MAX_PATTERN_VALUE_LIMIT = 65536;
9
+ var MAX_VECTOR_DIMENSION = 4096;
10
+ var MAX_STORED_VECTOR_DIMENSION = 65535;
11
+ var MAX_TEXT_FIELDS = 65536;
12
+ var MAX_DOCUMENT_BYTES = 16777216;
13
+ var PARSED_FIELD_TYPE_CACHE_ENTRIES = 1024;
14
+
15
+ // src/schema/validator/shared.ts
16
+ var VECTOR_PATTERN = /^vector\[(\d+)]$/;
17
+ function isGeopointOrVectorType(fieldType) {
18
+ return fieldType === "geopoint" || fieldType !== void 0 && VECTOR_PATTERN.test(fieldType);
19
+ }
20
+ var FIELD_NAME_PATTERN = /^[A-Za-z0-9_]+$/;
21
+ var RESERVED_ROOT_FIELDS = /* @__PURE__ */ new Set(["id"]);
22
+ var PROTOTYPE_POLLUTION_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
23
+ function isPlainObject(value) {
24
+ return typeof value === "object" && value !== null && !Array.isArray(value);
25
+ }
26
+
27
+ // src/schema/validator/field-type.ts
28
+ var FIELD_OPTION_ORDER = ["sortable", "pattern", "partial"];
29
+ var OPTIONS_BY_BASE = {
30
+ string: FIELD_OPTION_ORDER,
31
+ "string[]": FIELD_OPTION_ORDER,
32
+ verbatim: ["sortable"],
33
+ "verbatim[]": ["sortable"],
34
+ number: [],
35
+ "number[]": [],
36
+ boolean: [],
37
+ "boolean[]": [],
38
+ enum: [],
39
+ "enum[]": [],
40
+ geopoint: [],
41
+ vector: []
42
+ };
43
+ var TYPE_SEPARATOR = ":";
44
+ function isNamedBase(name) {
45
+ return name !== "vector" && Object.hasOwn(OPTIONS_BY_BASE, name);
46
+ }
47
+ function describeAllowedOptions(baseName, allowed) {
48
+ if (allowed.length === 0) return `the "${baseName}" type has no options`;
49
+ const quoted = allowed.map((option) => `"${option}"`);
50
+ if (quoted.length === 1) return `the only option of the "${baseName}" type is ${quoted[0]}`;
51
+ return `the options of the "${baseName}" type are ${quoted.slice(0, -1).join(", ")}, and ${quoted[quoted.length - 1]}`;
52
+ }
53
+ function invalidType(path, type, message) {
54
+ throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, message, { field: path, type });
55
+ }
56
+ function parseBase(baseName, path, type) {
57
+ if (isNamedBase(baseName)) return { base: baseName, dimension: 0 };
58
+ const vectorMatch = VECTOR_PATTERN.exec(baseName);
59
+ if (vectorMatch === null) {
60
+ invalidType(path, type, `Field "${path}" has an unsupported type: "${type}"`);
61
+ }
62
+ const dimension = Number.parseInt(vectorMatch[1], 10);
63
+ if (dimension <= 0 || dimension > MAX_STORED_VECTOR_DIMENSION) {
64
+ throw new NarsilError(
65
+ ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
66
+ `Vector field "${path}" has ${dimension} dimensions, but a vector field holds from 1 to ${MAX_STORED_VECTOR_DIMENSION}`,
67
+ { field: path, dimension, maximum: MAX_STORED_VECTOR_DIMENSION }
68
+ );
69
+ }
70
+ return { base: "vector", dimension };
71
+ }
72
+ function parseFieldType(type, path) {
73
+ const separatorIndex = type.indexOf(TYPE_SEPARATOR);
74
+ const baseName = separatorIndex === -1 ? type : type.slice(0, separatorIndex);
75
+ const { base, dimension } = parseBase(baseName, path, type);
76
+ const allowed = OPTIONS_BY_BASE[base];
77
+ const options = /* @__PURE__ */ new Set();
78
+ if (separatorIndex !== -1) {
79
+ for (const option of type.slice(separatorIndex + 1).split(TYPE_SEPARATOR)) {
80
+ if (!allowed.includes(option)) {
81
+ invalidType(
82
+ path,
83
+ type,
84
+ `Field "${path}" has type "${type}" with the option "${option}", but ${describeAllowedOptions(baseName, allowed)}`
85
+ );
86
+ }
87
+ if (options.has(option)) {
88
+ invalidType(path, type, `Field "${path}" has the "${option}" option twice in type "${type}"`);
89
+ }
90
+ options.add(option);
91
+ }
92
+ }
93
+ const ordered = FIELD_OPTION_ORDER.filter((option) => options.has(option));
94
+ const verbatim = base === "verbatim" || base === "verbatim[]";
95
+ return {
96
+ type: [baseName, ...ordered].join(TYPE_SEPARATOR),
97
+ base,
98
+ sortable: options.has("sortable"),
99
+ pattern: verbatim || options.has("pattern"),
100
+ partial: options.has("partial"),
101
+ dimension
102
+ };
103
+ }
104
+ var parsedTypesByName = /* @__PURE__ */ new Map();
105
+ function parsedTypeOf(type) {
106
+ if (type === void 0) return void 0;
107
+ const known = parsedTypesByName.get(type);
108
+ if (known !== void 0) return known;
109
+ try {
110
+ const parsed = Object.freeze(parseFieldType(type, ""));
111
+ if (parsedTypesByName.size < PARSED_FIELD_TYPE_CACHE_ENTRIES) parsedTypesByName.set(type, parsed);
112
+ return parsed;
113
+ } catch {
114
+ return void 0;
115
+ }
116
+ }
117
+ function withSortableOption(parsed) {
118
+ if (parsed.sortable) return parsed.type;
119
+ const separatorIndex = parsed.type.indexOf(TYPE_SEPARATOR);
120
+ const baseName = separatorIndex === -1 ? parsed.type : parsed.type.slice(0, separatorIndex);
121
+ return parseFieldType(`${baseName}${TYPE_SEPARATOR}sortable${parsed.type.slice(baseName.length)}`, "").type;
122
+ }
123
+ function isListBase(base) {
124
+ return base.endsWith("[]");
125
+ }
126
+ function isWordIndexedBase(base) {
127
+ return base === "string" || base === "string[]";
128
+ }
129
+ function isSingleStringFieldType(fieldType) {
130
+ return parsedTypeOf(fieldType)?.base === "string";
131
+ }
132
+ function isWordIndexedFieldType(fieldType) {
133
+ const parsed = parsedTypeOf(fieldType);
134
+ return parsed !== void 0 && isWordIndexedBase(parsed.base);
135
+ }
136
+ function isStringValuedBase(base) {
137
+ return base === "string" || base === "string[]" || base === "verbatim" || base === "verbatim[]";
138
+ }
139
+
140
+ export { DEFAULT_PATTERN_VALUE_LIMIT, FIELD_NAME_PATTERN, MAX_DOCUMENT_BYTES, MAX_DOCUMENT_NESTING_DEPTH, MAX_NESTING_DEPTH, MAX_PATTERN_VALUE_LIMIT, MAX_STORED_VECTOR_DIMENSION, MAX_TEXT_FIELDS, MAX_VECTOR_DIMENSION, MIN_PATTERN_VALUE_LIMIT, PROTOTYPE_POLLUTION_KEYS, RESERVED_ROOT_FIELDS, VECTOR_PATTERN, isGeopointOrVectorType, isListBase, isPlainObject, isSingleStringFieldType, isStringValuedBase, isWordIndexedBase, isWordIndexedFieldType, parseFieldType, parsedTypeOf, withSortableOption };
@@ -0,0 +1,30 @@
1
+ import { DEFAULT_PAGE_SIZE } from './chunk-OSF3SGJV.mjs';
2
+
3
+ // src/engine/constants.ts
4
+ var MIN_DOCUMENTS_FOR_SEGMENTS = 64;
5
+ var BATCH_CHUNK_SIZE = 1e3;
6
+ var DEFAULT_LIMIT = DEFAULT_PAGE_SIZE;
7
+ var DEFAULT_OFFSET = 0;
8
+ var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
9
+ var ABSOLUTE_WINDOWS_PATH_PATTERN = /^[a-zA-Z]:[\\/]/;
10
+ var MODULE_URL_SCHEME_PATTERN = /^(file|https?|data|node):/;
11
+ var MAX_INDEX_NAME_LENGTH = 255;
12
+ var MAX_DOC_ID_LENGTH = 512;
13
+ var EXPORT_CHUNK_BYTES = 1048576;
14
+ var MAX_EXPORT_LINE_BYTES = 117443592;
15
+ var MAX_INDEX_CONFIGURATION_BYTES = 16777216;
16
+ var IMPORT_BATCH_DOCUMENTS = 1e3;
17
+ var MAX_IMPORT_REFUSALS_REPORTED = 1e3;
18
+ var MIN_SWEEP_INTERVAL_MS = 100;
19
+ var MAX_SWEEP_INTERVAL_MS = 6e4;
20
+ var HEAP_PRESSURE_WARNING_FRACTION = 0.9;
21
+ var HEAP_PRESSURE_REARM_FRACTION = 0.8;
22
+ var STATISTICS_BROADCAST_INTERVAL_MS = 5e3;
23
+ var FOREIGN_STATS_TTL_MS = 6e4;
24
+ var DEFAULT_SUGGEST_LIMIT = 10;
25
+ var MAX_SUGGEST_LIMIT = 100;
26
+ var SUGGEST_OVERSAMPLE_FACTOR = 1.5;
27
+ var SUGGEST_OVERSAMPLE_PADDING = 10;
28
+ var MAX_SUGGEST_SCATTER_LIMIT = Math.ceil(MAX_SUGGEST_LIMIT * SUGGEST_OVERSAMPLE_FACTOR) + SUGGEST_OVERSAMPLE_PADDING;
29
+
30
+ export { ABSOLUTE_WINDOWS_PATH_PATTERN, BATCH_CHUNK_SIZE, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_SUGGEST_LIMIT, EXPORT_CHUNK_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, IMPORT_BATCH_DOCUMENTS, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_EXPORT_LINE_BYTES, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, STATISTICS_BROADCAST_INTERVAL_MS, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING };
@@ -1,4 +1,4 @@
1
- import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-PXDQFYD5.mjs';
1
+ import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-OSF3SGJV.mjs';
2
2
  import { ErrorCodes, NarsilError } from './chunk-I64LXKO3.mjs';
3
3
 
4
4
  // src/narsil/internals.ts
@@ -1,15 +1,17 @@
1
- import { requirePatternSearchFor, patternIndexOf, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, computeFacets, cloneProjected, projectionKeepsField, setNestedValue, fnv1a, requirePatternSearch, PATTERN_HOST, validatedFilterOf, everyValueFacetConfig, mergeFacets, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, undeclaredFilterField, coveringRanges } from './chunk-P2BEMMOD.mjs';
1
+ import { createGrowingPartialWords, requirePatternSearchFor, requirePartialWordSearchFor, buildPartialWordTableFrom, patternIndexOf, encodePartialSpellingsOf, countPartialSpelling, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, expandPartialWords, computeFacets, cloneProjected, projectionKeepsField, setNestedValue, fnv1a, partialSpellingsOfDocuments, createFrozenPartialWords, requirePatternSearch, PATTERN_HOST, validatedFilterOf, everyValueFacetConfig, mergeFacets, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, undeclaredFilterField, coveringRanges } from './chunk-YGLSR7BC.mjs';
2
2
  import { getLanguage, detectRuntime } from './chunk-4MQ5F4IL.mjs';
3
- import { MAX_DOC_ID_LENGTH, GEO_UNIT_METRES } from './chunk-IOZPBNSG.mjs';
4
- import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, isTombstoned, addToOrdinalFilter, ordinalFilterValues, nativeScoresOf, toScore, removeFromOrdinalFilter, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, ordinalFilterHas, STORE_CALIBRATION_GENERATION, entryForOrd, toDistance, sharedMemoryAvailable, requireWholeNeighbourLists, topLayerInParts, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-AMGDG4ZJ.mjs';
3
+ import { MAX_DOC_ID_LENGTH } from './chunk-GG2FBGVZ.mjs';
4
+ import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, isTombstoned, addToOrdinalFilter, ordinalFilterValues, nativeScoresOf, toScore, removeFromOrdinalFilter, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, ordinalFilterHas, STORE_CALIBRATION_GENERATION, entryForOrd, toDistance, sharedMemoryAvailable, requireWholeNeighbourLists, topLayerInParts, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-23HJ5QXQ.mjs';
5
5
  import { noteFallback } from './chunk-2E35UM3G.mjs';
6
- import { GrowableUint32, bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, PATTERN_RUN_CODE_POINTS, bitsetAndInto, bitsetFromSet, isPatternTolerance, bitsetIsEmpty, bitsetAnd, bitsetNot } from './chunk-4344I4MR.mjs';
7
- import { encodePartialSpellings, countSpelling, expandTermPrefix, suggestDisplayTerms, sameFields, spellingCountsOf, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, partialWordSearchModule, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, partitionDocumentFrequency } from './chunk-RTR44V2A.mjs';
8
- import { tokenizeOptions, getNestedValue, visitTokens, textFieldsChanged, getFlatSchema, tokenize, getOrCreateFieldNameIndex, getFieldValueByInternalId } from './chunk-WFVOBZS4.mjs';
9
- import { partialFieldsOf, schemaFieldsOf, isWordIndexedBase, validateDocument, validateDocumentStrict, encodeDocument, extractVectorFieldsFromSchema, parsedTypeOf, isListBase, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, VINCENTY_MAX_ITERATIONS, DEG_TO_RAD } from './chunk-PXDQFYD5.mjs';
6
+ import { GrowableUint32, bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet, bitsetIsEmpty } from './chunk-R2XCJIO7.mjs';
7
+ import { GEO_UNIT_METRES } from './chunk-WQBCBKY3.mjs';
8
+ import { tokenizeOptions, getNestedValue, visitTokens, expandTermPrefix, suggestDisplayTerms, textFieldsChanged, sameFields, codePointOrder, encodeStringBlob, encodeLookupKey, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, getFlatSchema, tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, getOrCreateFieldNameIndex, getFieldValueByInternalId, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, partitionDocumentFrequency } from './chunk-P2PYEZZE.mjs';
9
+ import { schemaFieldsOf, partialFieldsOf, validateDocument, validateDocumentStrict, encodeDocument, extractVectorFieldsFromSchema, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, VINCENTY_MAX_ITERATIONS, DEG_TO_RAD } from './chunk-OSF3SGJV.mjs';
10
10
  import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-KEHK2O6P.mjs';
11
11
  import { fuzzyPrefixOf, boundedLevenshtein } from './chunk-GES7U2PM.mjs';
12
- import { compareCodePoints, toReducedSortValue, readSortField, defaultSortMode, compareComparableValues, truncateSortString, SORT_VALUE_MAX_CODE_POINTS } from './chunk-IBR6634K.mjs';
12
+ import { isWordIndexedBase, parsedTypeOf, isListBase } from './chunk-DQABW5UQ.mjs';
13
+ import { toReducedSortValue, readSortField, defaultSortMode, compareComparableValues, truncateSortString, SORT_VALUE_MAX_CODE_POINTS } from './chunk-R6A773WK.mjs';
14
+ import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
13
15
  import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
14
16
  import { totalmem } from 'os';
15
17
  import { getHeapStatistics } from 'v8';
@@ -1644,7 +1646,7 @@ function countSurface(counts, surface, token) {
1644
1646
  }
1645
1647
  function countOccurrence(counts, spellings, surface, token) {
1646
1648
  countSurface(counts, surface, token);
1647
- if (spellings !== null) countSpelling(spellings, token, surface ?? token, 1);
1649
+ if (spellings !== null) countPartialSpelling(spellings, token, surface ?? token, 1);
1648
1650
  }
1649
1651
  function addFieldOccurrences(occurrencesByToken, fieldTokens) {
1650
1652
  const fieldCounts = /* @__PURE__ */ new Map();
@@ -1966,36 +1968,6 @@ function updateFieldIndexOnly(state, docId, oldFields, newDoc, fields) {
1966
1968
  }
1967
1969
  }
1968
1970
 
1969
- // src/filters/pattern-tests.ts
1970
- function excluding(context, fieldPath, excluded) {
1971
- const universe = bitsetAnd(context.presentBitset(fieldPath), context.allDocIdsBitset);
1972
- return bitsetAnd(universe, bitsetNot(excluded, context.capacity));
1973
- }
1974
- function textsOf(operand) {
1975
- const texts = /* @__PURE__ */ new Set();
1976
- if (!Array.isArray(operand)) return texts;
1977
- for (const text of operand) if (typeof text === "string") texts.add(text);
1978
- return texts;
1979
- }
1980
- function evaluatePatternTests(fieldPath, filter, context, bitsets) {
1981
- const caseFold = filter.caseFold === true;
1982
- const matching = (operator, texts, tolerance = 0) => {
1983
- const into = createBitSet(context.capacity);
1984
- for (const text of texts) context.matchPattern(fieldPath, operator, text, caseFold, tolerance, into);
1985
- return into;
1986
- };
1987
- const eqTolerance = isPatternTolerance(filter.tolerance) ? filter.tolerance : 0;
1988
- if (typeof filter.eq === "string") bitsets.push(matching("eq", [filter.eq], eqTolerance));
1989
- if (typeof filter.ne === "string") bitsets.push(excluding(context, fieldPath, matching("eq", [filter.ne])));
1990
- if (filter.in !== void 0) bitsets.push(matching("eq", textsOf(filter.in)));
1991
- if (filter.nin !== void 0) bitsets.push(excluding(context, fieldPath, matching("eq", textsOf(filter.nin))));
1992
- if (typeof filter.startsWith === "string") bitsets.push(matching("startsWith", [filter.startsWith]));
1993
- if (typeof filter.endsWith === "string") bitsets.push(matching("endsWith", [filter.endsWith]));
1994
- if (typeof filter.contains === "string") bitsets.push(matching("contains", [filter.contains]));
1995
- if (typeof filter.wildcard === "string") bitsets.push(matching("wildcard", [filter.wildcard]));
1996
- if (typeof filter.regex === "string") bitsets.push(matching("regex", [filter.regex]));
1997
- }
1998
-
1999
1971
  // src/filters/field-tests.ts
2000
1972
  var COMPARISON_OPERATORS = ["eq", "ne", "gt", "gte", "lt", "lte", "between"];
2001
1973
  var TEXT_RANGE_OPERATORS = ["gt", "gte", "lt", "lte"];
@@ -2175,7 +2147,7 @@ function narrowByFieldTests(fieldPath, f, parsed, context, narrow) {
2175
2147
  narrowByTextRange(fieldPath, f, context, narrow);
2176
2148
  if (parsed.pattern) {
2177
2149
  const patternBits = [];
2178
- evaluatePatternTests(fieldPath, f, context, patternBits);
2150
+ requirePatternSearch(fieldPath).evaluateTests(fieldPath, f, context, patternBits);
2179
2151
  for (const matched of patternBits) narrow(matched);
2180
2152
  } else {
2181
2153
  narrowByValue(fieldPath, f, parsed, context, bits, narrow);
@@ -3422,68 +3394,6 @@ function mergeSegmentState(target, source) {
3422
3394
  target.scoreBuffer = null;
3423
3395
  }
3424
3396
 
3425
- // src/core/partition/partial-words/platform.ts
3426
- function platformPartialWordSearch() {
3427
- return partialWordSearchModule;
3428
- }
3429
-
3430
- // src/core/partition/partial-words/registry.ts
3431
- var registered = null;
3432
- function isPartialWordSearchModule(value) {
3433
- return value !== null && typeof value === "object" && value.name === "partialWords" && "createGrowingSlot" in value && typeof value.createGrowingSlot === "function" && "createFrozenSlot" in value && typeof value.createFrozenSlot === "function" && "buildTable" in value && typeof value.buildTable === "function" && "expand" in value && typeof value.expand === "function";
3434
- }
3435
- function registerPartialWordSearch(module) {
3436
- if (!isPartialWordSearchModule(module)) {
3437
- throw new NarsilError(
3438
- ErrorCodes.CONFIG_INVALID,
3439
- 'Pass registerPartialWordSearch the partialWordSearch object that "@delali/narsil/partial-words" exports'
3440
- );
3441
- }
3442
- registered = module;
3443
- }
3444
- function availablePartialWordSearch() {
3445
- return registered ?? platformPartialWordSearch();
3446
- }
3447
- function requirePartialWordSearchFor(schema) {
3448
- const fields = partialFieldsOf(schema);
3449
- if (fields.length === 0 || availablePartialWordSearch() !== null) return;
3450
- throw new NarsilError(
3451
- ErrorCodes.CONFIG_INVALID,
3452
- `The type of field "${fields[0]}" includes "partial", but this bundle contains no partial-word search code. Add import { partialWordSearch } from '@delali/narsil/partial-words', then call registerPartialWordSearch(partialWordSearch) before you create or load the index`,
3453
- { field: fields[0] }
3454
- );
3455
- }
3456
- var EMPTY_GROWING_SLOT = {
3457
- readerFor: () => null,
3458
- spellings: () => null,
3459
- bytes: () => 0,
3460
- afterDocument: () => void 0,
3461
- afterMerge: () => void 0,
3462
- forgetSpellings: () => void 0,
3463
- clear: () => void 0
3464
- };
3465
- var EMPTY_FROZEN_SLOT = {
3466
- readerFor: () => null,
3467
- spellings: () => null,
3468
- bytes: () => 0,
3469
- afterRemoval: () => void 0,
3470
- tableOfEveryDocument: () => null,
3471
- countKeptSpellings: () => void 0
3472
- };
3473
- function createGrowingPartialWords(view) {
3474
- return availablePartialWordSearch()?.createGrowingSlot(view) ?? EMPTY_GROWING_SLOT;
3475
- }
3476
- function createFrozenPartialWords(initial, tokenTable, documents) {
3477
- return availablePartialWordSearch()?.createFrozenSlot(initial, tokenTable, documents) ?? EMPTY_FROZEN_SLOT;
3478
- }
3479
- function buildPartialWordTableFrom(source, tokenTable) {
3480
- return availablePartialWordSearch()?.buildTable(source, tokenTable) ?? null;
3481
- }
3482
- function expandPartialWords(subs, analysis, requests) {
3483
- const module = availablePartialWordSearch();
3484
- return module === null ? requests.map(() => []) : module.expand(subs, analysis, requests);
3485
- }
3486
-
3487
3397
  // src/core/partition/rebuild.ts
3488
3398
  var EMPTY_STATS = {
3489
3399
  totalDocuments: 0,
@@ -4585,99 +4495,18 @@ function mergeSegmentPayload(target, payload, documents) {
4585
4495
  }
4586
4496
 
4587
4497
  // src/core/partition/pattern-snapshot.ts
4588
- function loadFailed(fieldPath, problem) {
4589
- return new NarsilError(
4590
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
4591
- `The pattern index of field "${fieldPath}" in this partition is corrupt: ${problem}`,
4592
- { field: fieldPath }
4593
- );
4594
- }
4595
- function serializePatternIndex(state, fieldPath, index) {
4596
- const resolver = state.docStore.resolver();
4597
- const arrays = index.toArrays(fieldPath, null);
4598
- const positionOf = /* @__PURE__ */ new Map();
4599
- const docIds = [];
4600
- for (const ordinal of arrays.docIds) {
4601
- const externalId = resolver.toExternal(ordinal);
4602
- if (externalId === void 0) continue;
4603
- positionOf.set(ordinal, docIds.length);
4604
- docIds.push(externalId);
4605
- }
4606
- const runs = /* @__PURE__ */ Object.create(null);
4607
- for (let run = 0; run + 1 < arrays.offsets.length; run++) {
4608
- const positions = [];
4609
- for (let at = arrays.offsets[run]; at < arrays.offsets[run + 1]; at++) {
4610
- const position = positionOf.get(arrays.postings[at]);
4611
- if (position !== void 0) positions.push(position);
4612
- }
4613
- if (positions.length === 0) continue;
4614
- const base = run * PATTERN_RUN_CODE_POINTS;
4615
- runs[String.fromCodePoint(arrays.runs[base], arrays.runs[base + 1], arrays.runs[base + 2])] = positions;
4616
- }
4617
- return { docIds, runs };
4618
- }
4619
4498
  function serializePatternIndexes(state, schema) {
4620
4499
  const patternFields = schemaFieldsOf(schema).filter((field) => field.pattern);
4621
4500
  if (patternFields.length === 0) return void 0;
4501
+ const resolver = state.docStore.resolver();
4502
+ const toExternal = (ordinal) => resolver.toExternal(ordinal);
4622
4503
  const serialized = /* @__PURE__ */ Object.create(null);
4623
4504
  for (const { path: fieldPath } of patternFields) {
4624
4505
  const index = state.patternIndexes.get(fieldPath);
4625
- serialized[fieldPath] = index === void 0 ? { docIds: [], runs: {} } : serializePatternIndex(state, fieldPath, index);
4506
+ serialized[fieldPath] = index === void 0 ? { docIds: [], runs: {} } : requirePatternSearch(fieldPath).serializeIndex(fieldPath, index, toExternal);
4626
4507
  }
4627
4508
  return serialized;
4628
4509
  }
4629
- function appendRunCodePoints(fieldPath, key, out) {
4630
- const start = out.length;
4631
- for (const character of key) out.push(character.codePointAt(0) ?? 0);
4632
- const count = out.length - start;
4633
- if (count !== PATTERN_RUN_CODE_POINTS) {
4634
- throw loadFailed(
4635
- fieldPath,
4636
- `the key "${key}" holds ${count} code points, while each key must hold ${PATTERN_RUN_CODE_POINTS}`
4637
- );
4638
- }
4639
- }
4640
- function requirePositions(fieldPath, key, list, documentCount) {
4641
- if (!Array.isArray(list)) throw loadFailed(fieldPath, `the run "${key}" must map to a list of positions`);
4642
- let previous = -1;
4643
- for (const position of list) {
4644
- if (!Number.isInteger(position) || position < 0 || position >= documentCount) {
4645
- throw loadFailed(fieldPath, `the run "${key}" names position ${String(position)} outside its document list`);
4646
- }
4647
- if (position <= previous) {
4648
- throw loadFailed(fieldPath, `the positions of the run "${key}" are out of ascending order`);
4649
- }
4650
- previous = position;
4651
- }
4652
- return list;
4653
- }
4654
- function requireDistinctDocuments(fieldPath, docIds) {
4655
- const seen = /* @__PURE__ */ new Set();
4656
- for (const docId of docIds) {
4657
- if (seen.has(docId)) throw loadFailed(fieldPath, `the document list names "${docId}" twice`);
4658
- seen.add(docId);
4659
- }
4660
- }
4661
- function decodePatternEntry(fieldPath, entry) {
4662
- requireDistinctDocuments(fieldPath, entry.docIds);
4663
- const runs = [];
4664
- const offsets = [0];
4665
- const postings = [];
4666
- for (const [key, list] of Object.entries(entry.runs)) {
4667
- appendRunCodePoints(fieldPath, key, runs);
4668
- for (const position of requirePositions(fieldPath, key, list, entry.docIds.length)) postings.push(position);
4669
- offsets.push(postings.length);
4670
- }
4671
- const positions = new Uint32Array(entry.docIds.length);
4672
- for (let position = 0; position < positions.length; position++) positions[position] = position;
4673
- return {
4674
- fieldPath,
4675
- docIds: positions,
4676
- runs: Uint32Array.from(runs),
4677
- offsets: Uint32Array.from(offsets),
4678
- postings: Uint32Array.from(postings)
4679
- };
4680
- }
4681
4510
  function rebuildPatternIndex(state, fieldPath) {
4682
4511
  for (const [docId, stored] of state.docStore.all()) {
4683
4512
  const value = getNestedValue(stored.fields, fieldPath);
@@ -4695,7 +4524,7 @@ function loadPatternIndexes(state, serialized, schema) {
4695
4524
  rebuildPatternIndex(state, fieldPath);
4696
4525
  continue;
4697
4526
  }
4698
- const decoded = decodePatternEntry(fieldPath, entry);
4527
+ const decoded = requirePatternSearch(fieldPath).decodeIndex(fieldPath, entry);
4699
4528
  const ordinals = entry.docIds.map((docId) => resolver.toInternal(docId) ?? -1);
4700
4529
  patternIndexOf(state.patternIndexes, fieldPath).addArrays(decoded, (position) => ordinals[position]);
4701
4530
  }
@@ -9352,7 +9181,7 @@ function sharePartialWordTable(table) {
9352
9181
  function sharedPartialWordsOf(tokenData, spellings, documentData, analysis) {
9353
9182
  let source = spellings;
9354
9183
  if (source === null && analysis !== null && analysis.fields.length > 0) {
9355
- source = encodePartialSpellings(analysis.fields, spellingCountsOf(storedDocumentsOf(documentData), analysis));
9184
+ source = partialSpellingsOfDocuments(storedDocumentsOf(documentData), analysis);
9356
9185
  }
9357
9186
  if (source === null || source.fields.length === 0) return null;
9358
9187
  const table = buildPartialWordTableFrom(source, wrapFrozenTokenTable(tokenData));
@@ -10219,7 +10048,7 @@ function mergePartialWords(inputs, mergedTokenCount, analysis) {
10219
10048
  }
10220
10049
  const counts = /* @__PURE__ */ new Map();
10221
10050
  for (const input of inputs) input.segment.partialWords.countKeptSpellings(counts, analysis, input.remap);
10222
- return { spellings: encodePartialSpellings(analysis.fields, counts), reused: null };
10051
+ return { spellings: encodePartialSpellingsOf(analysis.fields, counts), reused: null };
10223
10052
  }
10224
10053
  function mergeFrozenSegments(segments, analysis) {
10225
10054
  if (typeof SharedArrayBuffer !== "function") return null;
@@ -10535,7 +10364,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
10535
10364
  const addToken = (token, position, surface) => {
10536
10365
  postings.addToken(token, positionOffset + position);
10537
10366
  surfaces.count(surface, token);
10538
- if (fieldSpellings !== null) countSpelling(fieldSpellings, token, surface ?? token, 1);
10367
+ if (fieldSpellings !== null) countPartialSpelling(fieldSpellings, token, surface ?? token, 1);
10539
10368
  };
10540
10369
  for (let ordinal = 0; ordinal < documents.length; ordinal++) {
10541
10370
  const document = documents[ordinal].document;
@@ -10569,7 +10398,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
10569
10398
  fieldLengthColumns: lengths.columns,
10570
10399
  totalFieldLengths: lengths.totalsByName(),
10571
10400
  surfaceForms: surfaces.encode(),
10572
- partialSpellings: partialFields.length === 0 ? null : encodePartialSpellings(partialFields, spellings),
10401
+ partialSpellings: partialFields.length === 0 ? null : encodePartialSpellingsOf(partialFields, spellings),
10573
10402
  ...encodeFieldIndexes(filters, identity)
10574
10403
  };
10575
10404
  }
@@ -11113,4 +10942,4 @@ function createDirectExecutor(options) {
11113
10942
  };
11114
10943
  }
11115
10944
 
11116
- export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES, NO_LIVE_VECTOR_AT_POSITION, SHUTDOWN_TIMEOUT_MS, VALID_HNSW_METRICS, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, assembleGraphParts, buildSegmentPayload, bytesToVectors, countDead, createDirectExecutor, createFrozenSegment, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, deadBitsWhere, deadBytesFor, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, encodeSegmentState, freezeSegmentShared, generateId, getStopWords, getTokenizer, graphPartsOfEarlierPayload, hasStopWords, hasTokenizer, isCompositePartition, isDead, mergeFrozenSegments, mergeSegmentContents, missingWorkerEntry, nodeThreadId, partitionStateOf, readHeapStatistics, registerPartialWordSearch, registerStopWords, registerTokenizer, requirePartialWordSearchFor, resolveIndexAnalysis, resolvePartitionInsertOptions, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, segmentArraysOf, selectionKeeps, shareSegmentArrays, spawnNodeWorker, splitWorkerBudget, survivorDocumentsOf, vectorsPerFile, vectorsToBytes, workerResourceLimits };
10945
+ export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES, NO_LIVE_VECTOR_AT_POSITION, SHUTDOWN_TIMEOUT_MS, VALID_HNSW_METRICS, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, assembleGraphParts, buildSegmentPayload, bytesToVectors, countDead, createDirectExecutor, createFrozenSegment, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, deadBitsWhere, deadBytesFor, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, encodeSegmentState, freezeSegmentShared, generateId, getStopWords, getTokenizer, graphPartsOfEarlierPayload, hasStopWords, hasTokenizer, isCompositePartition, isDead, mergeFrozenSegments, mergeSegmentContents, missingWorkerEntry, nodeThreadId, partitionStateOf, readHeapStatistics, registerStopWords, registerTokenizer, resolveIndexAnalysis, resolvePartitionInsertOptions, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, segmentArraysOf, selectionKeeps, shareSegmentArrays, spawnNodeWorker, splitWorkerBudget, survivorDocumentsOf, vectorsPerFile, vectorsToBytes, workerResourceLimits };
@@ -1,7 +1,7 @@
1
- import { readSharedConfigFields, restoredEmbedding, restoredPartitionConfig, MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-XSZTF4ZQ.mjs';
1
+ import { readSharedConfigFields, restoredEmbedding, restoredPartitionConfig, MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-HJASKX4F.mjs';
2
2
  import { getLanguage } from './chunk-4MQ5F4IL.mjs';
3
- import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-IOZPBNSG.mjs';
4
- import { compareCodePoints } from './chunk-IBR6634K.mjs';
3
+ import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-GG2FBGVZ.mjs';
4
+ import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
5
5
  import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
6
6
  import { encode, decode } from '@msgpack/msgpack';
7
7