@delali/narsil 0.3.1-next.72 → 0.3.1-next.74

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/README.md +2 -2
  2. package/dist/adapters/filesystem.mjs +1 -1
  3. package/dist/adapters/indexeddb.mjs +1 -1
  4. package/dist/{capabilities-D6gjjQyA.d.ts → capabilities-CAMzmkAg.d.ts} +19 -4
  5. package/dist/{chunk-P2ZZN32C.mjs → chunk-2E35UM3G.mjs} +2 -12
  6. package/dist/chunk-3DDUNI6N.mjs +48 -0
  7. package/dist/{chunk-KZTI25D3.mjs → chunk-4MQ5F4IL.mjs} +1 -1
  8. package/dist/{chunk-HVJYTKK7.mjs → chunk-BVBIQQKG.mjs} +947 -1327
  9. package/dist/{chunk-2X2ZQOAB.mjs → chunk-CTOSGRM7.mjs} +101 -21
  10. package/dist/{chunk-7QMTA7AA.mjs → chunk-CUZOX6CN.mjs} +7784 -4276
  11. package/dist/chunk-I23G3IZC.mjs +593 -0
  12. package/dist/{chunk-RV77FR2F.mjs → chunk-I64LXKO3.mjs} +0 -1
  13. package/dist/chunk-KEHK2O6P.mjs +13 -0
  14. package/dist/{chunk-CVPA5Q7R.mjs → chunk-LX4JERXI.mjs} +9 -6
  15. package/dist/{chunk-EDMCZLSJ.mjs → chunk-NOZUQM7J.mjs} +193 -13
  16. package/dist/{chunk-LCCL6EEH.mjs → chunk-PO53FV4Q.mjs} +93 -94
  17. package/dist/chunk-T5ZWEWVD.mjs +658 -0
  18. package/dist/{chunk-Y52WZWAU.mjs → chunk-TVFLHM7M.mjs} +4 -4
  19. package/dist/{chunk-QR4OKRVY.mjs → chunk-VKX2BPXD.mjs} +147 -642
  20. package/dist/{chunk-3HQPUCCT.mjs → chunk-XW25X6AU.mjs} +66 -237
  21. package/dist/{chunk-M6XF5HDE.mjs → chunk-Z3DQPOJQ.mjs} +127 -30
  22. package/dist/{client-C2TiW29y.d.ts → client-C-I70ArH.d.ts} +77 -26
  23. package/dist/client.d.ts +9 -9
  24. package/dist/client.mjs +44 -22
  25. package/dist/{config-BLguB-om.d.ts → config-C6Iw5rCX.d.ts} +11 -4
  26. package/dist/{config-C7Ip8Ife.d.ts → config-DJPH3IQB.d.ts} +11 -2
  27. package/dist/direct-executor-KFAGILJC.mjs +17 -0
  28. package/dist/distribution/coordinator/etcd.d.ts +29 -16
  29. package/dist/distribution/coordinator/etcd.mjs +162 -75
  30. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  31. package/dist/distribution/transport/grpc.d.ts +2 -2
  32. package/dist/distribution/transport/grpc.mjs +22 -2
  33. package/dist/distribution/transport/in-memory.d.ts +2 -2
  34. package/dist/distribution/transport/in-memory.mjs +6 -3
  35. package/dist/distribution/transport/tcp.d.ts +2 -2
  36. package/dist/distribution/transport/tcp.mjs +40 -16
  37. package/dist/distribution.d.ts +9 -9
  38. package/dist/distribution.mjs +647 -366
  39. package/dist/embeddings/openai.mjs +1 -1
  40. package/dist/{engine-Dm4446j1.d.ts → engine-B8eRPO7m.d.ts} +97 -188
  41. package/dist/index.browser.mjs +12132 -8592
  42. package/dist/index.d.ts +27 -8
  43. package/dist/index.mjs +15 -14
  44. package/dist/{memory-CKHTU1CL.d.ts → memory-DJEwWoCi.d.ts} +1204 -968
  45. package/dist/partial-words-DTIHO8G4.d.ts +19 -0
  46. package/dist/partial-words.d.ts +22 -0
  47. package/dist/partial-words.mjs +11 -0
  48. package/dist/pattern.mjs +1 -1
  49. package/dist/persistence/durability/checkpoint-worker.d.ts +57 -29
  50. package/dist/persistence/durability/checkpoint-worker.mjs +26 -16
  51. package/dist/react.d.ts +7 -7
  52. package/dist/react.mjs +0 -1
  53. package/dist/{schema-BBOCdhND.d.ts → schema-DgyDijhf.d.ts} +6 -2
  54. package/dist/server.d.ts +11 -11
  55. package/dist/server.mjs +212 -32
  56. package/dist/{tasks-BECd0b__.d.ts → tasks-BVmAKYBh.d.ts} +3 -3
  57. package/dist/{thread-IBSIZZ6W.mjs → thread-W6DPKEF6.mjs} +82 -21
  58. package/dist/{types-BQDlIvfS.d.ts → types-B3b68nfF.d.ts} +1 -1
  59. package/dist/{types-D5OwhtsQ.d.ts → types-Cb9kiY5P.d.ts} +1 -1
  60. package/dist/{types-BiUmaiWT.d.ts → types-ChrVy4lH.d.ts} +1 -1
  61. package/dist/vector/search-worker.d.ts +1 -1
  62. package/dist/vector/search-worker.mjs +4 -3
  63. package/dist/workers/entry.mjs +4 -4
  64. package/package.json +15 -7
  65. package/dist/chunk-3L4U6EJQ.mjs +0 -49
  66. package/dist/chunk-IFBKLBGB.mjs +0 -7
  67. package/dist/chunk-XZ4PAREY.mjs +0 -25
  68. package/dist/direct-executor-SVCHXR74.mjs +0 -14
  69. package/dist/serialization/crc32-worker.d.ts +0 -20
  70. package/dist/serialization/crc32-worker.mjs +0 -62
@@ -1,55 +1,6 @@
1
- import { MATCHER_STATE_BUDGET_BYTES, bitsetHas, bitsetSet, PATTERN_RUN_CODE_POINTS, MATCHER_SIMULATION_BYTES_PER_POSITION, MATCHER_STATE_OVERHEAD_BYTES, MATCHER_CODE_POINT_BYTES } from './chunk-ZKRV5M2P.mjs';
2
-
3
- // src/core/pattern-index/matcher-budget.ts
4
- var POSITIONS_PER_WORD = 32;
5
- function positionWords(positions) {
6
- return Math.max(1, Math.ceil(positions / POSITIONS_PER_WORD));
7
- }
8
- function automatonBytes(positions) {
9
- return 2 * positionWords(positions) * Uint32Array.BYTES_PER_ELEMENT + positions * MATCHER_CODE_POINT_BYTES;
10
- }
11
- function smallestMatcherBytes(positions) {
12
- return automatonBytes(positions) + positions * MATCHER_SIMULATION_BYTES_PER_POSITION + MATCHER_STATE_OVERHEAD_BYTES + positions * Uint32Array.BYTES_PER_ELEMENT;
13
- }
14
- function widestPatternSplit(positions) {
15
- return Math.max(1, Math.floor(MATCHER_STATE_BUDGET_BYTES / smallestMatcherBytes(positions)));
16
- }
17
-
18
- // src/pattern/automaton.ts
19
- function setPosition(words, position) {
20
- words[position >>> 5] |= 1 << (position & 31);
21
- }
22
- function hasPosition(words, position) {
23
- return (words[position >>> 5] & 1 << (position & 31)) !== 0;
24
- }
25
- function literalAutomaton(codePoints, anchoredStart, anchoredEnd) {
26
- const size = codePoints.length;
27
- const words = positionWords(size);
28
- const first = new Uint32Array(words);
29
- const last = new Uint32Array(words);
30
- if (size > 0) {
31
- setPosition(first, 0);
32
- setPosition(last, size - 1);
33
- }
34
- return {
35
- size,
36
- words,
37
- first,
38
- last,
39
- nullable: size === 0,
40
- anchoredStart,
41
- anchoredEnd,
42
- bytes: automatonBytes(size),
43
- accepts: (position, codePoint) => codePoints[position] === codePoint,
44
- followerCount: (position) => position + 1 < size ? 1 : 0,
45
- follower: (position) => position + 1
46
- };
47
- }
1
+ import { bitsetHas, bitsetSet, PATTERN_RUN_CODE_POINTS } from './chunk-ZKRV5M2P.mjs';
48
2
 
49
3
  // src/pattern/constants.ts
50
- var MATCHER_TRANSITION_BYTES = 48;
51
- var MATCHER_KEY_CHARACTER_BYTES = 2;
52
- var MATCHER_MIN_CODE_POINTS_PER_STATE = 10;
53
4
  var SHORTLIST_RUN_LOOKUPS = 16;
54
5
  var SHORTLIST_INTERSECT_RATIO = 8;
55
6
  var RUN_LIST_INITIAL_CAPACITY = 4;
@@ -57,205 +8,86 @@ var RUN_LIST_COMPACTION_FRACTION = 0.5;
57
8
  var RUN_ENTRY_OVERHEAD_BYTES = 140;
58
9
  var DOCUMENT_ENTRY_BYTES = 24;
59
10
 
60
- // src/pattern/position-simulation.ts
61
- var LAST_STAMP = 4294967295;
62
- var PositionSimulation = class {
63
- constructor(automaton) {
64
- this.automaton = automaton;
65
- this.active = new Uint32Array(automaton.size);
66
- this.following = new Uint32Array(automaton.size);
67
- this.marks = new Uint32Array(automaton.size);
68
- const first = [];
69
- for (let position = 0; position < automaton.size; position++) {
70
- if (hasPosition(automaton.first, position)) first.push(position);
71
- }
72
- this.first = Uint32Array.from(first);
73
- }
11
+ // src/pattern/literal-matcher.ts
12
+ function bordersOf(codePoints) {
13
+ const length = codePoints.length;
14
+ const borders = new Uint32Array(length + 1);
15
+ let border = 0;
16
+ for (let at = 1; at < length; at++) {
17
+ while (border > 0 && codePoints[at] !== codePoints[border]) border = borders[border];
18
+ if (codePoints[at] === codePoints[border]) border++;
19
+ borders[at + 1] = border;
20
+ }
21
+ return borders;
22
+ }
23
+ function reachablePositionsOf(borders, length) {
24
+ const reachable = new Uint32Array(length + 1);
25
+ for (let prefix = 1; prefix <= length; prefix++) {
26
+ reachable[prefix] = (prefix < length ? 1 : 0) + reachable[borders[prefix]];
27
+ }
28
+ return reachable;
29
+ }
30
+ var LiteralMatcher = class {
31
+ constructor(codePoints, anchoredStart, anchoredEnd) {
32
+ this.codePoints = codePoints;
33
+ this.anchoredStart = anchoredStart;
34
+ this.anchoredEnd = anchoredEnd;
35
+ this.borders = bordersOf(codePoints);
36
+ this.reachable = reachablePositionsOf(this.borders, codePoints.length);
37
+ }
38
+ borders;
39
+ reachable;
40
+ prefix = 0;
41
+ alive = true;
74
42
  accepting = false;
75
- active;
76
- following;
77
- marks;
78
- first;
79
- activeCount = 0;
80
- stamp = 0;
81
- get steps() {
82
- return this.activeCount;
83
- }
84
- get bytes() {
85
- return this.automaton.size * MATCHER_SIMULATION_BYTES_PER_POSITION;
86
- }
87
- reset() {
88
- this.active.set(this.first);
89
- this.activeCount = this.first.length;
90
- this.accepting = this.automaton.nullable;
91
- }
92
- get firstPositions() {
93
- return this.first;
94
- }
95
- load(active, accepting) {
96
- this.active.set(active);
97
- this.activeCount = active.length;
98
- this.accepting = accepting;
99
- }
100
- sortedActive() {
101
- return this.active.slice(0, this.activeCount).sort();
102
- }
103
- advance(codePoint) {
104
- const { automaton, marks } = this;
105
- const following = this.following;
106
- const stamp = this.nextStamp();
107
- let count = 0;
108
- let accepting = false;
109
- for (let at = 0; at < this.activeCount; at++) {
110
- const position = this.active[at];
111
- if (!automaton.accepts(position, codePoint)) continue;
112
- if (hasPosition(automaton.last, position)) accepting = true;
113
- const followers = automaton.followerCount(position);
114
- for (let index = 0; index < followers; index++) {
115
- const follower = automaton.follower(position, index);
116
- if (marks[follower] === stamp) continue;
117
- marks[follower] = stamp;
118
- following[count++] = follower;
119
- }
120
- }
121
- if (!automaton.anchoredStart) {
122
- for (const position of this.first) {
123
- if (marks[position] === stamp) continue;
124
- marks[position] = stamp;
125
- following[count++] = position;
126
- }
127
- if (automaton.nullable) accepting = true;
128
- }
129
- this.following = this.active;
130
- this.active = following;
131
- this.activeCount = count;
132
- this.accepting = accepting;
133
- }
134
- nextStamp() {
135
- if (this.stamp === LAST_STAMP) {
136
- this.marks.fill(0);
137
- this.stamp = 0;
138
- }
139
- this.stamp++;
140
- return this.stamp;
141
- }
142
- };
143
-
144
- // src/pattern/lazy-dfa.ts
145
- var UNCACHED_GENERATION = -1;
146
- function stateKey(active, accepting) {
147
- return `${active.join(",")}${accepting ? "+" : "-"}`;
148
- }
149
- function stateBytes(active, key) {
150
- return MATCHER_STATE_OVERHEAD_BYTES + active.byteLength + key.length * MATCHER_KEY_CHARACTER_BYTES;
151
- }
152
- var LazyDfa = class {
153
- constructor(automaton, stateLimitBytes) {
154
- this.automaton = automaton;
155
- this.simulation = new PositionSimulation(automaton);
156
- this.cacheLimitBytes = stateLimitBytes - automaton.bytes - this.simulation.bytes;
157
- this.start = this.intern(this.simulation.firstPositions, automaton.nullable);
158
- this.state = this.start;
159
- if (this.wantsSimulation()) this.simulate(this.start);
160
- }
161
- states = /* @__PURE__ */ new Map();
162
- generation = 0;
163
- cachedBytes = 0;
164
- codePointsSinceReset = 0;
165
- start;
166
- state;
167
- simulating = false;
168
43
  decided = null;
169
- simulation;
170
- cacheLimitBytes;
171
44
  begin() {
172
45
  this.decided = null;
173
- if (this.simulating) this.simulation.reset();
174
- else this.state = this.start;
175
- if (!this.automaton.anchoredEnd && this.accepting()) return this.decide(true);
46
+ this.prefix = 0;
47
+ this.alive = this.codePoints.length > 0;
48
+ this.accepting = this.codePoints.length === 0;
49
+ if (!this.anchoredEnd && this.accepting) return this.decide(true);
176
50
  return true;
177
51
  }
178
52
  feed(codePoint, tally) {
179
- const steps = this.simulating ? this.simulation.steps : this.state.active.length;
180
- if (steps === 0) return this.decide(!this.automaton.anchoredStart && this.automaton.nullable);
53
+ const steps = this.steps();
54
+ if (steps === 0) return this.decide(!this.anchoredStart && this.codePoints.length === 0);
181
55
  tally.add(steps);
182
- if (this.simulating) {
183
- this.simulation.advance(codePoint);
184
- } else {
185
- this.codePointsSinceReset++;
186
- this.state = this.step(this.state, codePoint);
187
- if (this.wantsSimulation()) this.simulate(this.state);
188
- }
189
- if (!this.automaton.anchoredEnd && this.accepting()) return this.decide(true);
56
+ if (this.anchoredStart) this.advanceAnchored(codePoint);
57
+ else this.advance(codePoint);
58
+ if (!this.anchoredEnd && this.accepting) return this.decide(true);
190
59
  return true;
191
60
  }
192
61
  matched() {
193
- return this.decided ?? (this.automaton.anchoredEnd && this.accepting());
62
+ return this.decided ?? (this.anchoredEnd && this.accepting);
63
+ }
64
+ steps() {
65
+ const length = this.codePoints.length;
66
+ if (length === 0) return 0;
67
+ if (this.anchoredStart) return this.alive && this.prefix < length ? 1 : 0;
68
+ return 1 + this.reachable[this.prefix];
69
+ }
70
+ advanceAnchored(codePoint) {
71
+ if (this.codePoints[this.prefix] === codePoint) {
72
+ this.prefix++;
73
+ this.accepting = this.prefix === this.codePoints.length;
74
+ return;
75
+ }
76
+ this.alive = false;
77
+ this.accepting = false;
194
78
  }
195
- accepting() {
196
- return this.simulating ? this.simulation.accepting : this.state.accepting;
79
+ advance(codePoint) {
80
+ const { codePoints, borders } = this;
81
+ let prefix = this.prefix === codePoints.length ? borders[this.prefix] : this.prefix;
82
+ while (prefix > 0 && codePoints[prefix] !== codePoint) prefix = borders[prefix];
83
+ if (codePoints[prefix] === codePoint) prefix++;
84
+ this.prefix = prefix;
85
+ this.accepting = prefix === codePoints.length;
197
86
  }
198
87
  decide(outcome) {
199
88
  this.decided = outcome;
200
89
  return false;
201
90
  }
202
- wantsSimulation() {
203
- return !this.simulating && this.state.generation === UNCACHED_GENERATION;
204
- }
205
- simulate(from) {
206
- this.dropCache();
207
- this.state = this.start;
208
- this.simulation.load(from.active, from.accepting);
209
- this.simulating = true;
210
- }
211
- dropCache() {
212
- for (const state of this.states.values()) state.next.clear();
213
- this.start.next.clear();
214
- this.state.next.clear();
215
- this.states = /* @__PURE__ */ new Map();
216
- this.cachedBytes = 0;
217
- }
218
- step(state, codePoint) {
219
- const current = state.generation === this.generation;
220
- if (current) {
221
- const cached = state.next.get(codePoint);
222
- if (cached !== void 0) return cached;
223
- }
224
- const { simulation } = this;
225
- simulation.load(state.active, state.accepting);
226
- simulation.advance(codePoint);
227
- const next = this.intern(simulation.sortedActive(), simulation.accepting);
228
- if (!current || state.generation !== this.generation || next.generation !== this.generation) return next;
229
- if (!this.reserve(MATCHER_TRANSITION_BYTES) || state.generation !== this.generation) return next;
230
- state.next.set(codePoint, next);
231
- return next;
232
- }
233
- intern(active, accepting) {
234
- const key = stateKey(active, accepting);
235
- const existing = this.states.get(key);
236
- if (existing !== void 0) return existing;
237
- if (!this.reserve(stateBytes(active, key))) {
238
- return { active, accepting, generation: UNCACHED_GENERATION, next: /* @__PURE__ */ new Map() };
239
- }
240
- const state = { active, accepting, generation: this.generation, next: /* @__PURE__ */ new Map() };
241
- this.states.set(key, state);
242
- return state;
243
- }
244
- reserve(bytes) {
245
- if (this.cachedBytes + bytes <= this.cacheLimitBytes) {
246
- this.cachedBytes += bytes;
247
- return true;
248
- }
249
- if (this.states.size === 0 || this.codePointsSinceReset < MATCHER_MIN_CODE_POINTS_PER_STATE * this.states.size) {
250
- return false;
251
- }
252
- this.dropCache();
253
- this.codePointsSinceReset = 0;
254
- this.generation++;
255
- this.start = this.intern(this.start.active, this.start.accepting);
256
- if (this.start.generation === UNCACHED_GENERATION) return false;
257
- return this.reserve(bytes);
258
- }
259
91
  };
260
92
 
261
93
  // src/pattern/text.ts
@@ -391,10 +223,7 @@ function matcherFor(request) {
391
223
  const { operator, text, caseFold, host } = request;
392
224
  const testCodePoints = [];
393
225
  appendTextCodePoints(text, caseFold, host, testCodePoints);
394
- return new LazyDfa(
395
- literalAutomaton(testCodePoints, ANCHORED_START.has(operator), ANCHORED_END.has(operator)),
396
- request.meter.matcherStateBytes
397
- );
226
+ return new LiteralMatcher(Uint32Array.from(testCodePoints), ANCHORED_START.has(operator), ANCHORED_END.has(operator));
398
227
  }
399
228
  function matchesText(matcher, text, caseFold, host, folded, tally) {
400
229
  if (!matcher.begin()) return matcher.matched();
@@ -782,4 +611,4 @@ var patternSearchModule = {
782
611
  matchInto: matchPattern
783
612
  };
784
613
 
785
- export { GrowableUint32, patternSearchModule, widestPatternSplit };
614
+ export { GrowableUint32, patternSearchModule };
@@ -1,6 +1,7 @@
1
- import { MAX_M } from './chunk-P2ZZN32C.mjs';
2
- import { holdsUnpairedSurrogate, exceedsCodePoints } from './chunk-LJYTW7GT.mjs';
3
- import { NarsilError, ErrorCodes } from './chunk-RV77FR2F.mjs';
1
+ import { EARLIER_RELEASE_MIN_M, MIN_M, MAX_M } from './chunk-KEHK2O6P.mjs';
2
+ import { compareCodePoints, holdsUnpairedSurrogate, exceedsCodePoints } from './chunk-LJYTW7GT.mjs';
3
+ import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
4
+ import { encode } from '@msgpack/msgpack';
4
5
 
5
6
  // src/schema/constants.ts
6
7
  var MAX_NESTING_DEPTH = 4;
@@ -8,6 +9,10 @@ var MAX_DOCUMENT_NESTING_DEPTH = 32;
8
9
  var DEFAULT_PATTERN_VALUE_LIMIT = 8192;
9
10
  var MIN_PATTERN_VALUE_LIMIT = 1;
10
11
  var MAX_PATTERN_VALUE_LIMIT = 65536;
12
+ var MAX_VECTOR_DIMENSION = 4096;
13
+ var MAX_STORED_VECTOR_DIMENSION = 65535;
14
+ var MAX_TEXT_FIELDS = 65536;
15
+ var MAX_DOCUMENT_BYTES = 16777216;
11
16
 
12
17
  // src/schema/validator/shared.ts
13
18
  var VECTOR_PATTERN = /^vector\[(\d+)]$/;
@@ -57,11 +62,11 @@ function parseBase(baseName, path, type) {
57
62
  invalidType(path, type, `Field "${path}" has an unsupported type: "${type}"`);
58
63
  }
59
64
  const dimension = Number.parseInt(vectorMatch[1], 10);
60
- if (dimension <= 0) {
65
+ if (dimension <= 0 || dimension > MAX_STORED_VECTOR_DIMENSION) {
61
66
  throw new NarsilError(
62
67
  ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
63
- `Vector field "${path}" must have a positive dimension, got ${dimension}`,
64
- { field: path, dimension }
68
+ `Vector field "${path}" has ${dimension} dimensions, but a vector field holds from 1 to ${MAX_STORED_VECTOR_DIMENSION}`,
69
+ { field: path, dimension, maximum: MAX_STORED_VECTOR_DIMENSION }
65
70
  );
66
71
  }
67
72
  return { base: "vector", dimension };
@@ -194,9 +199,30 @@ function validateSchema(schema) {
194
199
  if (Object.keys(schema).length === 0) {
195
200
  throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must define at least one field");
196
201
  }
197
- return validateSchemaFields(schema, 1, "");
202
+ const validated = validateSchemaFields(schema, 1, "");
203
+ requireTextFieldCount(schemaFieldsOf(validated).filter((field) => isWordIndexedBase(field.base)).length);
204
+ return validated;
205
+ }
206
+ function requireTextFieldCount(count) {
207
+ if (count <= MAX_TEXT_FIELDS) return;
208
+ throw new NarsilError(
209
+ ErrorCodes.SCHEMA_INVALID_TYPE,
210
+ `The schema declares ${count} string and string[] fields, but a schema holds at most ${MAX_TEXT_FIELDS}`,
211
+ { textFields: count, maximum: MAX_TEXT_FIELDS }
212
+ );
213
+ }
214
+ function requireCreatableSchema(schema) {
215
+ for (const field of schemaFieldsOf(schema)) {
216
+ if (field.base !== "vector" || field.dimension <= MAX_VECTOR_DIMENSION) continue;
217
+ throw new NarsilError(
218
+ ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
219
+ `Vector field "${field.path}" has ${field.dimension} dimensions, but a new index takes at most ${MAX_VECTOR_DIMENSION}`,
220
+ { field: field.path, dimension: field.dimension, maximum: MAX_VECTOR_DIMENSION }
221
+ );
222
+ }
198
223
  }
199
224
  function requireValidFieldTypes(flatSchema) {
225
+ let textFields = 0;
200
226
  for (const [path, type] of Object.entries(flatSchema)) {
201
227
  if (typeof type !== "string") {
202
228
  throw new NarsilError(
@@ -205,8 +231,9 @@ function requireValidFieldTypes(flatSchema) {
205
231
  { field: path, type: String(type) }
206
232
  );
207
233
  }
208
- parseFieldType(type, path);
234
+ if (isWordIndexedBase(parseFieldType(type, path).base)) textFields += 1;
209
235
  }
236
+ requireTextFieldCount(textFields);
210
237
  }
211
238
  function flattenRecursive(schema, prefix, result) {
212
239
  for (const [field, type] of Object.entries(schema)) {
@@ -239,6 +266,18 @@ function schemaFieldsOf(schema) {
239
266
  parsedSchemas.set(schema, frozen);
240
267
  return frozen;
241
268
  }
269
+ var partialFieldsBySchema = /* @__PURE__ */ new WeakMap();
270
+ function partialFieldsOf(schema) {
271
+ const cached = partialFieldsBySchema.get(schema);
272
+ if (cached !== void 0) return cached;
273
+ const paths = [];
274
+ for (const field of schemaFieldsOf(schema)) {
275
+ if (field.partial) paths.push(field.path);
276
+ }
277
+ const frozen = Object.freeze(paths.sort(compareCodePoints));
278
+ partialFieldsBySchema.set(schema, frozen);
279
+ return frozen;
280
+ }
242
281
  function extractVectorFieldsFromSchema(schema) {
243
282
  const result = /* @__PURE__ */ new Map();
244
283
  for (const field of schemaFieldsOf(schema)) {
@@ -474,7 +513,11 @@ function walkStorable(value, path, depth, ancestors, skip) {
474
513
  reject(path, `holds a ${kind}, which cannot be stored or recovered`);
475
514
  }
476
515
  const container = value;
477
- if (container instanceof Date || container instanceof Uint8Array) return;
516
+ if (container instanceof Date) {
517
+ if (Number.isNaN(container.getTime())) reject(path, "holds an invalid date, which the engine cannot store");
518
+ return;
519
+ }
520
+ if (container instanceof Uint8Array) return;
478
521
  const isArray = Array.isArray(container);
479
522
  if (!isArray && !isPlainObject2(container)) {
480
523
  reject(
@@ -628,6 +671,71 @@ function validateRequiredFields(document, required) {
628
671
  }
629
672
  }
630
673
  }
674
+ function holdsNegativeZero(value) {
675
+ const pending = [value];
676
+ while (pending.length > 0) {
677
+ const next = pending.pop();
678
+ if (typeof next === "number") {
679
+ if (Object.is(next, -0)) return true;
680
+ continue;
681
+ }
682
+ if (next === null || typeof next !== "object" || ArrayBuffer.isView(next)) continue;
683
+ for (const child of Array.isArray(next) ? next : Object.values(next)) pending.push(child);
684
+ }
685
+ return false;
686
+ }
687
+ function encodeDocument(document) {
688
+ return encode(document, holdsNegativeZero(document) ? { forceIntegerToFloat: true } : void 0);
689
+ }
690
+
691
+ // src/schema/validator/document-size.ts
692
+ var LARGEST_HEADER_BYTES = 5;
693
+ var LARGEST_NUMBER_BYTES = 9;
694
+ var LARGEST_TIMESTAMP_BYTES = 15;
695
+ var LARGEST_UTF8_BYTES_PER_UNIT = 3;
696
+ function upperBoundBytes(value, budget, depth) {
697
+ if (value === null || value === void 0 || typeof value === "boolean") return 1;
698
+ if (typeof value === "number") return LARGEST_NUMBER_BYTES;
699
+ if (typeof value === "string") return LARGEST_HEADER_BYTES + value.length * LARGEST_UTF8_BYTES_PER_UNIT;
700
+ if (value instanceof Date) return LARGEST_TIMESTAMP_BYTES;
701
+ if (ArrayBuffer.isView(value)) return LARGEST_HEADER_BYTES + value.byteLength;
702
+ if (typeof value !== "object" || depth > MAX_DOCUMENT_NESTING_DEPTH) return LARGEST_NUMBER_BYTES;
703
+ let total = LARGEST_HEADER_BYTES;
704
+ if (Array.isArray(value)) {
705
+ for (const item of value) {
706
+ total += upperBoundBytes(item, budget - total, depth + 1);
707
+ if (total > budget) return total;
708
+ }
709
+ return total;
710
+ }
711
+ const record = value;
712
+ for (const key of Object.keys(record)) {
713
+ total += LARGEST_HEADER_BYTES + key.length * LARGEST_UTF8_BYTES_PER_UNIT;
714
+ total += upperBoundBytes(record[key], budget - total, depth + 1);
715
+ if (total > budget) return total;
716
+ }
717
+ return total;
718
+ }
719
+ function messagePackUpperBoundBytes(value) {
720
+ return upperBoundBytes(value, Number.POSITIVE_INFINITY, 0);
721
+ }
722
+ function encodedBytesOf(document) {
723
+ try {
724
+ return encodeDocument(document).byteLength;
725
+ } catch {
726
+ return null;
727
+ }
728
+ }
729
+ function requireDocumentWithinSizeLimit(document) {
730
+ if (upperBoundBytes(document, MAX_DOCUMENT_BYTES, 0) <= MAX_DOCUMENT_BYTES) return;
731
+ const encodedBytes = encodedBytesOf(document);
732
+ if (encodedBytes === null || encodedBytes <= MAX_DOCUMENT_BYTES) return;
733
+ throw new NarsilError(
734
+ ErrorCodes.DOC_VALIDATION_FAILED,
735
+ `The document takes ${encodedBytes} bytes as MessagePack, vector values included, but a document may take at most ${MAX_DOCUMENT_BYTES}`,
736
+ { bytes: encodedBytes, maximum: MAX_DOCUMENT_BYTES }
737
+ );
738
+ }
631
739
 
632
740
  // src/schema/validator/vector-promotion.ts
633
741
  var QUANTIZATION_MODES = /* @__PURE__ */ new Set(["osq8", "osq4", "osq2", "osq1", "none"]);
@@ -651,6 +759,13 @@ function validateVectorStorage(config, filesystemDurability) {
651
759
  });
652
760
  }
653
761
  }
762
+ function raiseEarlierReleaseGraphSetting(config) {
763
+ if (typeof config !== "object" || config === null) return config;
764
+ const hnsw = config.hnswConfig;
765
+ if (typeof hnsw !== "object" || hnsw === null) return config;
766
+ if (hnsw.m !== EARLIER_RELEASE_MIN_M) return config;
767
+ return { ...config, hnswConfig: { ...hnsw, m: MIN_M } };
768
+ }
654
769
  function validateVectorPromotion(config) {
655
770
  if (config === void 0) return;
656
771
  if (config.threshold !== void 0 && !isPositiveInteger(config.threshold)) {
@@ -666,8 +781,8 @@ function validateVectorPromotion(config) {
666
781
  }
667
782
  const hnsw = config.hnswConfig;
668
783
  if (hnsw === void 0) return;
669
- if (hnsw.m !== void 0 && (!isPositiveInteger(hnsw.m) || hnsw.m > MAX_M)) {
670
- fail(`vectorPromotion.hnswConfig.m must be a positive integer no greater than ${MAX_M}`, { m: hnsw.m });
784
+ if (hnsw.m !== void 0 && (!Number.isInteger(hnsw.m) || hnsw.m < MIN_M || hnsw.m > MAX_M)) {
785
+ fail(`vectorPromotion.hnswConfig.m must be a whole number from ${MIN_M} to ${MAX_M}`, { m: hnsw.m });
671
786
  }
672
787
  if (hnsw.efConstruction !== void 0 && !isPositiveInteger(hnsw.efConstruction)) {
673
788
  fail("vectorPromotion.hnswConfig.efConstruction must be a positive integer", {
@@ -680,22 +795,4 @@ function validateVectorPromotion(config) {
680
795
  }
681
796
  }
682
797
 
683
- // src/search/constants.ts
684
- var RESULT_WINDOW = 1e4;
685
- var DEFAULT_PAGE_SIZE = 10;
686
- var MAX_CURSOR_LENGTH = 40960;
687
- var MAX_SORT_FIELDS = 8;
688
- var MAX_SORT_FIELD_NAME_LENGTH = 255;
689
- var MAX_CURSOR_ANCHOR_CODE_POINTS = 512;
690
- var PREFIX_MAX_EXPANSIONS = 50;
691
- var DEFAULT_FUZZY_PREFIX_LENGTH = 2;
692
- var MAX_TOLERANCE = 10;
693
- var MAX_PREFIX_LENGTH = 1024;
694
- var DEFAULT_MAX_PER_GROUP = 1;
695
- var DEFAULT_RANK_CONSTANT = 60;
696
- var MIN_RANK_CONSTANT = 1;
697
- var DEFAULT_LINEAR_ALPHA = 0.5;
698
- var MIN_LINEAR_ALPHA = 0;
699
- var MAX_LINEAR_ALPHA = 1;
700
-
701
- export { ANTIMERIDIAN_SPAN_DEGREES, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PER_GROUP, DEFAULT_PAGE_SIZE, DEFAULT_RANK_CONSTANT, DEG_TO_RAD, EARTH_RADIUS_METERS, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_LATITUDE_DEGREES, MAX_LINEAR_ALPHA, MAX_LONGITUDE_DEGREES, MAX_NESTING_DEPTH, MAX_PATTERN_VALUE_LIMIT, MAX_PREFIX_LENGTH, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_TOLERANCE, MIN_LINEAR_ALPHA, MIN_PATTERN_VALUE_LIMIT, MIN_POLYGON_POINTS, MIN_RANK_CONSTANT, PREFIX_MAX_EXPANSIONS, PROTOTYPE_POLLUTION_KEYS, RADIUS_BOX_MARGIN_RADIANS, RADIUS_BOX_MARGIN_RATIO, RESULT_WINDOW, VECTOR_PATTERN, VINCENTY_CONVERGENCE_THRESHOLD, VINCENTY_MAX_ITERATIONS, extractVectorFieldsFromSchema, flattenSchema, isGeopointOrVectorType, isLatitude, isListBase, isLongitude, isQuantizationMode, isSingleStringFieldType, isStorageMode, isStringValuedBase, isWordIndexedBase, isWordIndexedFieldType, parsedTypeOf, requireValidFieldTypes, schemaFieldsOf, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateVectorPromotion, validateVectorStorage, withSortableOption };
798
+ export { ANTIMERIDIAN_SPAN_DEGREES, DEG_TO_RAD, EARTH_RADIUS_METERS, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_DOCUMENT_NESTING_DEPTH, MAX_LATITUDE_DEGREES, MAX_LONGITUDE_DEGREES, MAX_NESTING_DEPTH, MAX_PATTERN_VALUE_LIMIT, MAX_STORED_VECTOR_DIMENSION, MIN_PATTERN_VALUE_LIMIT, MIN_POLYGON_POINTS, PROTOTYPE_POLLUTION_KEYS, RADIUS_BOX_MARGIN_RADIANS, RADIUS_BOX_MARGIN_RATIO, VECTOR_PATTERN, VINCENTY_CONVERGENCE_THRESHOLD, VINCENTY_MAX_ITERATIONS, encodeDocument, extractVectorFieldsFromSchema, flattenSchema, isGeopointOrVectorType, isLatitude, isListBase, isLongitude, isQuantizationMode, isSingleStringFieldType, isStorageMode, isStringValuedBase, isWordIndexedBase, isWordIndexedFieldType, messagePackUpperBoundBytes, parsedTypeOf, partialFieldsOf, raiseEarlierReleaseGraphSetting, requireCreatableSchema, requireDocumentWithinSizeLimit, requireValidFieldTypes, schemaFieldsOf, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateVectorPromotion, validateVectorStorage, withSortableOption };