@delali/narsil 0.3.1-next.72 → 0.3.1-next.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-D6gjjQyA.d.ts → capabilities-CAMzmkAg.d.ts} +19 -4
- package/dist/{chunk-P2ZZN32C.mjs → chunk-2E35UM3G.mjs} +2 -12
- package/dist/chunk-3DDUNI6N.mjs +48 -0
- package/dist/{chunk-KZTI25D3.mjs → chunk-4MQ5F4IL.mjs} +1 -1
- package/dist/{chunk-HVJYTKK7.mjs → chunk-BVBIQQKG.mjs} +947 -1327
- package/dist/{chunk-2X2ZQOAB.mjs → chunk-CTOSGRM7.mjs} +101 -21
- package/dist/{chunk-7QMTA7AA.mjs → chunk-CUZOX6CN.mjs} +7784 -4276
- package/dist/chunk-I23G3IZC.mjs +593 -0
- package/dist/{chunk-RV77FR2F.mjs → chunk-I64LXKO3.mjs} +0 -1
- package/dist/chunk-KEHK2O6P.mjs +13 -0
- package/dist/{chunk-CVPA5Q7R.mjs → chunk-LX4JERXI.mjs} +9 -6
- package/dist/{chunk-EDMCZLSJ.mjs → chunk-NOZUQM7J.mjs} +193 -13
- package/dist/{chunk-LCCL6EEH.mjs → chunk-PO53FV4Q.mjs} +93 -94
- package/dist/chunk-T5ZWEWVD.mjs +658 -0
- package/dist/{chunk-Y52WZWAU.mjs → chunk-TVFLHM7M.mjs} +4 -4
- package/dist/{chunk-QR4OKRVY.mjs → chunk-VKX2BPXD.mjs} +147 -642
- package/dist/{chunk-3HQPUCCT.mjs → chunk-XW25X6AU.mjs} +66 -237
- package/dist/{chunk-M6XF5HDE.mjs → chunk-Z3DQPOJQ.mjs} +127 -30
- package/dist/{client-C2TiW29y.d.ts → client-C-I70ArH.d.ts} +77 -26
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +44 -22
- package/dist/{config-BLguB-om.d.ts → config-C6Iw5rCX.d.ts} +11 -4
- package/dist/{config-C7Ip8Ife.d.ts → config-DJPH3IQB.d.ts} +11 -2
- package/dist/direct-executor-KFAGILJC.mjs +17 -0
- package/dist/distribution/coordinator/etcd.d.ts +29 -16
- package/dist/distribution/coordinator/etcd.mjs +162 -75
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +22 -2
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +6 -3
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +40 -16
- package/dist/distribution.d.ts +9 -9
- package/dist/distribution.mjs +647 -366
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-Dm4446j1.d.ts → engine-B8eRPO7m.d.ts} +97 -188
- package/dist/index.browser.mjs +12132 -8592
- package/dist/index.d.ts +27 -8
- package/dist/index.mjs +15 -14
- package/dist/{memory-CKHTU1CL.d.ts → memory-DJEwWoCi.d.ts} +1204 -968
- package/dist/partial-words-DTIHO8G4.d.ts +19 -0
- package/dist/partial-words.d.ts +22 -0
- package/dist/partial-words.mjs +11 -0
- package/dist/pattern.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.d.ts +57 -29
- package/dist/persistence/durability/checkpoint-worker.mjs +26 -16
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +0 -1
- package/dist/{schema-BBOCdhND.d.ts → schema-DgyDijhf.d.ts} +6 -2
- package/dist/server.d.ts +11 -11
- package/dist/server.mjs +212 -32
- package/dist/{tasks-BECd0b__.d.ts → tasks-BVmAKYBh.d.ts} +3 -3
- package/dist/{thread-IBSIZZ6W.mjs → thread-W6DPKEF6.mjs} +82 -21
- package/dist/{types-BQDlIvfS.d.ts → types-B3b68nfF.d.ts} +1 -1
- package/dist/{types-D5OwhtsQ.d.ts → types-Cb9kiY5P.d.ts} +1 -1
- package/dist/{types-BiUmaiWT.d.ts → types-ChrVy4lH.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -3
- package/dist/workers/entry.mjs +4 -4
- package/package.json +15 -7
- package/dist/chunk-3L4U6EJQ.mjs +0 -49
- package/dist/chunk-IFBKLBGB.mjs +0 -7
- package/dist/chunk-XZ4PAREY.mjs +0 -25
- package/dist/direct-executor-SVCHXR74.mjs +0 -14
- package/dist/serialization/crc32-worker.d.ts +0 -20
- package/dist/serialization/crc32-worker.mjs +0 -62
|
@@ -1,55 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
// src/core/pattern-index/matcher-budget.ts
|
|
4
|
-
var POSITIONS_PER_WORD = 32;
|
|
5
|
-
function positionWords(positions) {
|
|
6
|
-
return Math.max(1, Math.ceil(positions / POSITIONS_PER_WORD));
|
|
7
|
-
}
|
|
8
|
-
function automatonBytes(positions) {
|
|
9
|
-
return 2 * positionWords(positions) * Uint32Array.BYTES_PER_ELEMENT + positions * MATCHER_CODE_POINT_BYTES;
|
|
10
|
-
}
|
|
11
|
-
function smallestMatcherBytes(positions) {
|
|
12
|
-
return automatonBytes(positions) + positions * MATCHER_SIMULATION_BYTES_PER_POSITION + MATCHER_STATE_OVERHEAD_BYTES + positions * Uint32Array.BYTES_PER_ELEMENT;
|
|
13
|
-
}
|
|
14
|
-
function widestPatternSplit(positions) {
|
|
15
|
-
return Math.max(1, Math.floor(MATCHER_STATE_BUDGET_BYTES / smallestMatcherBytes(positions)));
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
// src/pattern/automaton.ts
|
|
19
|
-
function setPosition(words, position) {
|
|
20
|
-
words[position >>> 5] |= 1 << (position & 31);
|
|
21
|
-
}
|
|
22
|
-
function hasPosition(words, position) {
|
|
23
|
-
return (words[position >>> 5] & 1 << (position & 31)) !== 0;
|
|
24
|
-
}
|
|
25
|
-
function literalAutomaton(codePoints, anchoredStart, anchoredEnd) {
|
|
26
|
-
const size = codePoints.length;
|
|
27
|
-
const words = positionWords(size);
|
|
28
|
-
const first = new Uint32Array(words);
|
|
29
|
-
const last = new Uint32Array(words);
|
|
30
|
-
if (size > 0) {
|
|
31
|
-
setPosition(first, 0);
|
|
32
|
-
setPosition(last, size - 1);
|
|
33
|
-
}
|
|
34
|
-
return {
|
|
35
|
-
size,
|
|
36
|
-
words,
|
|
37
|
-
first,
|
|
38
|
-
last,
|
|
39
|
-
nullable: size === 0,
|
|
40
|
-
anchoredStart,
|
|
41
|
-
anchoredEnd,
|
|
42
|
-
bytes: automatonBytes(size),
|
|
43
|
-
accepts: (position, codePoint) => codePoints[position] === codePoint,
|
|
44
|
-
followerCount: (position) => position + 1 < size ? 1 : 0,
|
|
45
|
-
follower: (position) => position + 1
|
|
46
|
-
};
|
|
47
|
-
}
|
|
1
|
+
import { bitsetHas, bitsetSet, PATTERN_RUN_CODE_POINTS } from './chunk-ZKRV5M2P.mjs';
|
|
48
2
|
|
|
49
3
|
// src/pattern/constants.ts
|
|
50
|
-
var MATCHER_TRANSITION_BYTES = 48;
|
|
51
|
-
var MATCHER_KEY_CHARACTER_BYTES = 2;
|
|
52
|
-
var MATCHER_MIN_CODE_POINTS_PER_STATE = 10;
|
|
53
4
|
var SHORTLIST_RUN_LOOKUPS = 16;
|
|
54
5
|
var SHORTLIST_INTERSECT_RATIO = 8;
|
|
55
6
|
var RUN_LIST_INITIAL_CAPACITY = 4;
|
|
@@ -57,205 +8,86 @@ var RUN_LIST_COMPACTION_FRACTION = 0.5;
|
|
|
57
8
|
var RUN_ENTRY_OVERHEAD_BYTES = 140;
|
|
58
9
|
var DOCUMENT_ENTRY_BYTES = 24;
|
|
59
10
|
|
|
60
|
-
// src/pattern/
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
11
|
+
// src/pattern/literal-matcher.ts
|
|
12
|
+
function bordersOf(codePoints) {
|
|
13
|
+
const length = codePoints.length;
|
|
14
|
+
const borders = new Uint32Array(length + 1);
|
|
15
|
+
let border = 0;
|
|
16
|
+
for (let at = 1; at < length; at++) {
|
|
17
|
+
while (border > 0 && codePoints[at] !== codePoints[border]) border = borders[border];
|
|
18
|
+
if (codePoints[at] === codePoints[border]) border++;
|
|
19
|
+
borders[at + 1] = border;
|
|
20
|
+
}
|
|
21
|
+
return borders;
|
|
22
|
+
}
|
|
23
|
+
function reachablePositionsOf(borders, length) {
|
|
24
|
+
const reachable = new Uint32Array(length + 1);
|
|
25
|
+
for (let prefix = 1; prefix <= length; prefix++) {
|
|
26
|
+
reachable[prefix] = (prefix < length ? 1 : 0) + reachable[borders[prefix]];
|
|
27
|
+
}
|
|
28
|
+
return reachable;
|
|
29
|
+
}
|
|
30
|
+
var LiteralMatcher = class {
|
|
31
|
+
constructor(codePoints, anchoredStart, anchoredEnd) {
|
|
32
|
+
this.codePoints = codePoints;
|
|
33
|
+
this.anchoredStart = anchoredStart;
|
|
34
|
+
this.anchoredEnd = anchoredEnd;
|
|
35
|
+
this.borders = bordersOf(codePoints);
|
|
36
|
+
this.reachable = reachablePositionsOf(this.borders, codePoints.length);
|
|
37
|
+
}
|
|
38
|
+
borders;
|
|
39
|
+
reachable;
|
|
40
|
+
prefix = 0;
|
|
41
|
+
alive = true;
|
|
74
42
|
accepting = false;
|
|
75
|
-
active;
|
|
76
|
-
following;
|
|
77
|
-
marks;
|
|
78
|
-
first;
|
|
79
|
-
activeCount = 0;
|
|
80
|
-
stamp = 0;
|
|
81
|
-
get steps() {
|
|
82
|
-
return this.activeCount;
|
|
83
|
-
}
|
|
84
|
-
get bytes() {
|
|
85
|
-
return this.automaton.size * MATCHER_SIMULATION_BYTES_PER_POSITION;
|
|
86
|
-
}
|
|
87
|
-
reset() {
|
|
88
|
-
this.active.set(this.first);
|
|
89
|
-
this.activeCount = this.first.length;
|
|
90
|
-
this.accepting = this.automaton.nullable;
|
|
91
|
-
}
|
|
92
|
-
get firstPositions() {
|
|
93
|
-
return this.first;
|
|
94
|
-
}
|
|
95
|
-
load(active, accepting) {
|
|
96
|
-
this.active.set(active);
|
|
97
|
-
this.activeCount = active.length;
|
|
98
|
-
this.accepting = accepting;
|
|
99
|
-
}
|
|
100
|
-
sortedActive() {
|
|
101
|
-
return this.active.slice(0, this.activeCount).sort();
|
|
102
|
-
}
|
|
103
|
-
advance(codePoint) {
|
|
104
|
-
const { automaton, marks } = this;
|
|
105
|
-
const following = this.following;
|
|
106
|
-
const stamp = this.nextStamp();
|
|
107
|
-
let count = 0;
|
|
108
|
-
let accepting = false;
|
|
109
|
-
for (let at = 0; at < this.activeCount; at++) {
|
|
110
|
-
const position = this.active[at];
|
|
111
|
-
if (!automaton.accepts(position, codePoint)) continue;
|
|
112
|
-
if (hasPosition(automaton.last, position)) accepting = true;
|
|
113
|
-
const followers = automaton.followerCount(position);
|
|
114
|
-
for (let index = 0; index < followers; index++) {
|
|
115
|
-
const follower = automaton.follower(position, index);
|
|
116
|
-
if (marks[follower] === stamp) continue;
|
|
117
|
-
marks[follower] = stamp;
|
|
118
|
-
following[count++] = follower;
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
if (!automaton.anchoredStart) {
|
|
122
|
-
for (const position of this.first) {
|
|
123
|
-
if (marks[position] === stamp) continue;
|
|
124
|
-
marks[position] = stamp;
|
|
125
|
-
following[count++] = position;
|
|
126
|
-
}
|
|
127
|
-
if (automaton.nullable) accepting = true;
|
|
128
|
-
}
|
|
129
|
-
this.following = this.active;
|
|
130
|
-
this.active = following;
|
|
131
|
-
this.activeCount = count;
|
|
132
|
-
this.accepting = accepting;
|
|
133
|
-
}
|
|
134
|
-
nextStamp() {
|
|
135
|
-
if (this.stamp === LAST_STAMP) {
|
|
136
|
-
this.marks.fill(0);
|
|
137
|
-
this.stamp = 0;
|
|
138
|
-
}
|
|
139
|
-
this.stamp++;
|
|
140
|
-
return this.stamp;
|
|
141
|
-
}
|
|
142
|
-
};
|
|
143
|
-
|
|
144
|
-
// src/pattern/lazy-dfa.ts
|
|
145
|
-
var UNCACHED_GENERATION = -1;
|
|
146
|
-
function stateKey(active, accepting) {
|
|
147
|
-
return `${active.join(",")}${accepting ? "+" : "-"}`;
|
|
148
|
-
}
|
|
149
|
-
function stateBytes(active, key) {
|
|
150
|
-
return MATCHER_STATE_OVERHEAD_BYTES + active.byteLength + key.length * MATCHER_KEY_CHARACTER_BYTES;
|
|
151
|
-
}
|
|
152
|
-
var LazyDfa = class {
|
|
153
|
-
constructor(automaton, stateLimitBytes) {
|
|
154
|
-
this.automaton = automaton;
|
|
155
|
-
this.simulation = new PositionSimulation(automaton);
|
|
156
|
-
this.cacheLimitBytes = stateLimitBytes - automaton.bytes - this.simulation.bytes;
|
|
157
|
-
this.start = this.intern(this.simulation.firstPositions, automaton.nullable);
|
|
158
|
-
this.state = this.start;
|
|
159
|
-
if (this.wantsSimulation()) this.simulate(this.start);
|
|
160
|
-
}
|
|
161
|
-
states = /* @__PURE__ */ new Map();
|
|
162
|
-
generation = 0;
|
|
163
|
-
cachedBytes = 0;
|
|
164
|
-
codePointsSinceReset = 0;
|
|
165
|
-
start;
|
|
166
|
-
state;
|
|
167
|
-
simulating = false;
|
|
168
43
|
decided = null;
|
|
169
|
-
simulation;
|
|
170
|
-
cacheLimitBytes;
|
|
171
44
|
begin() {
|
|
172
45
|
this.decided = null;
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
46
|
+
this.prefix = 0;
|
|
47
|
+
this.alive = this.codePoints.length > 0;
|
|
48
|
+
this.accepting = this.codePoints.length === 0;
|
|
49
|
+
if (!this.anchoredEnd && this.accepting) return this.decide(true);
|
|
176
50
|
return true;
|
|
177
51
|
}
|
|
178
52
|
feed(codePoint, tally) {
|
|
179
|
-
const steps = this.
|
|
180
|
-
if (steps === 0) return this.decide(!this.
|
|
53
|
+
const steps = this.steps();
|
|
54
|
+
if (steps === 0) return this.decide(!this.anchoredStart && this.codePoints.length === 0);
|
|
181
55
|
tally.add(steps);
|
|
182
|
-
if (this.
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
this.codePointsSinceReset++;
|
|
186
|
-
this.state = this.step(this.state, codePoint);
|
|
187
|
-
if (this.wantsSimulation()) this.simulate(this.state);
|
|
188
|
-
}
|
|
189
|
-
if (!this.automaton.anchoredEnd && this.accepting()) return this.decide(true);
|
|
56
|
+
if (this.anchoredStart) this.advanceAnchored(codePoint);
|
|
57
|
+
else this.advance(codePoint);
|
|
58
|
+
if (!this.anchoredEnd && this.accepting) return this.decide(true);
|
|
190
59
|
return true;
|
|
191
60
|
}
|
|
192
61
|
matched() {
|
|
193
|
-
return this.decided ?? (this.
|
|
62
|
+
return this.decided ?? (this.anchoredEnd && this.accepting);
|
|
63
|
+
}
|
|
64
|
+
steps() {
|
|
65
|
+
const length = this.codePoints.length;
|
|
66
|
+
if (length === 0) return 0;
|
|
67
|
+
if (this.anchoredStart) return this.alive && this.prefix < length ? 1 : 0;
|
|
68
|
+
return 1 + this.reachable[this.prefix];
|
|
69
|
+
}
|
|
70
|
+
advanceAnchored(codePoint) {
|
|
71
|
+
if (this.codePoints[this.prefix] === codePoint) {
|
|
72
|
+
this.prefix++;
|
|
73
|
+
this.accepting = this.prefix === this.codePoints.length;
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
this.alive = false;
|
|
77
|
+
this.accepting = false;
|
|
194
78
|
}
|
|
195
|
-
|
|
196
|
-
|
|
79
|
+
advance(codePoint) {
|
|
80
|
+
const { codePoints, borders } = this;
|
|
81
|
+
let prefix = this.prefix === codePoints.length ? borders[this.prefix] : this.prefix;
|
|
82
|
+
while (prefix > 0 && codePoints[prefix] !== codePoint) prefix = borders[prefix];
|
|
83
|
+
if (codePoints[prefix] === codePoint) prefix++;
|
|
84
|
+
this.prefix = prefix;
|
|
85
|
+
this.accepting = prefix === codePoints.length;
|
|
197
86
|
}
|
|
198
87
|
decide(outcome) {
|
|
199
88
|
this.decided = outcome;
|
|
200
89
|
return false;
|
|
201
90
|
}
|
|
202
|
-
wantsSimulation() {
|
|
203
|
-
return !this.simulating && this.state.generation === UNCACHED_GENERATION;
|
|
204
|
-
}
|
|
205
|
-
simulate(from) {
|
|
206
|
-
this.dropCache();
|
|
207
|
-
this.state = this.start;
|
|
208
|
-
this.simulation.load(from.active, from.accepting);
|
|
209
|
-
this.simulating = true;
|
|
210
|
-
}
|
|
211
|
-
dropCache() {
|
|
212
|
-
for (const state of this.states.values()) state.next.clear();
|
|
213
|
-
this.start.next.clear();
|
|
214
|
-
this.state.next.clear();
|
|
215
|
-
this.states = /* @__PURE__ */ new Map();
|
|
216
|
-
this.cachedBytes = 0;
|
|
217
|
-
}
|
|
218
|
-
step(state, codePoint) {
|
|
219
|
-
const current = state.generation === this.generation;
|
|
220
|
-
if (current) {
|
|
221
|
-
const cached = state.next.get(codePoint);
|
|
222
|
-
if (cached !== void 0) return cached;
|
|
223
|
-
}
|
|
224
|
-
const { simulation } = this;
|
|
225
|
-
simulation.load(state.active, state.accepting);
|
|
226
|
-
simulation.advance(codePoint);
|
|
227
|
-
const next = this.intern(simulation.sortedActive(), simulation.accepting);
|
|
228
|
-
if (!current || state.generation !== this.generation || next.generation !== this.generation) return next;
|
|
229
|
-
if (!this.reserve(MATCHER_TRANSITION_BYTES) || state.generation !== this.generation) return next;
|
|
230
|
-
state.next.set(codePoint, next);
|
|
231
|
-
return next;
|
|
232
|
-
}
|
|
233
|
-
intern(active, accepting) {
|
|
234
|
-
const key = stateKey(active, accepting);
|
|
235
|
-
const existing = this.states.get(key);
|
|
236
|
-
if (existing !== void 0) return existing;
|
|
237
|
-
if (!this.reserve(stateBytes(active, key))) {
|
|
238
|
-
return { active, accepting, generation: UNCACHED_GENERATION, next: /* @__PURE__ */ new Map() };
|
|
239
|
-
}
|
|
240
|
-
const state = { active, accepting, generation: this.generation, next: /* @__PURE__ */ new Map() };
|
|
241
|
-
this.states.set(key, state);
|
|
242
|
-
return state;
|
|
243
|
-
}
|
|
244
|
-
reserve(bytes) {
|
|
245
|
-
if (this.cachedBytes + bytes <= this.cacheLimitBytes) {
|
|
246
|
-
this.cachedBytes += bytes;
|
|
247
|
-
return true;
|
|
248
|
-
}
|
|
249
|
-
if (this.states.size === 0 || this.codePointsSinceReset < MATCHER_MIN_CODE_POINTS_PER_STATE * this.states.size) {
|
|
250
|
-
return false;
|
|
251
|
-
}
|
|
252
|
-
this.dropCache();
|
|
253
|
-
this.codePointsSinceReset = 0;
|
|
254
|
-
this.generation++;
|
|
255
|
-
this.start = this.intern(this.start.active, this.start.accepting);
|
|
256
|
-
if (this.start.generation === UNCACHED_GENERATION) return false;
|
|
257
|
-
return this.reserve(bytes);
|
|
258
|
-
}
|
|
259
91
|
};
|
|
260
92
|
|
|
261
93
|
// src/pattern/text.ts
|
|
@@ -391,10 +223,7 @@ function matcherFor(request) {
|
|
|
391
223
|
const { operator, text, caseFold, host } = request;
|
|
392
224
|
const testCodePoints = [];
|
|
393
225
|
appendTextCodePoints(text, caseFold, host, testCodePoints);
|
|
394
|
-
return new
|
|
395
|
-
literalAutomaton(testCodePoints, ANCHORED_START.has(operator), ANCHORED_END.has(operator)),
|
|
396
|
-
request.meter.matcherStateBytes
|
|
397
|
-
);
|
|
226
|
+
return new LiteralMatcher(Uint32Array.from(testCodePoints), ANCHORED_START.has(operator), ANCHORED_END.has(operator));
|
|
398
227
|
}
|
|
399
228
|
function matchesText(matcher, text, caseFold, host, folded, tally) {
|
|
400
229
|
if (!matcher.begin()) return matcher.matched();
|
|
@@ -782,4 +611,4 @@ var patternSearchModule = {
|
|
|
782
611
|
matchInto: matchPattern
|
|
783
612
|
};
|
|
784
613
|
|
|
785
|
-
export { GrowableUint32, patternSearchModule
|
|
614
|
+
export { GrowableUint32, patternSearchModule };
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { MAX_M } from './chunk-
|
|
2
|
-
import { holdsUnpairedSurrogate, exceedsCodePoints } from './chunk-LJYTW7GT.mjs';
|
|
3
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { EARLIER_RELEASE_MIN_M, MIN_M, MAX_M } from './chunk-KEHK2O6P.mjs';
|
|
2
|
+
import { compareCodePoints, holdsUnpairedSurrogate, exceedsCodePoints } from './chunk-LJYTW7GT.mjs';
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
|
|
4
|
+
import { encode } from '@msgpack/msgpack';
|
|
4
5
|
|
|
5
6
|
// src/schema/constants.ts
|
|
6
7
|
var MAX_NESTING_DEPTH = 4;
|
|
@@ -8,6 +9,10 @@ var MAX_DOCUMENT_NESTING_DEPTH = 32;
|
|
|
8
9
|
var DEFAULT_PATTERN_VALUE_LIMIT = 8192;
|
|
9
10
|
var MIN_PATTERN_VALUE_LIMIT = 1;
|
|
10
11
|
var MAX_PATTERN_VALUE_LIMIT = 65536;
|
|
12
|
+
var MAX_VECTOR_DIMENSION = 4096;
|
|
13
|
+
var MAX_STORED_VECTOR_DIMENSION = 65535;
|
|
14
|
+
var MAX_TEXT_FIELDS = 65536;
|
|
15
|
+
var MAX_DOCUMENT_BYTES = 16777216;
|
|
11
16
|
|
|
12
17
|
// src/schema/validator/shared.ts
|
|
13
18
|
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
@@ -57,11 +62,11 @@ function parseBase(baseName, path, type) {
|
|
|
57
62
|
invalidType(path, type, `Field "${path}" has an unsupported type: "${type}"`);
|
|
58
63
|
}
|
|
59
64
|
const dimension = Number.parseInt(vectorMatch[1], 10);
|
|
60
|
-
if (dimension <= 0) {
|
|
65
|
+
if (dimension <= 0 || dimension > MAX_STORED_VECTOR_DIMENSION) {
|
|
61
66
|
throw new NarsilError(
|
|
62
67
|
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
63
|
-
`Vector field "${path}"
|
|
64
|
-
{ field: path, dimension }
|
|
68
|
+
`Vector field "${path}" has ${dimension} dimensions, but a vector field holds from 1 to ${MAX_STORED_VECTOR_DIMENSION}`,
|
|
69
|
+
{ field: path, dimension, maximum: MAX_STORED_VECTOR_DIMENSION }
|
|
65
70
|
);
|
|
66
71
|
}
|
|
67
72
|
return { base: "vector", dimension };
|
|
@@ -194,9 +199,30 @@ function validateSchema(schema) {
|
|
|
194
199
|
if (Object.keys(schema).length === 0) {
|
|
195
200
|
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must define at least one field");
|
|
196
201
|
}
|
|
197
|
-
|
|
202
|
+
const validated = validateSchemaFields(schema, 1, "");
|
|
203
|
+
requireTextFieldCount(schemaFieldsOf(validated).filter((field) => isWordIndexedBase(field.base)).length);
|
|
204
|
+
return validated;
|
|
205
|
+
}
|
|
206
|
+
function requireTextFieldCount(count) {
|
|
207
|
+
if (count <= MAX_TEXT_FIELDS) return;
|
|
208
|
+
throw new NarsilError(
|
|
209
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
210
|
+
`The schema declares ${count} string and string[] fields, but a schema holds at most ${MAX_TEXT_FIELDS}`,
|
|
211
|
+
{ textFields: count, maximum: MAX_TEXT_FIELDS }
|
|
212
|
+
);
|
|
213
|
+
}
|
|
214
|
+
function requireCreatableSchema(schema) {
|
|
215
|
+
for (const field of schemaFieldsOf(schema)) {
|
|
216
|
+
if (field.base !== "vector" || field.dimension <= MAX_VECTOR_DIMENSION) continue;
|
|
217
|
+
throw new NarsilError(
|
|
218
|
+
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
219
|
+
`Vector field "${field.path}" has ${field.dimension} dimensions, but a new index takes at most ${MAX_VECTOR_DIMENSION}`,
|
|
220
|
+
{ field: field.path, dimension: field.dimension, maximum: MAX_VECTOR_DIMENSION }
|
|
221
|
+
);
|
|
222
|
+
}
|
|
198
223
|
}
|
|
199
224
|
function requireValidFieldTypes(flatSchema) {
|
|
225
|
+
let textFields = 0;
|
|
200
226
|
for (const [path, type] of Object.entries(flatSchema)) {
|
|
201
227
|
if (typeof type !== "string") {
|
|
202
228
|
throw new NarsilError(
|
|
@@ -205,8 +231,9 @@ function requireValidFieldTypes(flatSchema) {
|
|
|
205
231
|
{ field: path, type: String(type) }
|
|
206
232
|
);
|
|
207
233
|
}
|
|
208
|
-
parseFieldType(type, path);
|
|
234
|
+
if (isWordIndexedBase(parseFieldType(type, path).base)) textFields += 1;
|
|
209
235
|
}
|
|
236
|
+
requireTextFieldCount(textFields);
|
|
210
237
|
}
|
|
211
238
|
function flattenRecursive(schema, prefix, result) {
|
|
212
239
|
for (const [field, type] of Object.entries(schema)) {
|
|
@@ -239,6 +266,18 @@ function schemaFieldsOf(schema) {
|
|
|
239
266
|
parsedSchemas.set(schema, frozen);
|
|
240
267
|
return frozen;
|
|
241
268
|
}
|
|
269
|
+
var partialFieldsBySchema = /* @__PURE__ */ new WeakMap();
|
|
270
|
+
function partialFieldsOf(schema) {
|
|
271
|
+
const cached = partialFieldsBySchema.get(schema);
|
|
272
|
+
if (cached !== void 0) return cached;
|
|
273
|
+
const paths = [];
|
|
274
|
+
for (const field of schemaFieldsOf(schema)) {
|
|
275
|
+
if (field.partial) paths.push(field.path);
|
|
276
|
+
}
|
|
277
|
+
const frozen = Object.freeze(paths.sort(compareCodePoints));
|
|
278
|
+
partialFieldsBySchema.set(schema, frozen);
|
|
279
|
+
return frozen;
|
|
280
|
+
}
|
|
242
281
|
function extractVectorFieldsFromSchema(schema) {
|
|
243
282
|
const result = /* @__PURE__ */ new Map();
|
|
244
283
|
for (const field of schemaFieldsOf(schema)) {
|
|
@@ -474,7 +513,11 @@ function walkStorable(value, path, depth, ancestors, skip) {
|
|
|
474
513
|
reject(path, `holds a ${kind}, which cannot be stored or recovered`);
|
|
475
514
|
}
|
|
476
515
|
const container = value;
|
|
477
|
-
if (container instanceof Date
|
|
516
|
+
if (container instanceof Date) {
|
|
517
|
+
if (Number.isNaN(container.getTime())) reject(path, "holds an invalid date, which the engine cannot store");
|
|
518
|
+
return;
|
|
519
|
+
}
|
|
520
|
+
if (container instanceof Uint8Array) return;
|
|
478
521
|
const isArray = Array.isArray(container);
|
|
479
522
|
if (!isArray && !isPlainObject2(container)) {
|
|
480
523
|
reject(
|
|
@@ -628,6 +671,71 @@ function validateRequiredFields(document, required) {
|
|
|
628
671
|
}
|
|
629
672
|
}
|
|
630
673
|
}
|
|
674
|
+
function holdsNegativeZero(value) {
|
|
675
|
+
const pending = [value];
|
|
676
|
+
while (pending.length > 0) {
|
|
677
|
+
const next = pending.pop();
|
|
678
|
+
if (typeof next === "number") {
|
|
679
|
+
if (Object.is(next, -0)) return true;
|
|
680
|
+
continue;
|
|
681
|
+
}
|
|
682
|
+
if (next === null || typeof next !== "object" || ArrayBuffer.isView(next)) continue;
|
|
683
|
+
for (const child of Array.isArray(next) ? next : Object.values(next)) pending.push(child);
|
|
684
|
+
}
|
|
685
|
+
return false;
|
|
686
|
+
}
|
|
687
|
+
function encodeDocument(document) {
|
|
688
|
+
return encode(document, holdsNegativeZero(document) ? { forceIntegerToFloat: true } : void 0);
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
// src/schema/validator/document-size.ts
|
|
692
|
+
var LARGEST_HEADER_BYTES = 5;
|
|
693
|
+
var LARGEST_NUMBER_BYTES = 9;
|
|
694
|
+
var LARGEST_TIMESTAMP_BYTES = 15;
|
|
695
|
+
var LARGEST_UTF8_BYTES_PER_UNIT = 3;
|
|
696
|
+
function upperBoundBytes(value, budget, depth) {
|
|
697
|
+
if (value === null || value === void 0 || typeof value === "boolean") return 1;
|
|
698
|
+
if (typeof value === "number") return LARGEST_NUMBER_BYTES;
|
|
699
|
+
if (typeof value === "string") return LARGEST_HEADER_BYTES + value.length * LARGEST_UTF8_BYTES_PER_UNIT;
|
|
700
|
+
if (value instanceof Date) return LARGEST_TIMESTAMP_BYTES;
|
|
701
|
+
if (ArrayBuffer.isView(value)) return LARGEST_HEADER_BYTES + value.byteLength;
|
|
702
|
+
if (typeof value !== "object" || depth > MAX_DOCUMENT_NESTING_DEPTH) return LARGEST_NUMBER_BYTES;
|
|
703
|
+
let total = LARGEST_HEADER_BYTES;
|
|
704
|
+
if (Array.isArray(value)) {
|
|
705
|
+
for (const item of value) {
|
|
706
|
+
total += upperBoundBytes(item, budget - total, depth + 1);
|
|
707
|
+
if (total > budget) return total;
|
|
708
|
+
}
|
|
709
|
+
return total;
|
|
710
|
+
}
|
|
711
|
+
const record = value;
|
|
712
|
+
for (const key of Object.keys(record)) {
|
|
713
|
+
total += LARGEST_HEADER_BYTES + key.length * LARGEST_UTF8_BYTES_PER_UNIT;
|
|
714
|
+
total += upperBoundBytes(record[key], budget - total, depth + 1);
|
|
715
|
+
if (total > budget) return total;
|
|
716
|
+
}
|
|
717
|
+
return total;
|
|
718
|
+
}
|
|
719
|
+
function messagePackUpperBoundBytes(value) {
|
|
720
|
+
return upperBoundBytes(value, Number.POSITIVE_INFINITY, 0);
|
|
721
|
+
}
|
|
722
|
+
function encodedBytesOf(document) {
|
|
723
|
+
try {
|
|
724
|
+
return encodeDocument(document).byteLength;
|
|
725
|
+
} catch {
|
|
726
|
+
return null;
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
function requireDocumentWithinSizeLimit(document) {
|
|
730
|
+
if (upperBoundBytes(document, MAX_DOCUMENT_BYTES, 0) <= MAX_DOCUMENT_BYTES) return;
|
|
731
|
+
const encodedBytes = encodedBytesOf(document);
|
|
732
|
+
if (encodedBytes === null || encodedBytes <= MAX_DOCUMENT_BYTES) return;
|
|
733
|
+
throw new NarsilError(
|
|
734
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
735
|
+
`The document takes ${encodedBytes} bytes as MessagePack, vector values included, but a document may take at most ${MAX_DOCUMENT_BYTES}`,
|
|
736
|
+
{ bytes: encodedBytes, maximum: MAX_DOCUMENT_BYTES }
|
|
737
|
+
);
|
|
738
|
+
}
|
|
631
739
|
|
|
632
740
|
// src/schema/validator/vector-promotion.ts
|
|
633
741
|
var QUANTIZATION_MODES = /* @__PURE__ */ new Set(["osq8", "osq4", "osq2", "osq1", "none"]);
|
|
@@ -651,6 +759,13 @@ function validateVectorStorage(config, filesystemDurability) {
|
|
|
651
759
|
});
|
|
652
760
|
}
|
|
653
761
|
}
|
|
762
|
+
function raiseEarlierReleaseGraphSetting(config) {
|
|
763
|
+
if (typeof config !== "object" || config === null) return config;
|
|
764
|
+
const hnsw = config.hnswConfig;
|
|
765
|
+
if (typeof hnsw !== "object" || hnsw === null) return config;
|
|
766
|
+
if (hnsw.m !== EARLIER_RELEASE_MIN_M) return config;
|
|
767
|
+
return { ...config, hnswConfig: { ...hnsw, m: MIN_M } };
|
|
768
|
+
}
|
|
654
769
|
function validateVectorPromotion(config) {
|
|
655
770
|
if (config === void 0) return;
|
|
656
771
|
if (config.threshold !== void 0 && !isPositiveInteger(config.threshold)) {
|
|
@@ -666,8 +781,8 @@ function validateVectorPromotion(config) {
|
|
|
666
781
|
}
|
|
667
782
|
const hnsw = config.hnswConfig;
|
|
668
783
|
if (hnsw === void 0) return;
|
|
669
|
-
if (hnsw.m !== void 0 && (!
|
|
670
|
-
fail(`vectorPromotion.hnswConfig.m must be a
|
|
784
|
+
if (hnsw.m !== void 0 && (!Number.isInteger(hnsw.m) || hnsw.m < MIN_M || hnsw.m > MAX_M)) {
|
|
785
|
+
fail(`vectorPromotion.hnswConfig.m must be a whole number from ${MIN_M} to ${MAX_M}`, { m: hnsw.m });
|
|
671
786
|
}
|
|
672
787
|
if (hnsw.efConstruction !== void 0 && !isPositiveInteger(hnsw.efConstruction)) {
|
|
673
788
|
fail("vectorPromotion.hnswConfig.efConstruction must be a positive integer", {
|
|
@@ -680,22 +795,4 @@ function validateVectorPromotion(config) {
|
|
|
680
795
|
}
|
|
681
796
|
}
|
|
682
797
|
|
|
683
|
-
|
|
684
|
-
var RESULT_WINDOW = 1e4;
|
|
685
|
-
var DEFAULT_PAGE_SIZE = 10;
|
|
686
|
-
var MAX_CURSOR_LENGTH = 40960;
|
|
687
|
-
var MAX_SORT_FIELDS = 8;
|
|
688
|
-
var MAX_SORT_FIELD_NAME_LENGTH = 255;
|
|
689
|
-
var MAX_CURSOR_ANCHOR_CODE_POINTS = 512;
|
|
690
|
-
var PREFIX_MAX_EXPANSIONS = 50;
|
|
691
|
-
var DEFAULT_FUZZY_PREFIX_LENGTH = 2;
|
|
692
|
-
var MAX_TOLERANCE = 10;
|
|
693
|
-
var MAX_PREFIX_LENGTH = 1024;
|
|
694
|
-
var DEFAULT_MAX_PER_GROUP = 1;
|
|
695
|
-
var DEFAULT_RANK_CONSTANT = 60;
|
|
696
|
-
var MIN_RANK_CONSTANT = 1;
|
|
697
|
-
var DEFAULT_LINEAR_ALPHA = 0.5;
|
|
698
|
-
var MIN_LINEAR_ALPHA = 0;
|
|
699
|
-
var MAX_LINEAR_ALPHA = 1;
|
|
700
|
-
|
|
701
|
-
export { ANTIMERIDIAN_SPAN_DEGREES, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PER_GROUP, DEFAULT_PAGE_SIZE, DEFAULT_RANK_CONSTANT, DEG_TO_RAD, EARTH_RADIUS_METERS, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_LATITUDE_DEGREES, MAX_LINEAR_ALPHA, MAX_LONGITUDE_DEGREES, MAX_NESTING_DEPTH, MAX_PATTERN_VALUE_LIMIT, MAX_PREFIX_LENGTH, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_TOLERANCE, MIN_LINEAR_ALPHA, MIN_PATTERN_VALUE_LIMIT, MIN_POLYGON_POINTS, MIN_RANK_CONSTANT, PREFIX_MAX_EXPANSIONS, PROTOTYPE_POLLUTION_KEYS, RADIUS_BOX_MARGIN_RADIANS, RADIUS_BOX_MARGIN_RATIO, RESULT_WINDOW, VECTOR_PATTERN, VINCENTY_CONVERGENCE_THRESHOLD, VINCENTY_MAX_ITERATIONS, extractVectorFieldsFromSchema, flattenSchema, isGeopointOrVectorType, isLatitude, isListBase, isLongitude, isQuantizationMode, isSingleStringFieldType, isStorageMode, isStringValuedBase, isWordIndexedBase, isWordIndexedFieldType, parsedTypeOf, requireValidFieldTypes, schemaFieldsOf, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateVectorPromotion, validateVectorStorage, withSortableOption };
|
|
798
|
+
export { ANTIMERIDIAN_SPAN_DEGREES, DEG_TO_RAD, EARTH_RADIUS_METERS, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_DOCUMENT_NESTING_DEPTH, MAX_LATITUDE_DEGREES, MAX_LONGITUDE_DEGREES, MAX_NESTING_DEPTH, MAX_PATTERN_VALUE_LIMIT, MAX_STORED_VECTOR_DIMENSION, MIN_PATTERN_VALUE_LIMIT, MIN_POLYGON_POINTS, PROTOTYPE_POLLUTION_KEYS, RADIUS_BOX_MARGIN_RADIANS, RADIUS_BOX_MARGIN_RATIO, VECTOR_PATTERN, VINCENTY_CONVERGENCE_THRESHOLD, VINCENTY_MAX_ITERATIONS, encodeDocument, extractVectorFieldsFromSchema, flattenSchema, isGeopointOrVectorType, isLatitude, isListBase, isLongitude, isQuantizationMode, isSingleStringFieldType, isStorageMode, isStringValuedBase, isWordIndexedBase, isWordIndexedFieldType, messagePackUpperBoundBytes, parsedTypeOf, partialFieldsOf, raiseEarlierReleaseGraphSetting, requireCreatableSchema, requireDocumentWithinSizeLimit, requireValidFieldTypes, schemaFieldsOf, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateVectorPromotion, validateVectorStorage, withSortableOption };
|