@huggingface/transformers-structured-output 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +56 -0
- package/dist/index.cjs +2425 -0
- package/dist/index.cjs.map +7 -0
- package/dist/index.js +2402 -0
- package/dist/index.js.map +7 -0
- package/package.json +65 -0
- package/src/StructuredOutputProcessor.ts +103 -0
- package/src/engine/constraint.ts +331 -0
- package/src/engine/index.ts +2 -0
- package/src/engine/json.ts +1853 -0
- package/src/engine/regex.ts +419 -0
- package/src/engine/tokenizer.ts +205 -0
- package/src/engine/types.ts +12 -0
- package/src/index.ts +2 -0
- package/src/utils/mask.ts +30 -0
- package/types/StructuredOutputProcessor.d.ts +22 -0
- package/types/StructuredOutputProcessor.d.ts.map +1 -0
- package/types/engine/constraint.d.ts +30 -0
- package/types/engine/constraint.d.ts.map +1 -0
- package/types/engine/index.d.ts +3 -0
- package/types/engine/index.d.ts.map +1 -0
- package/types/engine/json.d.ts +70 -0
- package/types/engine/json.d.ts.map +1 -0
- package/types/engine/regex.d.ts +3 -0
- package/types/engine/regex.d.ts.map +1 -0
- package/types/engine/tokenizer.d.ts +8 -0
- package/types/engine/tokenizer.d.ts.map +1 -0
- package/types/engine/types.d.ts +16 -0
- package/types/engine/types.d.ts.map +1 -0
- package/types/index.d.ts +3 -0
- package/types/index.d.ts.map +1 -0
- package/types/utils/mask.d.ts +3 -0
- package/types/utils/mask.d.ts.map +1 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,2402 @@
|
|
|
1
|
+
// src/StructuredOutputProcessor.ts
|
|
2
|
+
import { LogitsProcessor, LogitsProcessorList } from "@huggingface/transformers";
|
|
3
|
+
|
|
4
|
+
// src/engine/json.ts
|
|
5
|
+
var DEFAULT_GUIDANCE = {
|
|
6
|
+
itemSeparator: ",",
|
|
7
|
+
keySeparator: ":",
|
|
8
|
+
itemBytes: Uint8Array.of(44),
|
|
9
|
+
keyBytes: Uint8Array.of(58),
|
|
10
|
+
whitespaceFlexible: true
|
|
11
|
+
};
|
|
12
|
+
var DEAD = { mode: "dead", schema: false, stack: [], guidance: DEFAULT_GUIDANCE };
|
|
13
|
+
var encoder = new TextEncoder();
|
|
14
|
+
var decoder = new TextDecoder("utf-8", { fatal: true });
|
|
15
|
+
var schemaIds = /* @__PURE__ */ new WeakMap();
|
|
16
|
+
var propertyKeyBytes = /* @__PURE__ */ new WeakMap();
|
|
17
|
+
var finiteStringCache = /* @__PURE__ */ new WeakMap();
|
|
18
|
+
var nextSchemaId = 0;
|
|
19
|
+
var SIMPLE_ESCAPES = {
|
|
20
|
+
34: '"',
|
|
21
|
+
47: "/",
|
|
22
|
+
92: "\\",
|
|
23
|
+
98: "\b",
|
|
24
|
+
102: "\f",
|
|
25
|
+
110: "\n",
|
|
26
|
+
114: "\r",
|
|
27
|
+
116: " "
|
|
28
|
+
};
|
|
29
|
+
var ASSERTED_FORMATS = /* @__PURE__ */ new Set([
|
|
30
|
+
"date",
|
|
31
|
+
"time",
|
|
32
|
+
"date-time",
|
|
33
|
+
"duration",
|
|
34
|
+
"email",
|
|
35
|
+
"hostname",
|
|
36
|
+
"ipv4",
|
|
37
|
+
"ipv6",
|
|
38
|
+
"uuid",
|
|
39
|
+
"uri",
|
|
40
|
+
"uri-reference",
|
|
41
|
+
"regex",
|
|
42
|
+
"json-pointer",
|
|
43
|
+
"relative-json-pointer"
|
|
44
|
+
]);
|
|
45
|
+
function compileJsonSchema(schema, stringKeyClamp = Infinity) {
|
|
46
|
+
registerSchemaContext(schema, schema);
|
|
47
|
+
checkSchema(schema, "$");
|
|
48
|
+
checkReferences(schema);
|
|
49
|
+
const guidance = guidanceFrom(schema);
|
|
50
|
+
return {
|
|
51
|
+
initial: { mode: "value", schema, stack: [], guidance },
|
|
52
|
+
transition,
|
|
53
|
+
viable: (state) => state !== DEAD,
|
|
54
|
+
accepting: isAccepting,
|
|
55
|
+
stringCapacity,
|
|
56
|
+
maskKey: (state) => stateMaskKey(state, stringKeyClamp)
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
function stringCapacity(state) {
|
|
60
|
+
if (state.mode !== "string" || state.highSurrogate !== void 0 || finiteStringValues(state.schema) !== null)
|
|
61
|
+
return void 0;
|
|
62
|
+
if ((state.stringPending ?? 0) !== 0) return void 0;
|
|
63
|
+
const maximum = directStringMaxLength(state.schema);
|
|
64
|
+
if (maximum === void 0) return Infinity;
|
|
65
|
+
return maximum - (state.stringLength ?? 0);
|
|
66
|
+
}
|
|
67
|
+
function stateMaskKey(state, stringKeyClamp) {
|
|
68
|
+
if (state === DEAD) return void 0;
|
|
69
|
+
const frames = state.stack.map((frame) => {
|
|
70
|
+
if (frame.kind === "object") {
|
|
71
|
+
const entries = [...frame.entries].sort((left, right) => left.key.localeCompare(right.key)).map((entry) => `${JSON.stringify(entry.key)}:${nodeKey(entry.node)}`).join(",");
|
|
72
|
+
return `o${schemaId(frame.schema)}[${entries}]${frame.key === void 0 ? "" : `:${JSON.stringify(frame.key)}:${schemaId(frame.childSchema)}`}`;
|
|
73
|
+
}
|
|
74
|
+
return `a${schemaId(frame.schema)}[${frame.items.map(nodeKey).join(",")}]`;
|
|
75
|
+
});
|
|
76
|
+
const key = [
|
|
77
|
+
state.mode,
|
|
78
|
+
schemaId(state.schema),
|
|
79
|
+
state.bytes === void 0 ? "" : stringLengthKey(state, stringKeyClamp) ?? bytesKey(state.bytes),
|
|
80
|
+
state.text ?? "",
|
|
81
|
+
state.literal ?? "",
|
|
82
|
+
state.index ?? "",
|
|
83
|
+
state.highSurrogate ?? "",
|
|
84
|
+
...frames
|
|
85
|
+
].join("|");
|
|
86
|
+
return key.length <= 2048 ? key : void 0;
|
|
87
|
+
}
|
|
88
|
+
var CONTENT_DEPENDENT_STRING_KEYWORDS = [
|
|
89
|
+
"pattern",
|
|
90
|
+
"format",
|
|
91
|
+
"const",
|
|
92
|
+
"enum",
|
|
93
|
+
"$ref",
|
|
94
|
+
"allOf",
|
|
95
|
+
"anyOf",
|
|
96
|
+
"oneOf",
|
|
97
|
+
"not",
|
|
98
|
+
"if"
|
|
99
|
+
];
|
|
100
|
+
var CONTENT_DEPENDENT_FRAME_KEYWORDS = [
|
|
101
|
+
"$ref",
|
|
102
|
+
"allOf",
|
|
103
|
+
"anyOf",
|
|
104
|
+
"oneOf",
|
|
105
|
+
"not",
|
|
106
|
+
"if",
|
|
107
|
+
"uniqueItems",
|
|
108
|
+
"contains",
|
|
109
|
+
"dependentSchemas",
|
|
110
|
+
"dependencies"
|
|
111
|
+
];
|
|
112
|
+
var contentIndependentStrings = /* @__PURE__ */ new WeakMap();
|
|
113
|
+
var contentNeutralFrames = /* @__PURE__ */ new WeakMap();
|
|
114
|
+
function isContentIndependent(schema, keywords, cache) {
|
|
115
|
+
if (schema === true) return true;
|
|
116
|
+
if (schema === false) return false;
|
|
117
|
+
let result = cache.get(schema);
|
|
118
|
+
if (result === void 0) {
|
|
119
|
+
result = keywords.every((keyword) => schema[keyword] === void 0);
|
|
120
|
+
cache.set(schema, result);
|
|
121
|
+
}
|
|
122
|
+
return result;
|
|
123
|
+
}
|
|
124
|
+
function stringLengthKey(state, clamp) {
|
|
125
|
+
if (state.mode !== "string" || state.highSurrogate !== void 0) return void 0;
|
|
126
|
+
if (!isContentIndependent(state.schema, CONTENT_DEPENDENT_STRING_KEYWORDS, contentIndependentStrings)) {
|
|
127
|
+
return void 0;
|
|
128
|
+
}
|
|
129
|
+
for (const frame of state.stack) {
|
|
130
|
+
if (!isContentIndependent(frame.schema, CONTENT_DEPENDENT_FRAME_KEYWORDS, contentNeutralFrames)) {
|
|
131
|
+
return void 0;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
if ((state.stringPending ?? 0) !== 0) return void 0;
|
|
135
|
+
const length = state.stringLength ?? 0;
|
|
136
|
+
const maximum = directStringMaxLength(state.schema);
|
|
137
|
+
const remaining = Math.min(maximum === void 0 ? Infinity : maximum - length, clamp);
|
|
138
|
+
const minimum = state.schema !== true && state.schema !== false && typeof state.schema.minLength === "number" ? state.schema.minLength : 0;
|
|
139
|
+
return `#${remaining === Infinity ? "inf" : remaining}:${length >= minimum ? "" : length}`;
|
|
140
|
+
}
|
|
141
|
+
function nodeKey(node) {
|
|
142
|
+
if (node.kind === "number") return `n${node.raw}`;
|
|
143
|
+
if (node.kind === "string") return `s${JSON.stringify(node.value)}`;
|
|
144
|
+
if (node.kind === "boolean") return node.value ? "t" : "f";
|
|
145
|
+
if (node.kind === "null") return "z";
|
|
146
|
+
if (node.kind === "array") return `[${node.items.map(nodeKey).join(",")}]`;
|
|
147
|
+
return `{${[...node.entries].sort((left, right) => left.key.localeCompare(right.key)).map((entry) => `${JSON.stringify(entry.key)}:${nodeKey(entry.node)}`).join(",")}}`;
|
|
148
|
+
}
|
|
149
|
+
function bytesKey(bytes) {
|
|
150
|
+
let result = "";
|
|
151
|
+
for (const byte of bytes) result += byte.toString(16).padStart(2, "0");
|
|
152
|
+
return result;
|
|
153
|
+
}
|
|
154
|
+
function schemaId(schema) {
|
|
155
|
+
if (schema === true) return "t";
|
|
156
|
+
if (schema === false) return "f";
|
|
157
|
+
let id = schemaIds.get(schema);
|
|
158
|
+
if (id === void 0) {
|
|
159
|
+
id = nextSchemaId++;
|
|
160
|
+
schemaIds.set(schema, id);
|
|
161
|
+
}
|
|
162
|
+
return id;
|
|
163
|
+
}
|
|
164
|
+
function transition(state, byte) {
|
|
165
|
+
if (state === DEAD) return DEAD;
|
|
166
|
+
if (state.guidance.whitespaceFlexible && isWhitespace(byte) && allowsWhitespace(state.mode) && !separatorExpects(state, byte))
|
|
167
|
+
return state;
|
|
168
|
+
switch (state.mode) {
|
|
169
|
+
case "value":
|
|
170
|
+
case "array-value":
|
|
171
|
+
if (state.mode === "array-value" && byte === 93) return closeArray(state);
|
|
172
|
+
if (state.mode === "array-value" && exceedsMaxItems(state.stack.at(-1))) return DEAD;
|
|
173
|
+
return startValue(state, byte);
|
|
174
|
+
case "object-key":
|
|
175
|
+
if (byte === 125) return closeObject(state);
|
|
176
|
+
return byte === 34 ? { ...state, mode: "key-string", bytes: [], stringLength: 0, stringPending: 0 } : DEAD;
|
|
177
|
+
case "colon":
|
|
178
|
+
return separatorByte(state, byte, state.guidance.keyBytes, "value");
|
|
179
|
+
case "item-separator":
|
|
180
|
+
return separatorByte(
|
|
181
|
+
state,
|
|
182
|
+
byte,
|
|
183
|
+
state.guidance.itemBytes,
|
|
184
|
+
state.stack.at(-1)?.kind === "object" ? "object-key" : "array-value"
|
|
185
|
+
);
|
|
186
|
+
case "after-value":
|
|
187
|
+
return afterValue(state, byte);
|
|
188
|
+
case "string":
|
|
189
|
+
case "key-string":
|
|
190
|
+
return stringByte(state, byte);
|
|
191
|
+
case "escape":
|
|
192
|
+
case "key-escape":
|
|
193
|
+
return escapeByte(state, byte);
|
|
194
|
+
case "unicode":
|
|
195
|
+
case "key-unicode":
|
|
196
|
+
return unicodeByte(state, byte);
|
|
197
|
+
case "number":
|
|
198
|
+
return numberByte(state, byte);
|
|
199
|
+
case "literal":
|
|
200
|
+
return literalByte(state, byte);
|
|
201
|
+
case "done":
|
|
202
|
+
case "dead":
|
|
203
|
+
return DEAD;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
function startValue(state, byte) {
|
|
207
|
+
const kind = byteKind(byte);
|
|
208
|
+
if (kind === void 0) return DEAD;
|
|
209
|
+
const schema = selectSchema(state.schema, kind);
|
|
210
|
+
if (schema === null) return DEAD;
|
|
211
|
+
if (byte === 34) return { ...state, mode: "string", schema, bytes: [], stringLength: 0, stringPending: 0 };
|
|
212
|
+
if (byte === 123) {
|
|
213
|
+
return {
|
|
214
|
+
mode: "object-key",
|
|
215
|
+
schema,
|
|
216
|
+
stack: [...state.stack, { kind: "object", schema, seen: /* @__PURE__ */ new Set(), entries: [] }],
|
|
217
|
+
guidance: state.guidance
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
if (byte === 91) {
|
|
221
|
+
return {
|
|
222
|
+
mode: "array-value",
|
|
223
|
+
schema: itemSchema(schema, 0),
|
|
224
|
+
stack: [
|
|
225
|
+
...state.stack,
|
|
226
|
+
{
|
|
227
|
+
kind: "array",
|
|
228
|
+
schema,
|
|
229
|
+
length: 0,
|
|
230
|
+
items: []
|
|
231
|
+
}
|
|
232
|
+
],
|
|
233
|
+
guidance: state.guidance
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
if (byte === 116) return { ...state, mode: "literal", schema, literal: "true", literalValue: true, index: 1 };
|
|
237
|
+
if (byte === 102) return { ...state, mode: "literal", schema, literal: "false", literalValue: false, index: 1 };
|
|
238
|
+
if (byte === 110) return { ...state, mode: "literal", schema, literal: "null", literalValue: null, index: 1 };
|
|
239
|
+
const text = String.fromCharCode(byte);
|
|
240
|
+
if (!integerPrefixViable(schema, text)) return DEAD;
|
|
241
|
+
return { ...state, mode: "number", schema, text };
|
|
242
|
+
}
|
|
243
|
+
function isAccepting(state) {
|
|
244
|
+
if (state.mode === "done") return true;
|
|
245
|
+
return state.mode === "number" && state.stack.length === 0 && numberComplete(state.text) && validateNode(state.schema, { kind: "number", value: Number(state.text), raw: state.text });
|
|
246
|
+
}
|
|
247
|
+
function stringByte(state, byte) {
|
|
248
|
+
const isKey = state.mode === "key-string";
|
|
249
|
+
if (state.highSurrogate !== void 0 && byte !== 92) return DEAD;
|
|
250
|
+
if (byte === 34) {
|
|
251
|
+
const value = decodeString(state.bytes ?? []);
|
|
252
|
+
if (value === null) return DEAD;
|
|
253
|
+
if (isKey) return completeKey(state, value);
|
|
254
|
+
return completeValue(state, { kind: "string", value });
|
|
255
|
+
}
|
|
256
|
+
if (byte === 92) {
|
|
257
|
+
if (isKey ? !keyEscapeAllowed(state) : !stringEscapeAllowed(state)) return DEAD;
|
|
258
|
+
return { ...state, mode: isKey ? "key-escape" : "escape" };
|
|
259
|
+
}
|
|
260
|
+
if (byte < 32) return DEAD;
|
|
261
|
+
let length = state.stringLength ?? 0;
|
|
262
|
+
let pending = state.stringPending ?? 0;
|
|
263
|
+
if ((byte & 192) === 128) {
|
|
264
|
+
pending = pending > 0 ? pending - 1 : -1;
|
|
265
|
+
} else {
|
|
266
|
+
pending = pending === 0 ? utf8Continuations(byte) : -1;
|
|
267
|
+
length++;
|
|
268
|
+
}
|
|
269
|
+
const bytes = [...state.bytes ?? [], byte];
|
|
270
|
+
if (isKey && !keyPrefixAllowed(state, bytes)) return DEAD;
|
|
271
|
+
if (!isKey && !stringContentAllowed(state.schema, bytes, length)) return DEAD;
|
|
272
|
+
return { ...state, bytes, stringLength: length, stringPending: pending };
|
|
273
|
+
}
|
|
274
|
+
function utf8Continuations(lead) {
|
|
275
|
+
if (lead < 128) return 0;
|
|
276
|
+
if (lead < 194) return -1;
|
|
277
|
+
if (lead < 224) return 1;
|
|
278
|
+
if (lead < 240) return 2;
|
|
279
|
+
if (lead < 245) return 3;
|
|
280
|
+
return -1;
|
|
281
|
+
}
|
|
282
|
+
function escapeByte(state, byte) {
|
|
283
|
+
const isKey = state.mode === "key-escape";
|
|
284
|
+
if (state.highSurrogate !== void 0 && byte !== 117) return DEAD;
|
|
285
|
+
if (byte === 117) {
|
|
286
|
+
return { ...state, mode: isKey ? "key-unicode" : "unicode", text: "" };
|
|
287
|
+
}
|
|
288
|
+
const escaped = SIMPLE_ESCAPES[byte];
|
|
289
|
+
if (escaped === void 0) return DEAD;
|
|
290
|
+
const bytes = [...state.bytes ?? [], ...encoder.encode(escaped)];
|
|
291
|
+
const length = (state.stringLength ?? 0) + 1;
|
|
292
|
+
const pending = (state.stringPending ?? 0) === 0 ? 0 : -1;
|
|
293
|
+
if (isKey && !keyPrefixAllowed(state, bytes)) return DEAD;
|
|
294
|
+
if (!isKey && !stringContentAllowed(state.schema, bytes, length)) return DEAD;
|
|
295
|
+
return { ...state, mode: isKey ? "key-string" : "string", bytes, stringLength: length, stringPending: pending };
|
|
296
|
+
}
|
|
297
|
+
function unicodeByte(state, byte) {
|
|
298
|
+
if (!isHex(byte)) return DEAD;
|
|
299
|
+
const text = `${state.text ?? ""}${String.fromCharCode(byte)}`;
|
|
300
|
+
if (text.length < 4) {
|
|
301
|
+
if (state.mode === "key-unicode" && !unicodeKeyPrefixAllowed(state, text)) return DEAD;
|
|
302
|
+
return { ...state, text };
|
|
303
|
+
}
|
|
304
|
+
const codeUnit = Number.parseInt(text, 16);
|
|
305
|
+
const isKey = state.mode === "key-unicode";
|
|
306
|
+
const mode = isKey ? "key-string" : "string";
|
|
307
|
+
if (codeUnit >= 55296 && codeUnit <= 56319) {
|
|
308
|
+
if (state.highSurrogate !== void 0) return DEAD;
|
|
309
|
+
return { ...state, mode, text: void 0, highSurrogate: codeUnit };
|
|
310
|
+
}
|
|
311
|
+
let value;
|
|
312
|
+
if (codeUnit >= 56320 && codeUnit <= 57343) {
|
|
313
|
+
if (state.highSurrogate === void 0) return DEAD;
|
|
314
|
+
const codePoint = 65536 + (state.highSurrogate - 55296 << 10) + (codeUnit - 56320);
|
|
315
|
+
value = String.fromCodePoint(codePoint);
|
|
316
|
+
} else {
|
|
317
|
+
if (state.highSurrogate !== void 0) return DEAD;
|
|
318
|
+
value = String.fromCharCode(codeUnit);
|
|
319
|
+
}
|
|
320
|
+
const bytes = [...state.bytes ?? [], ...encoder.encode(value)];
|
|
321
|
+
const length = (state.stringLength ?? 0) + 1;
|
|
322
|
+
const pending = (state.stringPending ?? 0) === 0 ? 0 : -1;
|
|
323
|
+
if (isKey && !keyPrefixAllowed(state, bytes)) return DEAD;
|
|
324
|
+
if (!isKey && !stringContentAllowed(state.schema, bytes, length)) return DEAD;
|
|
325
|
+
return {
|
|
326
|
+
...state,
|
|
327
|
+
mode,
|
|
328
|
+
bytes,
|
|
329
|
+
text: void 0,
|
|
330
|
+
highSurrogate: void 0,
|
|
331
|
+
stringLength: length,
|
|
332
|
+
stringPending: pending
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
function completeKey(state, key) {
|
|
336
|
+
const frame = topObject(state);
|
|
337
|
+
if (frame.seen.has(key)) return DEAD;
|
|
338
|
+
const childSchema = propertySchema(frame.schema, key, frame.entries);
|
|
339
|
+
if (childSchema === null) return DEAD;
|
|
340
|
+
return {
|
|
341
|
+
...state,
|
|
342
|
+
mode: "colon",
|
|
343
|
+
bytes: void 0,
|
|
344
|
+
index: 0,
|
|
345
|
+
stack: replaceTop(state.stack, { ...frame, key, childSchema })
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
function literalByte(state, byte) {
|
|
349
|
+
const index = state.index ?? 0;
|
|
350
|
+
const literal2 = state.literal;
|
|
351
|
+
if (byte !== literal2.charCodeAt(index)) return DEAD;
|
|
352
|
+
if (index + 1 < literal2.length) return { ...state, index: index + 1 };
|
|
353
|
+
const node = state.literalValue === null ? { kind: "null", value: null } : { kind: "boolean", value: state.literalValue };
|
|
354
|
+
return completeValue(state, node);
|
|
355
|
+
}
|
|
356
|
+
function numberByte(state, byte) {
|
|
357
|
+
if (isNumberByte(byte)) {
|
|
358
|
+
const text = `${state.text}${String.fromCharCode(byte)}`;
|
|
359
|
+
if (numberPrefixValid(text) && integerPrefixViable(state.schema, text)) return { ...state, text };
|
|
360
|
+
}
|
|
361
|
+
const value = finishNumber(state);
|
|
362
|
+
return value === DEAD ? DEAD : transition(value, byte);
|
|
363
|
+
}
|
|
364
|
+
var SCALE_DIGIT_LIMIT = 3;
|
|
365
|
+
var integerRestrictions = /* @__PURE__ */ new WeakMap();
|
|
366
|
+
function integerPrefixViable(schema, text) {
|
|
367
|
+
const restriction = integerRestriction(schema);
|
|
368
|
+
if (restriction === null) return true;
|
|
369
|
+
const { lo, hi } = restriction;
|
|
370
|
+
if (text === "-") return lo <= Math.min(hi, 0);
|
|
371
|
+
const match = /^(-?)(0|[1-9]\d*)(\.0{0,3})?(?:[eE]\+?(\d{0,3}))?$/.exec(text);
|
|
372
|
+
if (match === null) return false;
|
|
373
|
+
if (match[3] === "." && match[4] !== void 0) return false;
|
|
374
|
+
const negative = match[1] === "-";
|
|
375
|
+
if (match[3] === void 0 && match[4] === void 0) {
|
|
376
|
+
return integerDigitsReachable(lo, hi, negative, match[2]);
|
|
377
|
+
}
|
|
378
|
+
return integerScaleReachable(lo, hi, negative, match[2], match[4] ?? "");
|
|
379
|
+
}
|
|
380
|
+
function integerRestriction(schema) {
|
|
381
|
+
if (schema === true || schema === false) return null;
|
|
382
|
+
let restriction = integerRestrictions.get(schema);
|
|
383
|
+
if (restriction === void 0) {
|
|
384
|
+
restriction = computeIntegerRestriction(schema);
|
|
385
|
+
integerRestrictions.set(schema, restriction);
|
|
386
|
+
}
|
|
387
|
+
return restriction;
|
|
388
|
+
}
|
|
389
|
+
function computeIntegerRestriction(schema) {
|
|
390
|
+
let integerOnly = false;
|
|
391
|
+
if (schema.type !== void 0) {
|
|
392
|
+
const types = Array.isArray(schema.type) ? schema.type.map(String) : [String(schema.type)];
|
|
393
|
+
integerOnly = types.includes("integer") && !types.includes("number");
|
|
394
|
+
} else if (typeof schema.const === "number") {
|
|
395
|
+
integerOnly = Number.isInteger(schema.const);
|
|
396
|
+
} else if (Array.isArray(schema.enum)) {
|
|
397
|
+
const numbers = schema.enum.filter((value) => typeof value === "number");
|
|
398
|
+
integerOnly = numbers.length > 0 && numbers.every((value) => Number.isInteger(value));
|
|
399
|
+
}
|
|
400
|
+
if (!integerOnly) return null;
|
|
401
|
+
let lo = -Infinity;
|
|
402
|
+
let hi = Infinity;
|
|
403
|
+
if (typeof schema.minimum === "number") lo = Math.ceil(schema.minimum);
|
|
404
|
+
if (typeof schema.exclusiveMinimum === "number") lo = Math.max(lo, Math.floor(schema.exclusiveMinimum) + 1);
|
|
405
|
+
if (typeof schema.maximum === "number") hi = Math.floor(schema.maximum);
|
|
406
|
+
if (typeof schema.exclusiveMaximum === "number") hi = Math.min(hi, Math.ceil(schema.exclusiveMaximum) - 1);
|
|
407
|
+
if (!Number.isSafeInteger(lo)) lo = -Infinity;
|
|
408
|
+
if (!Number.isSafeInteger(hi)) hi = Infinity;
|
|
409
|
+
return { lo, hi };
|
|
410
|
+
}
|
|
411
|
+
function integerDigitsReachable(lo, hi, negative, digits) {
|
|
412
|
+
if (digits.length > 16) return negative ? lo === -Infinity : hi === Infinity;
|
|
413
|
+
let low = Number(digits);
|
|
414
|
+
let high = low + 1;
|
|
415
|
+
const extensible = low !== 0;
|
|
416
|
+
for (; ; ) {
|
|
417
|
+
if (negative) {
|
|
418
|
+
if (-(high - 1) <= hi && -low >= lo) return true;
|
|
419
|
+
if (-low < lo) return false;
|
|
420
|
+
} else {
|
|
421
|
+
if (low <= hi && high - 1 >= lo) return true;
|
|
422
|
+
if (low > hi) return false;
|
|
423
|
+
}
|
|
424
|
+
if (!extensible) return false;
|
|
425
|
+
low *= 10;
|
|
426
|
+
high *= 10;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
function integerScaleReachable(lo, hi, negative, digits, exponentPrefix) {
|
|
430
|
+
if (digits.length > 16) return negative ? lo === -Infinity : hi === Infinity;
|
|
431
|
+
const base = Number(digits);
|
|
432
|
+
if (base === 0) return lo <= 0 && 0 <= hi;
|
|
433
|
+
let magnitude = base;
|
|
434
|
+
for (let exponent = 0; exponent <= 999; ++exponent) {
|
|
435
|
+
const value = negative ? -magnitude : magnitude;
|
|
436
|
+
const inRange = (lo === -Infinity || value >= lo) && (hi === Infinity || value <= hi);
|
|
437
|
+
if (inRange && exponentTypeable(exponent, exponentPrefix)) return true;
|
|
438
|
+
if (negative ? lo !== -Infinity && value < lo : hi !== Infinity && value > hi) return false;
|
|
439
|
+
magnitude *= 10;
|
|
440
|
+
}
|
|
441
|
+
return false;
|
|
442
|
+
}
|
|
443
|
+
function exponentTypeable(exponent, prefix) {
|
|
444
|
+
const typed = prefix === "" ? 0 : Number(prefix);
|
|
445
|
+
for (let extra = Math.max(0, 1 - prefix.length); extra <= SCALE_DIGIT_LIMIT - prefix.length; ++extra) {
|
|
446
|
+
const scale = 10 ** extra;
|
|
447
|
+
if (exponent >= typed * scale && exponent < (typed + 1) * scale) return true;
|
|
448
|
+
}
|
|
449
|
+
return false;
|
|
450
|
+
}
|
|
451
|
+
function finishNumber(state) {
|
|
452
|
+
const text = state.text;
|
|
453
|
+
if (!numberComplete(text)) return DEAD;
|
|
454
|
+
return completeValue(state, { kind: "number", value: Number(text), raw: text });
|
|
455
|
+
}
|
|
456
|
+
function completeValue(state, node) {
|
|
457
|
+
if (!validateNode(state.schema, node)) return DEAD;
|
|
458
|
+
if (state.stack.length === 0) return { mode: "done", schema: state.schema, stack: [], guidance: state.guidance };
|
|
459
|
+
const frame = state.stack.at(-1);
|
|
460
|
+
if (frame.kind === "object") {
|
|
461
|
+
if (frame.key === void 0) return DEAD;
|
|
462
|
+
const seen = new Set(frame.seen);
|
|
463
|
+
seen.add(frame.key);
|
|
464
|
+
const entries = [...frame.entries, { key: frame.key, node }];
|
|
465
|
+
if (!partialObjectValid(frame.schema, entries, /* @__PURE__ */ new Set())) return DEAD;
|
|
466
|
+
return {
|
|
467
|
+
mode: "after-value",
|
|
468
|
+
schema: frame.schema,
|
|
469
|
+
stack: replaceTop(state.stack, { kind: "object", schema: frame.schema, seen, entries }),
|
|
470
|
+
guidance: state.guidance
|
|
471
|
+
};
|
|
472
|
+
}
|
|
473
|
+
return {
|
|
474
|
+
mode: "after-value",
|
|
475
|
+
schema: frame.schema,
|
|
476
|
+
stack: replaceTop(state.stack, { ...frame, length: frame.length + 1, items: [...frame.items, node] }),
|
|
477
|
+
guidance: state.guidance
|
|
478
|
+
};
|
|
479
|
+
}
|
|
480
|
+
function afterValue(state, byte) {
|
|
481
|
+
const frame = state.stack.at(-1);
|
|
482
|
+
if (frame === void 0) return DEAD;
|
|
483
|
+
if (frame.kind === "object") {
|
|
484
|
+
if (byte === 125) return closeObject(state);
|
|
485
|
+
if (!objectCanAddProperty(frame)) return DEAD;
|
|
486
|
+
return separatorByte(
|
|
487
|
+
{ ...state, mode: "item-separator", index: 0 },
|
|
488
|
+
byte,
|
|
489
|
+
state.guidance.itemBytes,
|
|
490
|
+
"object-key"
|
|
491
|
+
);
|
|
492
|
+
}
|
|
493
|
+
if (byte === 93) return closeArray(state);
|
|
494
|
+
if (exceedsMaxItems(frame)) return DEAD;
|
|
495
|
+
return separatorByte({ ...state, mode: "item-separator", index: 0 }, byte, state.guidance.itemBytes, "array-value");
|
|
496
|
+
}
|
|
497
|
+
function separatorByte(state, byte, separator, completedMode) {
|
|
498
|
+
const index = state.index ?? 0;
|
|
499
|
+
if (byte !== separator[index]) return DEAD;
|
|
500
|
+
if (index + 1 < separator.length) return { ...state, index: index + 1 };
|
|
501
|
+
const frame = state.stack.at(-1);
|
|
502
|
+
return {
|
|
503
|
+
...state,
|
|
504
|
+
mode: completedMode,
|
|
505
|
+
schema: completedMode === "value" ? topObject(state).childSchema : completedMode === "array-value" && frame?.kind === "array" ? itemSchema(frame.schema, frame.length) : state.schema,
|
|
506
|
+
index: void 0
|
|
507
|
+
};
|
|
508
|
+
}
|
|
509
|
+
function separatorExpects(state, byte) {
|
|
510
|
+
if (state.mode === "colon") return state.guidance.keyBytes[state.index ?? 0] === byte;
|
|
511
|
+
if (state.mode === "item-separator") return state.guidance.itemBytes[state.index ?? 0] === byte;
|
|
512
|
+
return false;
|
|
513
|
+
}
|
|
514
|
+
function closeObject(state) {
|
|
515
|
+
const frame = state.stack.at(-1);
|
|
516
|
+
if (frame?.kind !== "object") return DEAD;
|
|
517
|
+
const value = /* @__PURE__ */ Object.create(null);
|
|
518
|
+
for (const entry of frame.entries) value[entry.key] = entry.node.value;
|
|
519
|
+
return completeValue(
|
|
520
|
+
{ ...state, schema: frame.schema, stack: state.stack.slice(0, -1) },
|
|
521
|
+
{ kind: "object", value, entries: [...frame.entries] }
|
|
522
|
+
);
|
|
523
|
+
}
|
|
524
|
+
function closeArray(state) {
|
|
525
|
+
const frame = state.stack.at(-1);
|
|
526
|
+
if (frame?.kind !== "array") return DEAD;
|
|
527
|
+
return completeValue(
|
|
528
|
+
{ ...state, schema: frame.schema, stack: state.stack.slice(0, -1) },
|
|
529
|
+
{ kind: "array", value: frame.items.map((item) => item.value), items: [...frame.items] }
|
|
530
|
+
);
|
|
531
|
+
}
|
|
532
|
+
function selectSchema(schema, kind) {
|
|
533
|
+
return schemaMayAcceptKind(schema, kind, /* @__PURE__ */ new Set()) ? schema : null;
|
|
534
|
+
}
|
|
535
|
+
function propertySchema(schema, key, entries = []) {
|
|
536
|
+
if (schema === true) return true;
|
|
537
|
+
if (schema === false) return null;
|
|
538
|
+
const schemas = [];
|
|
539
|
+
const direct = directPropertySchema(schema, key);
|
|
540
|
+
if (direct === null) return null;
|
|
541
|
+
if (direct !== true) schemas.push(direct);
|
|
542
|
+
for (const candidate of [schema, ...Array.isArray(schema.allOf) ? schema.allOf.filter(isSchema) : []]) {
|
|
543
|
+
if (candidate === true || candidate === false) continue;
|
|
544
|
+
const branch = selectedConditionalBranch(candidate, entries);
|
|
545
|
+
if (branch === void 0 || branch === true) continue;
|
|
546
|
+
if (branch === false) return null;
|
|
547
|
+
const branchSchema = directPropertySchema(branch, key);
|
|
548
|
+
if (branchSchema === null) return null;
|
|
549
|
+
if (branchSchema !== true) schemas.push(branchSchema);
|
|
550
|
+
}
|
|
551
|
+
if (schemas.length === 0) return true;
|
|
552
|
+
return schemas.length === 1 ? schemas[0] : { allOf: schemas };
|
|
553
|
+
}
|
|
554
|
+
function directPropertySchema(schema, key) {
|
|
555
|
+
const properties = isRecord(schema.properties) ? schema.properties : {};
|
|
556
|
+
const schemas = [];
|
|
557
|
+
if (isSchema(properties[key])) schemas.push(properties[key]);
|
|
558
|
+
if (isRecord(schema.patternProperties)) {
|
|
559
|
+
for (const [pattern, candidate] of Object.entries(schema.patternProperties)) {
|
|
560
|
+
if (new RegExp(pattern, "u").test(key) && isSchema(candidate)) schemas.push(candidate);
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
if (schemas.length === 0) {
|
|
564
|
+
if (schema.additionalProperties === false) return null;
|
|
565
|
+
return isSchema(schema.additionalProperties) ? schema.additionalProperties : true;
|
|
566
|
+
}
|
|
567
|
+
return schemas.length === 1 ? schemas[0] : { allOf: schemas };
|
|
568
|
+
}
|
|
569
|
+
function selectedConditionalBranch(schema, entries) {
|
|
570
|
+
if (!isSchema(schema.if) || schema.if === true || schema.if === false) return void 0;
|
|
571
|
+
const required = Array.isArray(schema.if.required) ? schema.if.required : [];
|
|
572
|
+
if (required.length === 0 || !required.every((key) => entries.some((entry) => entry.key === key))) return void 0;
|
|
573
|
+
const node = objectNode(entries);
|
|
574
|
+
const branch = validateNode(schema.if, node) ? schema.then : schema.else;
|
|
575
|
+
return isSchema(branch) ? branch : void 0;
|
|
576
|
+
}
|
|
577
|
+
function partialObjectValid(schema, entries, seen) {
|
|
578
|
+
if (schema === true) return true;
|
|
579
|
+
if (schema === false) return false;
|
|
580
|
+
if (seen.has(schema)) return true;
|
|
581
|
+
seen.add(schema);
|
|
582
|
+
try {
|
|
583
|
+
for (const entry of entries) {
|
|
584
|
+
const child = directPropertySchema(schema, entry.key);
|
|
585
|
+
if (child === null || child !== true && !validateNode(child, entry.node)) return false;
|
|
586
|
+
}
|
|
587
|
+
if (typeof schema.$ref === "string" && !partialObjectValid(resolveReference(schema, schema.$ref), entries, seen))
|
|
588
|
+
return false;
|
|
589
|
+
if (Array.isArray(schema.allOf) && !schema.allOf.every((child) => !isSchema(child) || partialObjectValid(child, entries, seen)))
|
|
590
|
+
return false;
|
|
591
|
+
const branch = selectedConditionalBranch(schema, entries);
|
|
592
|
+
return branch === void 0 || partialObjectValid(branch, entries, seen);
|
|
593
|
+
} finally {
|
|
594
|
+
seen.delete(schema);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
function objectNode(entries) {
|
|
598
|
+
const value = /* @__PURE__ */ Object.create(null);
|
|
599
|
+
for (const entry of entries) value[entry.key] = entry.node.value;
|
|
600
|
+
return { kind: "object", value, entries: [...entries] };
|
|
601
|
+
}
|
|
602
|
+
function itemSchema(schema, index) {
|
|
603
|
+
if (schema === true || schema === false || hasDeferredStructure(schema)) return schema === false ? false : true;
|
|
604
|
+
if (Array.isArray(schema.prefixItems) && isSchema(schema.prefixItems[index])) return schema.prefixItems[index];
|
|
605
|
+
if (Array.isArray(schema.items)) {
|
|
606
|
+
if (isSchema(schema.items[index])) return schema.items[index];
|
|
607
|
+
return isSchema(schema.additionalItems) ? schema.additionalItems : true;
|
|
608
|
+
}
|
|
609
|
+
return isSchema(schema.items) ? schema.items : true;
|
|
610
|
+
}
|
|
611
|
+
function keyPrefixAllowed(state, bytes) {
|
|
612
|
+
const frame = topObject(state);
|
|
613
|
+
if (frame.schema === true || frame.schema === false) return true;
|
|
614
|
+
const properties = encodedPropertyKeys(frame.schema);
|
|
615
|
+
if (properties === null) return true;
|
|
616
|
+
return properties.some(({ key, bytes: propertyBytes }) => {
|
|
617
|
+
if (frame.seen.has(key) || bytes.length > propertyBytes.length) return false;
|
|
618
|
+
for (let index = 0; index < bytes.length; ++index) {
|
|
619
|
+
if (bytes[index] !== propertyBytes[index]) return false;
|
|
620
|
+
}
|
|
621
|
+
return true;
|
|
622
|
+
});
|
|
623
|
+
}
|
|
624
|
+
function keyEscapeAllowed(state) {
|
|
625
|
+
const frame = topObject(state);
|
|
626
|
+
if (frame.schema === true || frame.schema === false) return true;
|
|
627
|
+
const properties = encodedPropertyKeys(frame.schema);
|
|
628
|
+
if (properties === null) return true;
|
|
629
|
+
if (state.highSurrogate !== void 0) return true;
|
|
630
|
+
const bytes = state.bytes ?? [];
|
|
631
|
+
return properties.some(({ key, bytes: propertyBytes }) => {
|
|
632
|
+
if (frame.seen.has(key) || propertyBytes.length <= bytes.length) return false;
|
|
633
|
+
for (let index = 0; index < bytes.length; ++index) {
|
|
634
|
+
if (bytes[index] !== propertyBytes[index]) return false;
|
|
635
|
+
}
|
|
636
|
+
const next = propertyBytes[bytes.length];
|
|
637
|
+
return next < 32 || next === 34 || next === 92 || next >= 128;
|
|
638
|
+
});
|
|
639
|
+
}
|
|
640
|
+
function objectCanAddProperty(frame) {
|
|
641
|
+
if (frame.schema === true || frame.schema === false) return true;
|
|
642
|
+
const properties = encodedPropertyKeys(frame.schema);
|
|
643
|
+
return properties === null || properties.some(({ key }) => !frame.seen.has(key));
|
|
644
|
+
}
|
|
645
|
+
function stringContentAllowed(schema, bytes, length) {
|
|
646
|
+
const values = finiteStringValues(schema);
|
|
647
|
+
if (values !== null && !values.some(({ bytes: valueBytes }) => {
|
|
648
|
+
if (bytes.length > valueBytes.length) return false;
|
|
649
|
+
for (let index = 0; index < bytes.length; ++index) {
|
|
650
|
+
if (bytes[index] !== valueBytes[index]) return false;
|
|
651
|
+
}
|
|
652
|
+
return true;
|
|
653
|
+
}))
|
|
654
|
+
return false;
|
|
655
|
+
const maximum = directStringMaxLength(schema);
|
|
656
|
+
return maximum === void 0 || length <= maximum;
|
|
657
|
+
}
|
|
658
|
+
function stringEscapeAllowed(state) {
|
|
659
|
+
const values = finiteStringValues(state.schema);
|
|
660
|
+
const maximum = directStringMaxLength(state.schema);
|
|
661
|
+
if (maximum !== void 0 && (state.stringLength ?? 0) >= maximum) return false;
|
|
662
|
+
if (values === null) return true;
|
|
663
|
+
if (state.highSurrogate !== void 0) return true;
|
|
664
|
+
const bytes = state.bytes ?? [];
|
|
665
|
+
return values.some(({ bytes: valueBytes }) => {
|
|
666
|
+
if (valueBytes.length <= bytes.length) return false;
|
|
667
|
+
for (let index = 0; index < bytes.length; ++index) {
|
|
668
|
+
if (bytes[index] !== valueBytes[index]) return false;
|
|
669
|
+
}
|
|
670
|
+
const next = valueBytes[bytes.length];
|
|
671
|
+
return next < 32 || next === 34 || next === 92 || next >= 128;
|
|
672
|
+
});
|
|
673
|
+
}
|
|
674
|
+
function directStringMaxLength(schema) {
|
|
675
|
+
return schema !== true && schema !== false && typeof schema.maxLength === "number" ? schema.maxLength : void 0;
|
|
676
|
+
}
|
|
677
|
+
function finiteStringValues(schema) {
|
|
678
|
+
if (schema === true || schema === false) return null;
|
|
679
|
+
let values = finiteStringCache.get(schema);
|
|
680
|
+
if (values !== void 0) return values;
|
|
681
|
+
let candidates = null;
|
|
682
|
+
if (typeof schema.const === "string") candidates = [schema.const];
|
|
683
|
+
else if (Array.isArray(schema.enum))
|
|
684
|
+
candidates = schema.enum.filter((value) => typeof value === "string");
|
|
685
|
+
values = candidates === null ? null : [...new Set(candidates)].filter((value) => validateNode(schema, { kind: "string", value })).map((value) => ({ value, bytes: encoder.encode(value) }));
|
|
686
|
+
finiteStringCache.set(schema, values);
|
|
687
|
+
return values;
|
|
688
|
+
}
|
|
689
|
+
function unicodeKeyPrefixAllowed(state, hexPrefix) {
|
|
690
|
+
const frame = topObject(state);
|
|
691
|
+
if (frame.schema === true || frame.schema === false) return true;
|
|
692
|
+
const properties = encodedPropertyKeys(frame.schema);
|
|
693
|
+
if (properties === null) return true;
|
|
694
|
+
const prefix = decodeString(state.bytes ?? []);
|
|
695
|
+
if (prefix === null) return true;
|
|
696
|
+
return properties.some(({ key }) => {
|
|
697
|
+
if (frame.seen.has(key) || !key.startsWith(prefix)) return false;
|
|
698
|
+
const offset = prefix.length;
|
|
699
|
+
let codeUnit = key.charCodeAt(offset);
|
|
700
|
+
if (state.highSurrogate !== void 0) {
|
|
701
|
+
if (codeUnit !== state.highSurrogate) return false;
|
|
702
|
+
codeUnit = key.charCodeAt(offset + 1);
|
|
703
|
+
}
|
|
704
|
+
return Number.isInteger(codeUnit) && codeUnit.toString(16).padStart(4, "0").startsWith(hexPrefix.toLowerCase());
|
|
705
|
+
});
|
|
706
|
+
}
|
|
707
|
+
function encodedPropertyKeys(schema) {
|
|
708
|
+
let properties = propertyKeyBytes.get(schema);
|
|
709
|
+
if (properties === void 0) {
|
|
710
|
+
const keys = constrainedPropertyKeys(schema, /* @__PURE__ */ new Set());
|
|
711
|
+
properties = keys === null ? null : [...keys].map((key) => ({ key, bytes: encoder.encode(key) }));
|
|
712
|
+
propertyKeyBytes.set(schema, properties);
|
|
713
|
+
}
|
|
714
|
+
return properties;
|
|
715
|
+
}
|
|
716
|
+
function constrainedPropertyKeys(schema, seen) {
|
|
717
|
+
if (schema === true || schema === false || seen.has(schema)) return null;
|
|
718
|
+
seen.add(schema);
|
|
719
|
+
try {
|
|
720
|
+
const properties = isRecord(schema.properties) ? schema.properties : {};
|
|
721
|
+
let result = schema.additionalProperties === false && !isRecord(schema.patternProperties) ? new Set(Object.keys(properties)) : null;
|
|
722
|
+
if (typeof schema.$ref === "string") {
|
|
723
|
+
result = intersectKeySets(result, constrainedPropertyKeys(resolveReference(schema, schema.$ref), seen));
|
|
724
|
+
}
|
|
725
|
+
if (Array.isArray(schema.allOf)) {
|
|
726
|
+
for (const child of schema.allOf) {
|
|
727
|
+
if (isSchema(child)) result = intersectKeySets(result, constrainedPropertyKeys(child, seen));
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
for (const keyword of ["anyOf", "oneOf"]) {
|
|
731
|
+
if (!Array.isArray(schema[keyword])) continue;
|
|
732
|
+
let union2 = /* @__PURE__ */ new Set();
|
|
733
|
+
for (const child of schema[keyword]) {
|
|
734
|
+
if (!isSchema(child) || !schemaMayAcceptKind(child, "object", /* @__PURE__ */ new Set())) continue;
|
|
735
|
+
const childKeys = constrainedPropertyKeys(child, seen);
|
|
736
|
+
if (childKeys === null) {
|
|
737
|
+
union2 = null;
|
|
738
|
+
break;
|
|
739
|
+
}
|
|
740
|
+
for (const key of childKeys) union2.add(key);
|
|
741
|
+
}
|
|
742
|
+
result = intersectKeySets(result, union2);
|
|
743
|
+
}
|
|
744
|
+
return result;
|
|
745
|
+
} finally {
|
|
746
|
+
seen.delete(schema);
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
function intersectKeySets(left, right) {
|
|
750
|
+
if (left === null) return right;
|
|
751
|
+
if (right === null) return left;
|
|
752
|
+
return new Set([...left].filter((key) => right.has(key)));
|
|
753
|
+
}
|
|
754
|
+
function exceedsMaxItems(frame) {
|
|
755
|
+
return frame.schema !== true && frame.schema !== false && !hasDeferredStructure(frame.schema) && typeof frame.schema.maxItems === "number" ? frame.length >= frame.schema.maxItems : false;
|
|
756
|
+
}
|
|
757
|
+
var schemaContexts = /* @__PURE__ */ new WeakMap();
|
|
758
|
+
function validateNode(schema, node, active = /* @__PURE__ */ new Map()) {
|
|
759
|
+
if (schema === true) return true;
|
|
760
|
+
if (schema === false) return false;
|
|
761
|
+
let nodes = active.get(schema);
|
|
762
|
+
if (nodes?.has(node)) return true;
|
|
763
|
+
if (nodes === void 0) {
|
|
764
|
+
nodes = /* @__PURE__ */ new Set();
|
|
765
|
+
active.set(schema, nodes);
|
|
766
|
+
}
|
|
767
|
+
nodes.add(node);
|
|
768
|
+
try {
|
|
769
|
+
if (typeof schema.$ref === "string" && !validateNode(resolveReference(schema, schema.$ref), node, active))
|
|
770
|
+
return false;
|
|
771
|
+
if (schema.type !== void 0) {
|
|
772
|
+
const types = Array.isArray(schema.type) ? schema.type.map(String) : [String(schema.type)];
|
|
773
|
+
if (!types.some((type) => nodeHasType(node, type))) return false;
|
|
774
|
+
}
|
|
775
|
+
if (schema.const !== void 0 && !nodeEqualsValue(node, schema.const)) return false;
|
|
776
|
+
if (Array.isArray(schema.enum) && !schema.enum.some((value) => nodeEqualsValue(node, value))) return false;
|
|
777
|
+
if (Array.isArray(schema.allOf) && !schema.allOf.every((child) => isSchema(child) && validateNode(child, node, active)))
|
|
778
|
+
return false;
|
|
779
|
+
if (Array.isArray(schema.anyOf) && !schema.anyOf.some((child) => isSchema(child) && validateNode(child, node, active)))
|
|
780
|
+
return false;
|
|
781
|
+
if (Array.isArray(schema.oneOf) && schema.oneOf.filter((child) => isSchema(child) && validateNode(child, node, active)).length !== 1)
|
|
782
|
+
return false;
|
|
783
|
+
if (isSchema(schema.not) && validateNode(schema.not, node, active)) return false;
|
|
784
|
+
if (isSchema(schema.if)) {
|
|
785
|
+
const branch = validateNode(schema.if, node, active) ? schema.then : schema.else;
|
|
786
|
+
if (isSchema(branch) && !validateNode(branch, node, active)) return false;
|
|
787
|
+
}
|
|
788
|
+
if (node.kind === "number" && !validateNumericNode(schema, node)) return false;
|
|
789
|
+
if (node.kind === "string" && !validateStringNode(schema, node.value)) return false;
|
|
790
|
+
if (node.kind === "array" && !validateArrayNode(schema, node, active)) return false;
|
|
791
|
+
if (node.kind === "object" && !validateObjectNode(schema, node, active)) return false;
|
|
792
|
+
return true;
|
|
793
|
+
} finally {
|
|
794
|
+
nodes.delete(node);
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
function validateNumericNode(schema, node) {
|
|
798
|
+
const value = decimal(node.raw);
|
|
799
|
+
if (schema.minimum !== void 0 && compareDecimal(value, decimal(String(schema.minimum))) < 0) return false;
|
|
800
|
+
if (schema.maximum !== void 0 && compareDecimal(value, decimal(String(schema.maximum))) > 0) return false;
|
|
801
|
+
if (schema.exclusiveMinimum !== void 0 && compareDecimal(value, decimal(String(schema.exclusiveMinimum))) <= 0)
|
|
802
|
+
return false;
|
|
803
|
+
if (schema.exclusiveMaximum !== void 0 && compareDecimal(value, decimal(String(schema.exclusiveMaximum))) >= 0)
|
|
804
|
+
return false;
|
|
805
|
+
return schema.multipleOf === void 0 || decimalMultiple(value, decimal(String(schema.multipleOf)));
|
|
806
|
+
}
|
|
807
|
+
function validateStringNode(schema, value) {
|
|
808
|
+
const length = [...value].length;
|
|
809
|
+
if (typeof schema.minLength === "number" && length < schema.minLength) return false;
|
|
810
|
+
if (typeof schema.maxLength === "number" && length > schema.maxLength) return false;
|
|
811
|
+
if (typeof schema.pattern === "string" && !new RegExp(schema.pattern, "u").test(value)) return false;
|
|
812
|
+
return typeof schema.format !== "string" || formatMatches(schema.format, value);
|
|
813
|
+
}
|
|
814
|
+
function validateArrayNode(schema, node, active) {
|
|
815
|
+
if (typeof schema.minItems === "number" && node.items.length < schema.minItems) return false;
|
|
816
|
+
if (typeof schema.maxItems === "number" && node.items.length > schema.maxItems) return false;
|
|
817
|
+
if (schema.uniqueItems === true) {
|
|
818
|
+
for (let index = 0; index < node.items.length; ++index) {
|
|
819
|
+
if (node.items.slice(0, index).some((other) => nodesEqual(other, node.items[index]))) return false;
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
const legacyTuple = Array.isArray(schema.items) ? schema.items : void 0;
|
|
823
|
+
const prefix = (Array.isArray(schema.prefixItems) ? schema.prefixItems : legacyTuple) ?? [];
|
|
824
|
+
for (let index = 0; index < Math.min(prefix.length, node.items.length); ++index) {
|
|
825
|
+
if (!isSchema(prefix[index]) || !validateNode(prefix[index], node.items[index], active)) return false;
|
|
826
|
+
}
|
|
827
|
+
const remaining = legacyTuple ? schema.additionalItems : schema.items;
|
|
828
|
+
if (isSchema(remaining)) {
|
|
829
|
+
for (let index = prefix.length; index < node.items.length; ++index) {
|
|
830
|
+
if (!validateNode(remaining, node.items[index], active)) return false;
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
if (isSchema(schema.contains)) {
|
|
834
|
+
const matches = node.items.filter((item) => validateNode(schema.contains, item, active)).length;
|
|
835
|
+
const minimum = typeof schema.minContains === "number" ? schema.minContains : 1;
|
|
836
|
+
if (matches < minimum || typeof schema.maxContains === "number" && matches > schema.maxContains) return false;
|
|
837
|
+
}
|
|
838
|
+
return true;
|
|
839
|
+
}
|
|
840
|
+
function validateObjectNode(schema, node, active) {
|
|
841
|
+
const entries = new Map(node.entries.map((entry) => [entry.key, entry.node]));
|
|
842
|
+
if (entries.size !== node.entries.length) return false;
|
|
843
|
+
if (typeof schema.minProperties === "number" && entries.size < schema.minProperties) return false;
|
|
844
|
+
if (typeof schema.maxProperties === "number" && entries.size > schema.maxProperties) return false;
|
|
845
|
+
const required = Array.isArray(schema.required) ? schema.required : [];
|
|
846
|
+
if (!required.every((key) => typeof key === "string" && entries.has(key))) return false;
|
|
847
|
+
const properties = isRecord(schema.properties) ? schema.properties : {};
|
|
848
|
+
const patterns = isRecord(schema.patternProperties) ? schema.patternProperties : {};
|
|
849
|
+
for (const [key, child] of entries) {
|
|
850
|
+
if (isSchema(schema.propertyNames) && !validateNode(schema.propertyNames, { kind: "string", value: key }, active))
|
|
851
|
+
return false;
|
|
852
|
+
let matched = false;
|
|
853
|
+
if (isSchema(properties[key])) {
|
|
854
|
+
matched = true;
|
|
855
|
+
if (!validateNode(properties[key], child, active)) return false;
|
|
856
|
+
}
|
|
857
|
+
for (const [pattern, patternSchema] of Object.entries(patterns)) {
|
|
858
|
+
if (new RegExp(pattern, "u").test(key)) {
|
|
859
|
+
matched = true;
|
|
860
|
+
if (!isSchema(patternSchema) || !validateNode(patternSchema, child, active)) return false;
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
if (!matched) {
|
|
864
|
+
if (schema.additionalProperties === false) return false;
|
|
865
|
+
if (isSchema(schema.additionalProperties) && !validateNode(schema.additionalProperties, child, active))
|
|
866
|
+
return false;
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
const dependentRequired = isRecord(schema.dependentRequired) ? schema.dependentRequired : {};
|
|
870
|
+
for (const [key, dependencies2] of Object.entries(dependentRequired)) {
|
|
871
|
+
if (entries.has(key) && Array.isArray(dependencies2) && !dependencies2.every((dependency) => typeof dependency === "string" && entries.has(dependency)))
|
|
872
|
+
return false;
|
|
873
|
+
}
|
|
874
|
+
const dependentSchemas = isRecord(schema.dependentSchemas) ? schema.dependentSchemas : {};
|
|
875
|
+
for (const [key, dependency] of Object.entries(dependentSchemas)) {
|
|
876
|
+
if (entries.has(key) && (!isSchema(dependency) || !validateNode(dependency, node, active))) return false;
|
|
877
|
+
}
|
|
878
|
+
const dependencies = isRecord(schema.dependencies) ? schema.dependencies : {};
|
|
879
|
+
for (const [key, dependency] of Object.entries(dependencies)) {
|
|
880
|
+
if (!entries.has(key)) continue;
|
|
881
|
+
if (Array.isArray(dependency)) {
|
|
882
|
+
if (!dependency.every((requiredKey) => typeof requiredKey === "string" && entries.has(requiredKey)))
|
|
883
|
+
return false;
|
|
884
|
+
} else if (!isSchema(dependency) || !validateNode(dependency, node, active)) return false;
|
|
885
|
+
}
|
|
886
|
+
return true;
|
|
887
|
+
}
|
|
888
|
+
function nodeHasType(node, type) {
|
|
889
|
+
if (type === "integer") return node.kind === "number" && decimalInteger(decimal(node.raw));
|
|
890
|
+
if (type === "number") return node.kind === "number";
|
|
891
|
+
return node.kind === type;
|
|
892
|
+
}
|
|
893
|
+
function nodeEqualsValue(node, value) {
|
|
894
|
+
if (node.kind === "number")
|
|
895
|
+
return typeof value === "number" && compareDecimal(decimal(node.raw), decimal(String(value))) === 0;
|
|
896
|
+
if (node.kind === "array")
|
|
897
|
+
return Array.isArray(value) && node.items.length === value.length && node.items.every((item, index) => nodeEqualsValue(item, value[index]));
|
|
898
|
+
if (node.kind === "object") {
|
|
899
|
+
if (!isRecord(value)) return false;
|
|
900
|
+
const entries = new Map(node.entries.map((entry) => [entry.key, entry.node]));
|
|
901
|
+
return entries.size === Object.keys(value).length && Object.entries(value).every(([key, child]) => entries.has(key) && nodeEqualsValue(entries.get(key), child));
|
|
902
|
+
}
|
|
903
|
+
return Object.is(node.value, value);
|
|
904
|
+
}
|
|
905
|
+
function nodesEqual(left, right) {
|
|
906
|
+
if (left.kind !== right.kind) return false;
|
|
907
|
+
if (left.kind === "number" && right.kind === "number")
|
|
908
|
+
return compareDecimal(decimal(left.raw), decimal(right.raw)) === 0;
|
|
909
|
+
if (left.kind === "array" && right.kind === "array")
|
|
910
|
+
return left.items.length === right.items.length && left.items.every((item, index) => nodesEqual(item, right.items[index]));
|
|
911
|
+
if (left.kind === "object" && right.kind === "object") {
|
|
912
|
+
const rightEntries = new Map(right.entries.map((entry) => [entry.key, entry.node]));
|
|
913
|
+
return left.entries.length === rightEntries.size && left.entries.every(
|
|
914
|
+
(entry) => rightEntries.has(entry.key) && nodesEqual(entry.node, rightEntries.get(entry.key))
|
|
915
|
+
);
|
|
916
|
+
}
|
|
917
|
+
return Object.is(left.value, right.value);
|
|
918
|
+
}
|
|
919
|
+
function decimal(raw) {
|
|
920
|
+
const match = /^(-?)(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/.exec(raw);
|
|
921
|
+
const fraction = match[3] ?? "";
|
|
922
|
+
let digits = `${match[2]}${fraction}`.replace(/^0+/, "");
|
|
923
|
+
if (digits === "") return { coefficient: 0n, exponent: 0n };
|
|
924
|
+
const trailing = /0+$/.exec(digits)?.[0].length ?? 0;
|
|
925
|
+
if (trailing > 0) digits = digits.slice(0, -trailing);
|
|
926
|
+
return {
|
|
927
|
+
coefficient: BigInt(`${match[1]}${digits}`),
|
|
928
|
+
exponent: BigInt(match[4] ?? "0") - BigInt(fraction.length) + BigInt(trailing)
|
|
929
|
+
};
|
|
930
|
+
}
|
|
931
|
+
function compareDecimal(left, right) {
|
|
932
|
+
if (left.coefficient === right.coefficient && left.exponent === right.exponent) return 0;
|
|
933
|
+
if (left.coefficient === 0n) return right.coefficient < 0n ? 1 : -1;
|
|
934
|
+
if (right.coefficient === 0n) return left.coefficient < 0n ? -1 : 1;
|
|
935
|
+
if (left.coefficient < 0n && right.coefficient >= 0n) return -1;
|
|
936
|
+
if (left.coefficient >= 0n && right.coefficient < 0n) return 1;
|
|
937
|
+
const negative = left.coefficient < 0n;
|
|
938
|
+
const leftAbsolute = left.coefficient < 0n ? -left.coefficient : left.coefficient;
|
|
939
|
+
const rightAbsolute = right.coefficient < 0n ? -right.coefficient : right.coefficient;
|
|
940
|
+
const leftMagnitude = BigInt(leftAbsolute.toString().length) + left.exponent;
|
|
941
|
+
const rightMagnitude = BigInt(rightAbsolute.toString().length) + right.exponent;
|
|
942
|
+
if (leftMagnitude !== rightMagnitude) {
|
|
943
|
+
const comparison = leftMagnitude < rightMagnitude ? -1 : 1;
|
|
944
|
+
return negative ? -comparison : comparison;
|
|
945
|
+
}
|
|
946
|
+
const exponent = left.exponent < right.exponent ? left.exponent : right.exponent;
|
|
947
|
+
const scaledLeft = left.coefficient * 10n ** (left.exponent - exponent);
|
|
948
|
+
const scaledRight = right.coefficient * 10n ** (right.exponent - exponent);
|
|
949
|
+
return scaledLeft < scaledRight ? -1 : scaledLeft > scaledRight ? 1 : 0;
|
|
950
|
+
}
|
|
951
|
+
function decimalInteger(value) {
|
|
952
|
+
return value.coefficient === 0n || value.exponent >= 0n;
|
|
953
|
+
}
|
|
954
|
+
function decimalMultiple(value, divisor) {
|
|
955
|
+
if (divisor.coefficient === 0n) return false;
|
|
956
|
+
if (value.coefficient === 0n) return true;
|
|
957
|
+
const numerator = value.coefficient < 0n ? -value.coefficient : value.coefficient;
|
|
958
|
+
const denominator = divisor.coefficient < 0n ? -divisor.coefficient : divisor.coefficient;
|
|
959
|
+
const exponent = value.exponent - divisor.exponent;
|
|
960
|
+
if (exponent >= 0n) {
|
|
961
|
+
return numerator % denominator * modularPower(10n, exponent, denominator) % denominator === 0n;
|
|
962
|
+
}
|
|
963
|
+
const places = -exponent;
|
|
964
|
+
if (places >= BigInt(numerator.toString().length)) return false;
|
|
965
|
+
return numerator % (denominator * 10n ** places) === 0n;
|
|
966
|
+
}
|
|
967
|
+
function modularPower(base, exponent, modulus) {
|
|
968
|
+
if (modulus === 1n) return 0n;
|
|
969
|
+
let result = 1n;
|
|
970
|
+
base %= modulus;
|
|
971
|
+
while (exponent > 0n) {
|
|
972
|
+
if (exponent & 1n) result = result * base % modulus;
|
|
973
|
+
base = base * base % modulus;
|
|
974
|
+
exponent >>= 1n;
|
|
975
|
+
}
|
|
976
|
+
return result;
|
|
977
|
+
}
|
|
978
|
+
function hasDeferredStructure(schema) {
|
|
979
|
+
return schema !== true && schema !== false && ["$ref", "allOf", "anyOf", "oneOf", "not", "if"].some((key) => schema[key] !== void 0);
|
|
980
|
+
}
|
|
981
|
+
function schemaMayAcceptKind(schema, kind, seen) {
|
|
982
|
+
if (schema === true) return true;
|
|
983
|
+
if (schema === false) return false;
|
|
984
|
+
if (seen.has(schema)) return true;
|
|
985
|
+
seen.add(schema);
|
|
986
|
+
if (typeof schema.$ref === "string" && !schemaMayAcceptKind(resolveReference(schema, schema.$ref), kind, seen))
|
|
987
|
+
return false;
|
|
988
|
+
if (!allowsKind(schema, kind)) return false;
|
|
989
|
+
if (Array.isArray(schema.allOf) && !schema.allOf.every((child) => isSchema(child) && schemaMayAcceptKind(child, kind, seen)))
|
|
990
|
+
return false;
|
|
991
|
+
if (Array.isArray(schema.anyOf) && !schema.anyOf.some((child) => isSchema(child) && schemaMayAcceptKind(child, kind, seen)))
|
|
992
|
+
return false;
|
|
993
|
+
if (Array.isArray(schema.oneOf) && !schema.oneOf.some((child) => isSchema(child) && schemaMayAcceptKind(child, kind, seen)))
|
|
994
|
+
return false;
|
|
995
|
+
return true;
|
|
996
|
+
}
|
|
997
|
+
function resolveReference(owner, reference) {
|
|
998
|
+
if (!reference.startsWith("#"))
|
|
999
|
+
throw new TypeError(`External JSON Schema reference ${JSON.stringify(reference)} is unsupported.`);
|
|
1000
|
+
const root = schemaContexts.get(owner);
|
|
1001
|
+
if (root === void 0) throw new Error("Missing JSON Schema context.");
|
|
1002
|
+
if (reference === "#") return root;
|
|
1003
|
+
if (!reference.startsWith("#/")) {
|
|
1004
|
+
throw new TypeError(`JSON Schema reference ${JSON.stringify(reference)} must contain a JSON Pointer.`);
|
|
1005
|
+
}
|
|
1006
|
+
let current = root;
|
|
1007
|
+
for (const encoded of reference.slice(2).split("/")) {
|
|
1008
|
+
const key = decodeURIComponent(encoded).replace(/~1/g, "/").replace(/~0/g, "~");
|
|
1009
|
+
if (!isRecord(current) || !(key in current))
|
|
1010
|
+
throw new TypeError(`JSON Schema reference ${JSON.stringify(reference)} does not resolve.`);
|
|
1011
|
+
current = current[key];
|
|
1012
|
+
}
|
|
1013
|
+
if (!isSchema(current))
|
|
1014
|
+
throw new TypeError(`JSON Schema reference ${JSON.stringify(reference)} does not resolve to a schema.`);
|
|
1015
|
+
return current;
|
|
1016
|
+
}
|
|
1017
|
+
function registerSchemaContext(schema, root, seen = /* @__PURE__ */ new Set()) {
|
|
1018
|
+
if (!isRecord(schema) || seen.has(schema)) return;
|
|
1019
|
+
seen.add(schema);
|
|
1020
|
+
schemaContexts.set(schema, root);
|
|
1021
|
+
for (const child of schemaChildren(schema)) registerSchemaContext(child, root, seen);
|
|
1022
|
+
}
|
|
1023
|
+
function schemaChildren(schema) {
|
|
1024
|
+
const result = [];
|
|
1025
|
+
for (const key of [
|
|
1026
|
+
"not",
|
|
1027
|
+
"if",
|
|
1028
|
+
"then",
|
|
1029
|
+
"else",
|
|
1030
|
+
"contains",
|
|
1031
|
+
"propertyNames",
|
|
1032
|
+
"additionalProperties",
|
|
1033
|
+
"additionalItems"
|
|
1034
|
+
]) {
|
|
1035
|
+
if (isSchema(schema[key])) result.push(schema[key]);
|
|
1036
|
+
}
|
|
1037
|
+
if (isSchema(schema.items)) result.push(schema.items);
|
|
1038
|
+
if (Array.isArray(schema.items)) result.push(...schema.items.filter(isSchema));
|
|
1039
|
+
for (const key of ["prefixItems", "allOf", "anyOf", "oneOf"]) {
|
|
1040
|
+
if (Array.isArray(schema[key])) result.push(...schema[key].filter(isSchema));
|
|
1041
|
+
}
|
|
1042
|
+
for (const key of ["properties", "patternProperties", "dependentSchemas", "$defs", "definitions"]) {
|
|
1043
|
+
if (isRecord(schema[key]))
|
|
1044
|
+
result.push(...Object.values(schema[key]).filter(isSchema));
|
|
1045
|
+
}
|
|
1046
|
+
if (isRecord(schema.dependencies)) {
|
|
1047
|
+
result.push(...Object.values(schema.dependencies).filter(isSchema));
|
|
1048
|
+
}
|
|
1049
|
+
return result;
|
|
1050
|
+
}
|
|
1051
|
+
function checkReferences(schema, seen = /* @__PURE__ */ new Set()) {
|
|
1052
|
+
if (!isRecord(schema) || seen.has(schema)) return;
|
|
1053
|
+
seen.add(schema);
|
|
1054
|
+
if (typeof schema.$ref === "string") resolveReference(schema, schema.$ref);
|
|
1055
|
+
for (const child of schemaChildren(schema)) checkReferences(child, seen);
|
|
1056
|
+
}
|
|
1057
|
+
function assertPattern(value, path) {
|
|
1058
|
+
if (typeof value !== "string") throw new TypeError(`${path} must be a string.`);
|
|
1059
|
+
try {
|
|
1060
|
+
new RegExp(value, "u");
|
|
1061
|
+
} catch {
|
|
1062
|
+
throw new TypeError(`${path} must be a valid Unicode RegExp.`);
|
|
1063
|
+
}
|
|
1064
|
+
}
|
|
1065
|
+
function guidanceFrom(schema) {
|
|
1066
|
+
if (!isRecord(schema) || schema["x-guidance"] === void 0) return DEFAULT_GUIDANCE;
|
|
1067
|
+
if (!isRecord(schema["x-guidance"])) throw new TypeError("x-guidance must be an object.");
|
|
1068
|
+
const source = schema["x-guidance"];
|
|
1069
|
+
for (const key of Object.keys(source)) {
|
|
1070
|
+
if (!["item_separator", "key_separator", "whitespace_flexible"].includes(key)) {
|
|
1071
|
+
throw new TypeError(`Unsupported x-guidance option ${JSON.stringify(key)}.`);
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
const itemSeparator = source.item_separator ?? ",";
|
|
1075
|
+
const keySeparator = source.key_separator ?? ":";
|
|
1076
|
+
const whitespaceFlexible = source.whitespace_flexible ?? true;
|
|
1077
|
+
if (typeof itemSeparator !== "string" || itemSeparator.length === 0) {
|
|
1078
|
+
throw new TypeError("x-guidance.item_separator must be a non-empty string.");
|
|
1079
|
+
}
|
|
1080
|
+
if (typeof keySeparator !== "string" || keySeparator.length === 0) {
|
|
1081
|
+
throw new TypeError("x-guidance.key_separator must be a non-empty string.");
|
|
1082
|
+
}
|
|
1083
|
+
if (typeof whitespaceFlexible !== "boolean")
|
|
1084
|
+
throw new TypeError("x-guidance.whitespace_flexible must be a boolean.");
|
|
1085
|
+
return {
|
|
1086
|
+
itemSeparator,
|
|
1087
|
+
keySeparator,
|
|
1088
|
+
itemBytes: encoder.encode(itemSeparator),
|
|
1089
|
+
keyBytes: encoder.encode(keySeparator),
|
|
1090
|
+
whitespaceFlexible
|
|
1091
|
+
};
|
|
1092
|
+
}
|
|
1093
|
+
function formatMatches(format, value) {
|
|
1094
|
+
switch (format) {
|
|
1095
|
+
case "date":
|
|
1096
|
+
return validDate(value);
|
|
1097
|
+
case "time":
|
|
1098
|
+
return validTime(value);
|
|
1099
|
+
case "date-time": {
|
|
1100
|
+
const separator = value.search(/[Tt]/);
|
|
1101
|
+
return separator > 0 && validDate(value.slice(0, separator)) && validTime(value.slice(separator + 1));
|
|
1102
|
+
}
|
|
1103
|
+
case "duration":
|
|
1104
|
+
return /^(?:P\d+W|P(?=\d|T\d)(?:\d+Y)?(?:\d+M)?(?:\d+D)?(?:T(?=\d)(?:\d+H)?(?:\d+M)?(?:\d+S)?)?)$/.test(
|
|
1105
|
+
value
|
|
1106
|
+
);
|
|
1107
|
+
case "email": {
|
|
1108
|
+
const at = value.lastIndexOf("@");
|
|
1109
|
+
if (at <= 0 || at === value.length - 1) return false;
|
|
1110
|
+
const local = value.slice(0, at);
|
|
1111
|
+
const domain = value.slice(at + 1);
|
|
1112
|
+
if (local.startsWith(".") || local.endsWith(".") || local.includes("..") || !/^[A-Za-z0-9!#$%&'*+\-/=?^_`{|}~.]+$/.test(local))
|
|
1113
|
+
return false;
|
|
1114
|
+
return /^\[(.+)\]$/.test(domain) ? validIpv4(domain.slice(1, -1)) : validHostname(domain);
|
|
1115
|
+
}
|
|
1116
|
+
case "hostname":
|
|
1117
|
+
return validHostname(value);
|
|
1118
|
+
case "ipv4":
|
|
1119
|
+
return validIpv4(value);
|
|
1120
|
+
case "ipv6":
|
|
1121
|
+
return validIpv6(value);
|
|
1122
|
+
case "uuid":
|
|
1123
|
+
return /^[\da-f]{8}(?:-[\da-f]{4}){3}-[\da-f]{12}$/i.test(value);
|
|
1124
|
+
case "uri":
|
|
1125
|
+
return validUriText(value, true);
|
|
1126
|
+
case "uri-reference":
|
|
1127
|
+
return validUriText(value, false);
|
|
1128
|
+
case "regex":
|
|
1129
|
+
try {
|
|
1130
|
+
new RegExp(value, "u");
|
|
1131
|
+
return true;
|
|
1132
|
+
} catch {
|
|
1133
|
+
return false;
|
|
1134
|
+
}
|
|
1135
|
+
case "json-pointer":
|
|
1136
|
+
return /^(?:\/(?:[^~/]|~[01])*)*$/u.test(value);
|
|
1137
|
+
case "relative-json-pointer":
|
|
1138
|
+
return /^(?:0|[1-9]\d*)(?:#|(?:\/(?:[^~/]|~[01])*)*)$/u.test(value);
|
|
1139
|
+
default:
|
|
1140
|
+
return true;
|
|
1141
|
+
}
|
|
1142
|
+
}
|
|
1143
|
+
function validDate(value) {
|
|
1144
|
+
const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value);
|
|
1145
|
+
if (!match) return false;
|
|
1146
|
+
const year = Number(match[1]);
|
|
1147
|
+
const month = Number(match[2]);
|
|
1148
|
+
const day = Number(match[3]);
|
|
1149
|
+
const days = [
|
|
1150
|
+
31,
|
|
1151
|
+
year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0) ? 29 : 28,
|
|
1152
|
+
31,
|
|
1153
|
+
30,
|
|
1154
|
+
31,
|
|
1155
|
+
30,
|
|
1156
|
+
31,
|
|
1157
|
+
31,
|
|
1158
|
+
30,
|
|
1159
|
+
31,
|
|
1160
|
+
30,
|
|
1161
|
+
31
|
|
1162
|
+
];
|
|
1163
|
+
return month >= 1 && month <= 12 && day >= 1 && day <= days[month - 1];
|
|
1164
|
+
}
|
|
1165
|
+
function validTime(value) {
|
|
1166
|
+
const match = /^(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:[Zz]|[+-](\d{2}):(\d{2}))$/.exec(value);
|
|
1167
|
+
if (!match) return false;
|
|
1168
|
+
const hour = Number(match[1]);
|
|
1169
|
+
const minute = Number(match[2]);
|
|
1170
|
+
const second = Number(match[3]);
|
|
1171
|
+
if (hour > 23 || minute > 59 || second > 60 || second === 60 && (hour !== 23 || minute !== 59)) return false;
|
|
1172
|
+
return match[4] === void 0 || Number(match[4]) <= 23 && Number(match[5]) <= 59;
|
|
1173
|
+
}
|
|
1174
|
+
function validHostname(value) {
|
|
1175
|
+
return value.length > 0 && value.length <= 253 && !value.endsWith(".") && value.split(".").every((label) => /^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?$/.test(label));
|
|
1176
|
+
}
|
|
1177
|
+
function validIpv4(value) {
|
|
1178
|
+
const parts = value.split(".");
|
|
1179
|
+
return parts.length === 4 && parts.every((part) => /^(?:0|[1-9]\d{0,2})$/.test(part) && Number(part) <= 255);
|
|
1180
|
+
}
|
|
1181
|
+
function validIpv6(value) {
|
|
1182
|
+
if (value.includes(":::") || value.split("::").length > 2) return false;
|
|
1183
|
+
const compressed = value.includes("::");
|
|
1184
|
+
const groups = value.split(":").filter(Boolean);
|
|
1185
|
+
if (!groups.every(
|
|
1186
|
+
(group, index) => group.includes(".") ? index === groups.length - 1 && validIpv4(group) : /^[\da-f]{1,4}$/i.test(group)
|
|
1187
|
+
))
|
|
1188
|
+
return false;
|
|
1189
|
+
const count = groups.reduce((total, group) => total + (group.includes(".") ? 2 : 1), 0);
|
|
1190
|
+
return compressed ? count < 8 : count === 8;
|
|
1191
|
+
}
|
|
1192
|
+
function validUriText(value, absolute) {
|
|
1193
|
+
if (/[\s\\<>^`{|}\0]/.test(value) || /%(?![\da-f]{2})/i.test(value)) return false;
|
|
1194
|
+
let rest = value;
|
|
1195
|
+
const scheme = /^[A-Za-z][A-Za-z0-9+.-]*:/.exec(rest);
|
|
1196
|
+
if (absolute && !scheme) return false;
|
|
1197
|
+
if (scheme) rest = rest.slice(scheme[0].length);
|
|
1198
|
+
const fragment = rest.indexOf("#");
|
|
1199
|
+
if (fragment !== -1) {
|
|
1200
|
+
if (!uriQuery.test(rest.slice(fragment + 1))) return false;
|
|
1201
|
+
rest = rest.slice(0, fragment);
|
|
1202
|
+
}
|
|
1203
|
+
const query = rest.indexOf("?");
|
|
1204
|
+
if (query !== -1) {
|
|
1205
|
+
if (!uriQuery.test(rest.slice(query + 1))) return false;
|
|
1206
|
+
rest = rest.slice(0, query);
|
|
1207
|
+
}
|
|
1208
|
+
if (rest.startsWith("//")) {
|
|
1209
|
+
const slash = rest.indexOf("/", 2);
|
|
1210
|
+
const authority = rest.slice(2, slash === -1 ? void 0 : slash);
|
|
1211
|
+
const path = slash === -1 ? "" : rest.slice(slash);
|
|
1212
|
+
return validAuthority(authority) && uriPath.test(path);
|
|
1213
|
+
}
|
|
1214
|
+
if (!uriPath.test(rest)) return false;
|
|
1215
|
+
if (scheme || rest === "" || rest.startsWith("/")) return true;
|
|
1216
|
+
return !rest.slice(0, rest.indexOf("/") === -1 ? void 0 : rest.indexOf("/")).includes(":");
|
|
1217
|
+
}
|
|
1218
|
+
var uriEncoded = "%[\\da-fA-F]{2}";
|
|
1219
|
+
var uriPchar = `(?:[A-Za-z0-9._~!$&'()*+,;=:@-]|${uriEncoded})`;
|
|
1220
|
+
var uriPath = new RegExp(`^(?:${uriPchar}|/)*$`);
|
|
1221
|
+
var uriQuery = new RegExp(`^(?:${uriPchar}|[/?])*$`);
|
|
1222
|
+
var uriRegName = new RegExp(`^(?:[A-Za-z0-9._~!$&'()*+,;=-]|${uriEncoded})*$`);
|
|
1223
|
+
var uriUserInfo = new RegExp(`^(?:[A-Za-z0-9._~!$&'()*+,;=:-]|${uriEncoded})*$`);
|
|
1224
|
+
function validAuthority(authority) {
|
|
1225
|
+
const at = authority.lastIndexOf("@");
|
|
1226
|
+
if (at !== -1) {
|
|
1227
|
+
if (authority.indexOf("@") !== at || !uriUserInfo.test(authority.slice(0, at))) return false;
|
|
1228
|
+
authority = authority.slice(at + 1);
|
|
1229
|
+
}
|
|
1230
|
+
if (authority.startsWith("[")) {
|
|
1231
|
+
const close = authority.indexOf("]");
|
|
1232
|
+
if (close === -1) return false;
|
|
1233
|
+
const host2 = authority.slice(1, close);
|
|
1234
|
+
const suffix = authority.slice(close + 1);
|
|
1235
|
+
return (validIpv6(host2) || /^v[\da-f]+\.[A-Za-z0-9._~!$&'()*+,;=:-]+$/i.test(host2)) && (suffix === "" || /^:\d*$/.test(suffix));
|
|
1236
|
+
}
|
|
1237
|
+
const colon = authority.lastIndexOf(":");
|
|
1238
|
+
let host = authority;
|
|
1239
|
+
if (colon !== -1) {
|
|
1240
|
+
if (authority.indexOf(":") !== colon || !/^\d*$/.test(authority.slice(colon + 1))) return false;
|
|
1241
|
+
host = authority.slice(0, colon);
|
|
1242
|
+
}
|
|
1243
|
+
return uriRegName.test(host);
|
|
1244
|
+
}
|
|
1245
|
+
function checkSchema(schema, path) {
|
|
1246
|
+
if (typeof schema === "boolean") return;
|
|
1247
|
+
if (!isRecord(schema)) throw new TypeError(`${path} must be a boolean or object schema.`);
|
|
1248
|
+
const supported = /* @__PURE__ */ new Set([
|
|
1249
|
+
"type",
|
|
1250
|
+
"const",
|
|
1251
|
+
"enum",
|
|
1252
|
+
"properties",
|
|
1253
|
+
"required",
|
|
1254
|
+
"additionalProperties",
|
|
1255
|
+
"minProperties",
|
|
1256
|
+
"maxProperties",
|
|
1257
|
+
"items",
|
|
1258
|
+
"prefixItems",
|
|
1259
|
+
"minItems",
|
|
1260
|
+
"maxItems",
|
|
1261
|
+
"uniqueItems",
|
|
1262
|
+
"minLength",
|
|
1263
|
+
"maxLength",
|
|
1264
|
+
"pattern",
|
|
1265
|
+
"format",
|
|
1266
|
+
"minimum",
|
|
1267
|
+
"maximum",
|
|
1268
|
+
"exclusiveMinimum",
|
|
1269
|
+
"exclusiveMaximum",
|
|
1270
|
+
"multipleOf",
|
|
1271
|
+
"contains",
|
|
1272
|
+
"minContains",
|
|
1273
|
+
"maxContains",
|
|
1274
|
+
"patternProperties",
|
|
1275
|
+
"propertyNames",
|
|
1276
|
+
"dependentRequired",
|
|
1277
|
+
"dependentSchemas",
|
|
1278
|
+
"dependencies",
|
|
1279
|
+
"anyOf",
|
|
1280
|
+
"allOf",
|
|
1281
|
+
"oneOf",
|
|
1282
|
+
"not",
|
|
1283
|
+
"if",
|
|
1284
|
+
"then",
|
|
1285
|
+
"else",
|
|
1286
|
+
"$ref",
|
|
1287
|
+
"$defs",
|
|
1288
|
+
"definitions",
|
|
1289
|
+
"additionalItems",
|
|
1290
|
+
"x-guidance",
|
|
1291
|
+
"title",
|
|
1292
|
+
"description",
|
|
1293
|
+
"$schema",
|
|
1294
|
+
"$id",
|
|
1295
|
+
"$comment",
|
|
1296
|
+
"default",
|
|
1297
|
+
"examples",
|
|
1298
|
+
"deprecated",
|
|
1299
|
+
"readOnly",
|
|
1300
|
+
"writeOnly",
|
|
1301
|
+
"contentEncoding",
|
|
1302
|
+
"contentMediaType",
|
|
1303
|
+
"contentSchema"
|
|
1304
|
+
]);
|
|
1305
|
+
for (const key of Object.keys(schema)) {
|
|
1306
|
+
if (!supported.has(key))
|
|
1307
|
+
throw new TypeError(`${path}: unsupported JSON Schema keyword ${JSON.stringify(key)}.`);
|
|
1308
|
+
}
|
|
1309
|
+
const configuredTypes = schema.type === void 0 ? [] : Array.isArray(schema.type) ? schema.type : [schema.type];
|
|
1310
|
+
if (configuredTypes.some(
|
|
1311
|
+
(type) => !["null", "boolean", "number", "integer", "string", "array", "object"].includes(String(type))
|
|
1312
|
+
)) {
|
|
1313
|
+
throw new TypeError(`${path}.type contains an unsupported JSON type.`);
|
|
1314
|
+
}
|
|
1315
|
+
for (const keyword of [
|
|
1316
|
+
"not",
|
|
1317
|
+
"if",
|
|
1318
|
+
"then",
|
|
1319
|
+
"else",
|
|
1320
|
+
"contains",
|
|
1321
|
+
"propertyNames",
|
|
1322
|
+
"additionalProperties",
|
|
1323
|
+
"additionalItems"
|
|
1324
|
+
]) {
|
|
1325
|
+
if (schema[keyword] !== void 0 && !isSchema(schema[keyword])) {
|
|
1326
|
+
throw new TypeError(`${path}.${keyword} must be a boolean or object schema.`);
|
|
1327
|
+
}
|
|
1328
|
+
}
|
|
1329
|
+
for (const keyword of ["allOf", "anyOf", "oneOf", "prefixItems"]) {
|
|
1330
|
+
const value = schema[keyword];
|
|
1331
|
+
if (value !== void 0 && (!Array.isArray(value) || keyword !== "prefixItems" && value.length === 0 || value.some((child) => !isSchema(child)))) {
|
|
1332
|
+
throw new TypeError(`${path}.${keyword} must be an array of schemas.`);
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
for (const keyword of ["properties", "patternProperties", "dependentSchemas", "$defs", "definitions"]) {
|
|
1336
|
+
const value = schema[keyword];
|
|
1337
|
+
if (value !== void 0 && (!isRecord(value) || Object.values(value).some((child) => !isSchema(child)))) {
|
|
1338
|
+
throw new TypeError(`${path}.${keyword} must be an object containing schemas.`);
|
|
1339
|
+
}
|
|
1340
|
+
}
|
|
1341
|
+
if (schema.items !== void 0 && !isSchema(schema.items) && !(Array.isArray(schema.items) && schema.items.every(isSchema))) {
|
|
1342
|
+
throw new TypeError(`${path}.items must be a schema or array of schemas.`);
|
|
1343
|
+
}
|
|
1344
|
+
if (schema.required !== void 0 && (!Array.isArray(schema.required) || schema.required.some((key) => typeof key !== "string") || new Set(schema.required).size !== schema.required.length)) {
|
|
1345
|
+
throw new TypeError(`${path}.required must be an array of unique strings.`);
|
|
1346
|
+
}
|
|
1347
|
+
if (schema.dependentRequired !== void 0) {
|
|
1348
|
+
if (!isRecord(schema.dependentRequired)) throw new TypeError(`${path}.dependentRequired must be an object.`);
|
|
1349
|
+
for (const dependency of Object.values(schema.dependentRequired)) {
|
|
1350
|
+
if (!Array.isArray(dependency) || dependency.some((key) => typeof key !== "string") || new Set(dependency).size !== dependency.length) {
|
|
1351
|
+
throw new TypeError(`${path}.dependentRequired values must be arrays of unique strings.`);
|
|
1352
|
+
}
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
if (schema.dependencies !== void 0 && !isRecord(schema.dependencies)) {
|
|
1356
|
+
throw new TypeError(`${path}.dependencies must be an object.`);
|
|
1357
|
+
}
|
|
1358
|
+
if (schema.$ref !== void 0 && typeof schema.$ref !== "string")
|
|
1359
|
+
throw new TypeError(`${path}.$ref must be a string.`);
|
|
1360
|
+
if (schema.enum !== void 0 && (!Array.isArray(schema.enum) || schema.enum.length === 0)) {
|
|
1361
|
+
throw new TypeError(`${path}.enum must be a non-empty array.`);
|
|
1362
|
+
}
|
|
1363
|
+
for (const keyword of ["minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum", "multipleOf"]) {
|
|
1364
|
+
const value = schema[keyword];
|
|
1365
|
+
if (value !== void 0 && (typeof value !== "number" || !Number.isFinite(value) || keyword === "multipleOf" && value <= 0)) {
|
|
1366
|
+
throw new TypeError(
|
|
1367
|
+
`${path}.${keyword} must be ${keyword === "multipleOf" ? "a positive" : "a finite"} number.`
|
|
1368
|
+
);
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
if (schema.pattern !== void 0) {
|
|
1372
|
+
assertPattern(schema.pattern, `${path}.pattern`);
|
|
1373
|
+
throw new TypeError(`${path}.pattern cannot be enforced incrementally and is unsupported.`);
|
|
1374
|
+
}
|
|
1375
|
+
if (schema.format !== void 0 && typeof schema.format !== "string") {
|
|
1376
|
+
throw new TypeError(`${path}.format must be a string.`);
|
|
1377
|
+
}
|
|
1378
|
+
if (typeof schema.format === "string" && ASSERTED_FORMATS.has(schema.format)) {
|
|
1379
|
+
throw new TypeError(
|
|
1380
|
+
`${path}.format ${JSON.stringify(schema.format)} cannot be enforced incrementally and is unsupported.`
|
|
1381
|
+
);
|
|
1382
|
+
}
|
|
1383
|
+
if (isRecord(schema.patternProperties)) {
|
|
1384
|
+
for (const pattern of Object.keys(schema.patternProperties))
|
|
1385
|
+
assertPattern(pattern, `${path}.patternProperties`);
|
|
1386
|
+
}
|
|
1387
|
+
if (schema.uniqueItems !== void 0 && typeof schema.uniqueItems !== "boolean") {
|
|
1388
|
+
throw new TypeError(`${path}.uniqueItems must be a boolean.`);
|
|
1389
|
+
}
|
|
1390
|
+
for (const keyword of [
|
|
1391
|
+
"minItems",
|
|
1392
|
+
"maxItems",
|
|
1393
|
+
"minContains",
|
|
1394
|
+
"maxContains",
|
|
1395
|
+
"minProperties",
|
|
1396
|
+
"maxProperties",
|
|
1397
|
+
"minLength",
|
|
1398
|
+
"maxLength"
|
|
1399
|
+
]) {
|
|
1400
|
+
const value = schema[keyword];
|
|
1401
|
+
if (value !== void 0 && (!Number.isInteger(value) || value < 0)) {
|
|
1402
|
+
throw new TypeError(`${path}.${keyword} must be a non-negative integer.`);
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
if (schema["x-guidance"] !== void 0 && path !== "$") {
|
|
1406
|
+
throw new TypeError("x-guidance is only supported on the root schema.");
|
|
1407
|
+
}
|
|
1408
|
+
schemaChildren(schema).forEach((child, index) => checkSchema(child, `${path}.schema[${index}]`));
|
|
1409
|
+
}
|
|
1410
|
+
function allowsKind(schema, kind) {
|
|
1411
|
+
if (schema.const !== void 0 && valueKind(schema.const) !== kind && !(kind === "number" && valueKind(schema.const) === "integer"))
|
|
1412
|
+
return false;
|
|
1413
|
+
if (Array.isArray(schema.enum) && !schema.enum.some((value) => valueKind(value) === kind || kind === "number" && valueKind(value) === "integer"))
|
|
1414
|
+
return false;
|
|
1415
|
+
if (schema.type === void 0) return true;
|
|
1416
|
+
const types = Array.isArray(schema.type) ? schema.type : [schema.type];
|
|
1417
|
+
return types.includes(kind) || kind === "number" && types.includes("integer") || kind === "integer" && types.includes("number");
|
|
1418
|
+
}
|
|
1419
|
+
function valueKind(value) {
|
|
1420
|
+
if (value === null) return "null";
|
|
1421
|
+
if (Array.isArray(value)) return "array";
|
|
1422
|
+
if (isRecord(value)) return "object";
|
|
1423
|
+
if (typeof value === "number") return Number.isInteger(value) ? "integer" : "number";
|
|
1424
|
+
if (["boolean", "string"].includes(typeof value)) return typeof value;
|
|
1425
|
+
return void 0;
|
|
1426
|
+
}
|
|
1427
|
+
function byteKind(byte) {
|
|
1428
|
+
if (byte === 34) return "string";
|
|
1429
|
+
if (byte === 123) return "object";
|
|
1430
|
+
if (byte === 91) return "array";
|
|
1431
|
+
if (byte === 116 || byte === 102) return "boolean";
|
|
1432
|
+
if (byte === 110) return "null";
|
|
1433
|
+
if (byte === 45 || isDigit(byte)) return "number";
|
|
1434
|
+
return void 0;
|
|
1435
|
+
}
|
|
1436
|
+
function numberPrefixValid(value) {
|
|
1437
|
+
return /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d*)?$/.test(value) || /^-?(?:0|[1-9]\d*)\.$/.test(value) || value === "-";
|
|
1438
|
+
}
|
|
1439
|
+
function numberComplete(value) {
|
|
1440
|
+
return /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$/.test(value);
|
|
1441
|
+
}
|
|
1442
|
+
function isNumberByte(byte) {
|
|
1443
|
+
return isDigit(byte) || byte === 46 || byte === 69 || byte === 101 || byte === 43 || byte === 45;
|
|
1444
|
+
}
|
|
1445
|
+
function decodeString(bytes) {
|
|
1446
|
+
try {
|
|
1447
|
+
return decoder.decode(Uint8Array.from(bytes));
|
|
1448
|
+
} catch {
|
|
1449
|
+
return null;
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
function topObject(state) {
|
|
1453
|
+
return state.stack.at(-1);
|
|
1454
|
+
}
|
|
1455
|
+
function replaceTop(stack, frame) {
|
|
1456
|
+
return [...stack.slice(0, -1), frame];
|
|
1457
|
+
}
|
|
1458
|
+
function allowsWhitespace(mode) {
|
|
1459
|
+
return ["value", "array-value", "object-key", "colon", "item-separator", "after-value", "done"].includes(mode);
|
|
1460
|
+
}
|
|
1461
|
+
function isWhitespace(byte) {
|
|
1462
|
+
return byte === 9 || byte === 10 || byte === 13 || byte === 32;
|
|
1463
|
+
}
|
|
1464
|
+
function isDigit(byte) {
|
|
1465
|
+
return byte >= 48 && byte <= 57;
|
|
1466
|
+
}
|
|
1467
|
+
function isHex(byte) {
|
|
1468
|
+
return isDigit(byte) || byte >= 65 && byte <= 70 || byte >= 97 && byte <= 102;
|
|
1469
|
+
}
|
|
1470
|
+
function isSchema(value) {
|
|
1471
|
+
return typeof value === "boolean" || isRecord(value);
|
|
1472
|
+
}
|
|
1473
|
+
function isRecord(value) {
|
|
1474
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
1475
|
+
}
|
|
1476
|
+
|
|
1477
|
+
// src/engine/regex.ts
|
|
1478
|
+
var EMPTY = { kind: "empty" };
|
|
1479
|
+
var encoder2 = new TextEncoder();
|
|
1480
|
+
function compileRegex(source) {
|
|
1481
|
+
const machine = new RegexMachine(new RegexParser(source).parse());
|
|
1482
|
+
return {
|
|
1483
|
+
initial: machine.initial,
|
|
1484
|
+
transition: (state, byte) => machine.transition(state, byte),
|
|
1485
|
+
viable: (state) => state >= 0,
|
|
1486
|
+
accepting: (state) => machine.accepting(state),
|
|
1487
|
+
maskKey: (state) => machine.stateKey(state)
|
|
1488
|
+
};
|
|
1489
|
+
}
|
|
1490
|
+
var RegexParser = class {
|
|
1491
|
+
constructor(source) {
|
|
1492
|
+
this.source = source;
|
|
1493
|
+
}
|
|
1494
|
+
index = 0;
|
|
1495
|
+
parse() {
|
|
1496
|
+
if (this.source.startsWith("^")) this.index++;
|
|
1497
|
+
const expression = this.alternation();
|
|
1498
|
+
if (this.peek() === "$" && this.index === this.source.length - 1) this.index++;
|
|
1499
|
+
if (this.index !== this.source.length) this.fail(`unexpected ${JSON.stringify(this.peek())}`);
|
|
1500
|
+
return expression;
|
|
1501
|
+
}
|
|
1502
|
+
alternation() {
|
|
1503
|
+
const choices = [this.concatenation()];
|
|
1504
|
+
while (this.peek() === "|") {
|
|
1505
|
+
this.index++;
|
|
1506
|
+
choices.push(this.concatenation());
|
|
1507
|
+
}
|
|
1508
|
+
return choice(choices);
|
|
1509
|
+
}
|
|
1510
|
+
concatenation() {
|
|
1511
|
+
const parts = [];
|
|
1512
|
+
while (this.index < this.source.length && this.peek() !== ")" && this.peek() !== "|") {
|
|
1513
|
+
if (this.peek() === "$" && this.index === this.source.length - 1) break;
|
|
1514
|
+
parts.push(this.quantified());
|
|
1515
|
+
}
|
|
1516
|
+
return sequence(parts);
|
|
1517
|
+
}
|
|
1518
|
+
quantified() {
|
|
1519
|
+
const atom = this.atom();
|
|
1520
|
+
const quantifier = this.peek();
|
|
1521
|
+
if (!["*", "+", "?", "{"].includes(quantifier ?? "")) return atom;
|
|
1522
|
+
let minimum;
|
|
1523
|
+
let maximum;
|
|
1524
|
+
if (quantifier === "*") {
|
|
1525
|
+
this.index++;
|
|
1526
|
+
[minimum, maximum] = [0, null];
|
|
1527
|
+
} else if (quantifier === "+") {
|
|
1528
|
+
this.index++;
|
|
1529
|
+
[minimum, maximum] = [1, null];
|
|
1530
|
+
} else if (quantifier === "?") {
|
|
1531
|
+
this.index++;
|
|
1532
|
+
[minimum, maximum] = [0, 1];
|
|
1533
|
+
} else {
|
|
1534
|
+
[minimum, maximum] = this.bounds();
|
|
1535
|
+
}
|
|
1536
|
+
if (this.peek() === "?" || this.peek() === "+") this.fail("lazy and possessive quantifiers are unsupported");
|
|
1537
|
+
const parts = Array.from({ length: minimum }, () => atom);
|
|
1538
|
+
if (maximum === null) parts.push(star(atom));
|
|
1539
|
+
else for (let count = minimum; count < maximum; ++count) parts.push(choice([EMPTY, atom]));
|
|
1540
|
+
return sequence(parts);
|
|
1541
|
+
}
|
|
1542
|
+
atom() {
|
|
1543
|
+
const character = this.peek();
|
|
1544
|
+
if (character === void 0) this.fail("expected an expression");
|
|
1545
|
+
if (character === "(") {
|
|
1546
|
+
this.index++;
|
|
1547
|
+
if (this.source.startsWith("?:", this.index)) this.index += 2;
|
|
1548
|
+
else if (this.peek() === "?") this.fail("only non-capturing special groups are supported");
|
|
1549
|
+
const result = this.alternation();
|
|
1550
|
+
if (this.peek() !== ")") this.fail("unterminated group");
|
|
1551
|
+
this.index++;
|
|
1552
|
+
return result;
|
|
1553
|
+
}
|
|
1554
|
+
if (character === "[") return this.characterClass();
|
|
1555
|
+
if (character === ".") {
|
|
1556
|
+
this.index++;
|
|
1557
|
+
return byteSet(range(0, 255));
|
|
1558
|
+
}
|
|
1559
|
+
if (character === "\\") return this.escape(false).expression;
|
|
1560
|
+
if ("*+?{})".includes(character)) this.fail(`unexpected ${JSON.stringify(character)}`);
|
|
1561
|
+
this.index += character.length;
|
|
1562
|
+
return literal(character);
|
|
1563
|
+
}
|
|
1564
|
+
characterClass() {
|
|
1565
|
+
this.index++;
|
|
1566
|
+
const negated = this.peek() === "^";
|
|
1567
|
+
if (negated) this.index++;
|
|
1568
|
+
const bytes = new Uint32Array(8);
|
|
1569
|
+
let hasValue = false;
|
|
1570
|
+
while (this.peek() !== "]") {
|
|
1571
|
+
if (this.peek() === void 0) this.fail("unterminated character class");
|
|
1572
|
+
const first = this.classValue();
|
|
1573
|
+
if (this.peek() === "-" && this.source[this.index + 1] !== "]") {
|
|
1574
|
+
this.index++;
|
|
1575
|
+
const last = this.classValue();
|
|
1576
|
+
if (first.single === void 0 || last.single === void 0 || first.single > last.single) {
|
|
1577
|
+
this.fail("invalid character class range");
|
|
1578
|
+
}
|
|
1579
|
+
addRange(bytes, first.single, last.single);
|
|
1580
|
+
} else {
|
|
1581
|
+
union(bytes, first.bytes);
|
|
1582
|
+
}
|
|
1583
|
+
hasValue = true;
|
|
1584
|
+
}
|
|
1585
|
+
this.index++;
|
|
1586
|
+
if (!hasValue) this.fail("empty character class");
|
|
1587
|
+
if (negated) for (let word = 0; word < bytes.length; ++word) bytes[word] = ~bytes[word];
|
|
1588
|
+
return byteSet(bytes);
|
|
1589
|
+
}
|
|
1590
|
+
classValue() {
|
|
1591
|
+
if (this.peek() === "\\") {
|
|
1592
|
+
const escaped = this.escape(true);
|
|
1593
|
+
if (escaped.bytes === void 0) this.fail("multi-byte escapes are unsupported in character classes");
|
|
1594
|
+
return { bytes: escaped.bytes, single: escaped.single };
|
|
1595
|
+
}
|
|
1596
|
+
const character = this.peek();
|
|
1597
|
+
this.index += character.length;
|
|
1598
|
+
const encoded = encoder2.encode(character);
|
|
1599
|
+
if (encoded.length !== 1) this.fail("non-ASCII character classes are unsupported");
|
|
1600
|
+
return { bytes: singleton(encoded[0]), single: encoded[0] };
|
|
1601
|
+
}
|
|
1602
|
+
escape(inClass) {
|
|
1603
|
+
this.index++;
|
|
1604
|
+
const code = this.peek();
|
|
1605
|
+
if (code === void 0) this.fail("trailing escape");
|
|
1606
|
+
this.index++;
|
|
1607
|
+
if ("dDsSwW".includes(code)) {
|
|
1608
|
+
const bytes2 = shorthand(code.toLowerCase());
|
|
1609
|
+
if (code === code.toUpperCase()) for (let word = 0; word < bytes2.length; ++word) bytes2[word] = ~bytes2[word];
|
|
1610
|
+
return { expression: byteSet(bytes2), bytes: bytes2 };
|
|
1611
|
+
}
|
|
1612
|
+
if (code === "b" && !inClass) this.fail("word boundaries are unsupported");
|
|
1613
|
+
let value;
|
|
1614
|
+
if (code === "x") value = this.hex(2);
|
|
1615
|
+
else if (code === "u") value = this.hex(4);
|
|
1616
|
+
else
|
|
1617
|
+
value = { n: 10, r: 13, t: 9, f: 12, v: 11, b: 8 }[code] ?? code.codePointAt(0);
|
|
1618
|
+
const text = String.fromCodePoint(value);
|
|
1619
|
+
const encoded = encoder2.encode(text);
|
|
1620
|
+
const bytes = encoded.length === 1 ? singleton(encoded[0]) : void 0;
|
|
1621
|
+
return { expression: literal(text), bytes, single: encoded.length === 1 ? encoded[0] : void 0 };
|
|
1622
|
+
}
|
|
1623
|
+
bounds() {
|
|
1624
|
+
this.index++;
|
|
1625
|
+
const minimum = this.decimal();
|
|
1626
|
+
let maximum = minimum;
|
|
1627
|
+
if (this.peek() === ",") {
|
|
1628
|
+
this.index++;
|
|
1629
|
+
maximum = this.peek() === "}" ? null : this.decimal();
|
|
1630
|
+
}
|
|
1631
|
+
if (this.peek() !== "}") this.fail("unterminated repetition");
|
|
1632
|
+
this.index++;
|
|
1633
|
+
if (minimum > 1e3 || maximum !== null && (maximum < minimum || maximum > 1e3)) {
|
|
1634
|
+
this.fail("invalid or excessive repetition");
|
|
1635
|
+
}
|
|
1636
|
+
return [minimum, maximum];
|
|
1637
|
+
}
|
|
1638
|
+
decimal() {
|
|
1639
|
+
const start = this.index;
|
|
1640
|
+
while (/\d/.test(this.peek() ?? "")) this.index++;
|
|
1641
|
+
if (start === this.index) this.fail("expected a repetition count");
|
|
1642
|
+
return Number(this.source.slice(start, this.index));
|
|
1643
|
+
}
|
|
1644
|
+
hex(length) {
|
|
1645
|
+
const value = this.source.slice(this.index, this.index + length);
|
|
1646
|
+
if (!new RegExp(`^[\\da-f]{${length}}$`, "i").test(value)) this.fail("invalid hexadecimal escape");
|
|
1647
|
+
this.index += length;
|
|
1648
|
+
return Number.parseInt(value, 16);
|
|
1649
|
+
}
|
|
1650
|
+
peek() {
|
|
1651
|
+
return this.source[this.index];
|
|
1652
|
+
}
|
|
1653
|
+
fail(message) {
|
|
1654
|
+
throw new SyntaxError(`Invalid regex at index ${this.index}: ${message}.`);
|
|
1655
|
+
}
|
|
1656
|
+
};
|
|
1657
|
+
var OP_SET = 0;
|
|
1658
|
+
var OP_SPLIT = 1;
|
|
1659
|
+
var OP_JUMP = 2;
|
|
1660
|
+
var OP_MATCH = 3;
|
|
1661
|
+
var UNKNOWN = -2;
|
|
1662
|
+
var NfaBuilder = class {
|
|
1663
|
+
ops = [];
|
|
1664
|
+
out1 = [];
|
|
1665
|
+
out2 = [];
|
|
1666
|
+
sets = [];
|
|
1667
|
+
compile(expression) {
|
|
1668
|
+
switch (expression.kind) {
|
|
1669
|
+
case "empty": {
|
|
1670
|
+
const state = this.emit(OP_JUMP);
|
|
1671
|
+
return { start: state, outs: [state << 1] };
|
|
1672
|
+
}
|
|
1673
|
+
case "set": {
|
|
1674
|
+
const state = this.emit(OP_SET, -1, -1, expression.bytes);
|
|
1675
|
+
return { start: state, outs: [state << 1] };
|
|
1676
|
+
}
|
|
1677
|
+
case "sequence": {
|
|
1678
|
+
let result = this.compile(expression.parts[0]);
|
|
1679
|
+
for (let index = 1; index < expression.parts.length; ++index) {
|
|
1680
|
+
const next = this.compile(expression.parts[index]);
|
|
1681
|
+
this.patch(result.outs, next.start);
|
|
1682
|
+
result = { start: result.start, outs: next.outs };
|
|
1683
|
+
}
|
|
1684
|
+
return result;
|
|
1685
|
+
}
|
|
1686
|
+
case "choice": {
|
|
1687
|
+
let result = this.compile(expression.choices[0]);
|
|
1688
|
+
for (let index = 1; index < expression.choices.length; ++index) {
|
|
1689
|
+
const right = this.compile(expression.choices[index]);
|
|
1690
|
+
result = {
|
|
1691
|
+
start: this.emit(OP_SPLIT, result.start, right.start),
|
|
1692
|
+
outs: [...result.outs, ...right.outs]
|
|
1693
|
+
};
|
|
1694
|
+
}
|
|
1695
|
+
return result;
|
|
1696
|
+
}
|
|
1697
|
+
case "star": {
|
|
1698
|
+
const child = this.compile(expression.child);
|
|
1699
|
+
const split = this.emit(OP_SPLIT, child.start);
|
|
1700
|
+
this.patch(child.outs, split);
|
|
1701
|
+
return { start: split, outs: [split << 1 | 1] };
|
|
1702
|
+
}
|
|
1703
|
+
}
|
|
1704
|
+
}
|
|
1705
|
+
emit(op, first = -1, second = -1, set) {
|
|
1706
|
+
const state = this.ops.length;
|
|
1707
|
+
this.ops.push(op);
|
|
1708
|
+
this.out1.push(first);
|
|
1709
|
+
this.out2.push(second);
|
|
1710
|
+
this.sets.push(set ?? new Uint32Array(0));
|
|
1711
|
+
return state;
|
|
1712
|
+
}
|
|
1713
|
+
patch(outs, target) {
|
|
1714
|
+
for (const output of outs) {
|
|
1715
|
+
if (output & 1) this.out2[output >>> 1] = target;
|
|
1716
|
+
else this.out1[output >>> 1] = target;
|
|
1717
|
+
}
|
|
1718
|
+
}
|
|
1719
|
+
};
|
|
1720
|
+
var RegexMachine = class {
|
|
1721
|
+
initial;
|
|
1722
|
+
builder = new NfaBuilder();
|
|
1723
|
+
states = [];
|
|
1724
|
+
acceptingStates = [];
|
|
1725
|
+
transitionTables = [];
|
|
1726
|
+
stateIds = /* @__PURE__ */ new Map();
|
|
1727
|
+
stateKeys = [];
|
|
1728
|
+
constructor(expression) {
|
|
1729
|
+
const fragment = this.builder.compile(expression);
|
|
1730
|
+
const match = this.builder.emit(OP_MATCH);
|
|
1731
|
+
this.builder.patch(fragment.outs, match);
|
|
1732
|
+
this.initial = this.intern(this.closure([fragment.start]));
|
|
1733
|
+
}
|
|
1734
|
+
transition(state, byte) {
|
|
1735
|
+
if (state < 0) return -1;
|
|
1736
|
+
const table = this.transitionTables[state];
|
|
1737
|
+
const cached = table[byte];
|
|
1738
|
+
if (cached !== UNKNOWN) return cached;
|
|
1739
|
+
const seeds = [];
|
|
1740
|
+
for (const pc of this.states[state]) {
|
|
1741
|
+
if (this.builder.ops[pc] !== OP_SET) continue;
|
|
1742
|
+
const set = this.builder.sets[pc];
|
|
1743
|
+
if (set[byte >>> 5] & 1 << (byte & 31)) seeds.push(this.builder.out1[pc]);
|
|
1744
|
+
}
|
|
1745
|
+
const next = seeds.length === 0 ? -1 : this.intern(this.closure(seeds));
|
|
1746
|
+
table[byte] = next;
|
|
1747
|
+
return next;
|
|
1748
|
+
}
|
|
1749
|
+
accepting(state) {
|
|
1750
|
+
return state >= 0 && this.acceptingStates[state];
|
|
1751
|
+
}
|
|
1752
|
+
// The interned NFA state set is intrinsic to the regex (unlike the interned
|
|
1753
|
+
// ids, which depend on discovery order), so it is a stable mask-cache key
|
|
1754
|
+
// across constraint instances compiled from the same source.
|
|
1755
|
+
stateKey(state) {
|
|
1756
|
+
return state >= 0 ? this.stateKeys[state] : void 0;
|
|
1757
|
+
}
|
|
1758
|
+
closure(seeds) {
|
|
1759
|
+
const result = [];
|
|
1760
|
+
const stack = [...seeds];
|
|
1761
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1762
|
+
while (stack.length > 0) {
|
|
1763
|
+
const state = stack.pop();
|
|
1764
|
+
if (state < 0 || seen.has(state)) continue;
|
|
1765
|
+
seen.add(state);
|
|
1766
|
+
const op = this.builder.ops[state];
|
|
1767
|
+
if (op === OP_SPLIT) {
|
|
1768
|
+
stack.push(this.builder.out1[state], this.builder.out2[state]);
|
|
1769
|
+
} else if (op === OP_JUMP) {
|
|
1770
|
+
stack.push(this.builder.out1[state]);
|
|
1771
|
+
} else {
|
|
1772
|
+
result.push(state);
|
|
1773
|
+
}
|
|
1774
|
+
}
|
|
1775
|
+
result.sort((left, right) => left - right);
|
|
1776
|
+
return result;
|
|
1777
|
+
}
|
|
1778
|
+
intern(active) {
|
|
1779
|
+
const key = active.join(",");
|
|
1780
|
+
const existing = this.stateIds.get(key);
|
|
1781
|
+
if (existing !== void 0) return existing;
|
|
1782
|
+
if (this.states.length >= 4096) throw new Error("Regex produced too many runtime states.");
|
|
1783
|
+
const id = this.states.length;
|
|
1784
|
+
const transitions = new Int32Array(256);
|
|
1785
|
+
transitions.fill(UNKNOWN);
|
|
1786
|
+
this.states.push(active);
|
|
1787
|
+
this.acceptingStates.push(active.some((state) => this.builder.ops[state] === OP_MATCH));
|
|
1788
|
+
this.transitionTables.push(transitions);
|
|
1789
|
+
this.stateIds.set(key, id);
|
|
1790
|
+
this.stateKeys.push(key);
|
|
1791
|
+
return id;
|
|
1792
|
+
}
|
|
1793
|
+
};
|
|
1794
|
+
function choice(items) {
|
|
1795
|
+
const flattened = items.flatMap((item) => item.kind === "choice" ? item.choices : [item]);
|
|
1796
|
+
if (flattened.length === 1) return flattened[0];
|
|
1797
|
+
return { kind: "choice", choices: flattened };
|
|
1798
|
+
}
|
|
1799
|
+
function sequence(items) {
|
|
1800
|
+
const flattened = items.flatMap((item) => item.kind === "sequence" ? item.parts : [item]).filter((item) => item !== EMPTY);
|
|
1801
|
+
if (flattened.length === 0) return EMPTY;
|
|
1802
|
+
if (flattened.length === 1) return flattened[0];
|
|
1803
|
+
return { kind: "sequence", parts: flattened };
|
|
1804
|
+
}
|
|
1805
|
+
function star(child) {
|
|
1806
|
+
if (child === EMPTY) return EMPTY;
|
|
1807
|
+
if (child.kind === "star") return child;
|
|
1808
|
+
return { kind: "star", child };
|
|
1809
|
+
}
|
|
1810
|
+
function literal(value) {
|
|
1811
|
+
return sequence([...encoder2.encode(value)].map((byte) => byteSet(singleton(byte))));
|
|
1812
|
+
}
|
|
1813
|
+
function byteSet(bytes) {
|
|
1814
|
+
return { kind: "set", bytes };
|
|
1815
|
+
}
|
|
1816
|
+
function shorthand(code) {
|
|
1817
|
+
const bytes = new Uint32Array(8);
|
|
1818
|
+
if (code === "d" || code === "w") addRange(bytes, 48, 57);
|
|
1819
|
+
if (code === "w") {
|
|
1820
|
+
addRange(bytes, 65, 90);
|
|
1821
|
+
addRange(bytes, 97, 122);
|
|
1822
|
+
add(bytes, 95);
|
|
1823
|
+
}
|
|
1824
|
+
if (code === "s") for (const byte of [9, 10, 11, 12, 13, 32]) add(bytes, byte);
|
|
1825
|
+
return bytes;
|
|
1826
|
+
}
|
|
1827
|
+
function singleton(byte) {
|
|
1828
|
+
const bytes = new Uint32Array(8);
|
|
1829
|
+
add(bytes, byte);
|
|
1830
|
+
return bytes;
|
|
1831
|
+
}
|
|
1832
|
+
function range(first, last) {
|
|
1833
|
+
const bytes = new Uint32Array(8);
|
|
1834
|
+
addRange(bytes, first, last);
|
|
1835
|
+
return bytes;
|
|
1836
|
+
}
|
|
1837
|
+
function addRange(bytes, first, last) {
|
|
1838
|
+
for (let byte = first; byte <= last; ++byte) add(bytes, byte);
|
|
1839
|
+
}
|
|
1840
|
+
function add(bytes, byte) {
|
|
1841
|
+
bytes[byte >>> 5] |= 1 << (byte & 31);
|
|
1842
|
+
}
|
|
1843
|
+
function union(target, source) {
|
|
1844
|
+
for (let word = 0; word < target.length; ++word) target[word] |= source[word];
|
|
1845
|
+
}
|
|
1846
|
+
|
|
1847
|
+
// src/engine/tokenizer.ts
|
|
1848
|
+
var encoder3 = new TextEncoder();
|
|
1849
|
+
var byteLevelMap;
|
|
1850
|
+
function extractTokenizer(tokenizer) {
|
|
1851
|
+
const source = asRecord(tokenizer, "tokenizer");
|
|
1852
|
+
if (Array.isArray(source.tokens)) {
|
|
1853
|
+
return normalizeDirectTokenizer(source);
|
|
1854
|
+
}
|
|
1855
|
+
const tokenizerJson = getTokenizerJson(source);
|
|
1856
|
+
const vocabulary = getVocabulary(source, tokenizerJson);
|
|
1857
|
+
if (vocabulary === void 0) {
|
|
1858
|
+
throw new TypeError("Could not extract the tokenizer vocabulary.");
|
|
1859
|
+
}
|
|
1860
|
+
const vocabularyTokens = Object.keys(vocabulary);
|
|
1861
|
+
let size = 0;
|
|
1862
|
+
for (const token of vocabularyTokens) {
|
|
1863
|
+
const id = Number(vocabulary[token]);
|
|
1864
|
+
if (!Number.isInteger(id) || id < 0) throw new TypeError(`Tokenizer has an invalid token ID for ${token}.`);
|
|
1865
|
+
if (id + 1 > size) size = id + 1;
|
|
1866
|
+
}
|
|
1867
|
+
const tokenBytes = tokenBytesConverter(source, tokenizerJson);
|
|
1868
|
+
const tokens = new Array(size);
|
|
1869
|
+
for (const token of vocabularyTokens) {
|
|
1870
|
+
const id = Number(vocabulary[token]);
|
|
1871
|
+
tokens[id] = tokenBytes(token, id);
|
|
1872
|
+
}
|
|
1873
|
+
const addedTokens = field(tokenizerJson, "added_tokens");
|
|
1874
|
+
if (Array.isArray(addedTokens)) {
|
|
1875
|
+
for (const added of addedTokens) {
|
|
1876
|
+
if (!isRecord2(added) || !Number.isInteger(added.id)) continue;
|
|
1877
|
+
while (tokens.length <= Number(added.id)) tokens.push(void 0);
|
|
1878
|
+
const id = Number(added.id);
|
|
1879
|
+
tokens[id] = tokenBytes(typeof added.content === "string" ? added.content : "", id);
|
|
1880
|
+
}
|
|
1881
|
+
}
|
|
1882
|
+
for (let id = 0; id < tokens.length; ++id) {
|
|
1883
|
+
if (tokens[id] === void 0) throw new Error(`Tokenizer vocabulary is missing token ID ${id}.`);
|
|
1884
|
+
}
|
|
1885
|
+
const eosTokenId = tokenId(source, tokenizerJson, ["eos_token_id", "eosTokenId", "eos_token", "eosToken"]);
|
|
1886
|
+
if (eosTokenId === void 0) throw new TypeError("Tokenizer does not expose an EOS token ID.");
|
|
1887
|
+
const specialTokenIds = /* @__PURE__ */ new Set([eosTokenId]);
|
|
1888
|
+
for (const value of [
|
|
1889
|
+
source.special_token_ids,
|
|
1890
|
+
source.specialTokenIds,
|
|
1891
|
+
source.all_special_ids,
|
|
1892
|
+
source.allSpecialIds
|
|
1893
|
+
]) {
|
|
1894
|
+
if (Array.isArray(value)) {
|
|
1895
|
+
for (const id of value) if (Number.isInteger(id)) specialTokenIds.add(Number(id));
|
|
1896
|
+
}
|
|
1897
|
+
}
|
|
1898
|
+
if (Array.isArray(addedTokens)) {
|
|
1899
|
+
for (const added of addedTokens) {
|
|
1900
|
+
if (isRecord2(added) && added.special === true && Number.isInteger(added.id))
|
|
1901
|
+
specialTokenIds.add(Number(added.id));
|
|
1902
|
+
}
|
|
1903
|
+
}
|
|
1904
|
+
return { tokens, eosTokenId, specialTokenIds };
|
|
1905
|
+
}
|
|
1906
|
+
function normalizeDirectTokenizer(source) {
|
|
1907
|
+
const configuredTokens = source.tokens;
|
|
1908
|
+
const tokens = configuredTokens.map((token, id) => {
|
|
1909
|
+
if (!(token instanceof Uint8Array) && !Array.isArray(token)) {
|
|
1910
|
+
throw new TypeError(`Tokenizer token ${id} must be a byte array.`);
|
|
1911
|
+
}
|
|
1912
|
+
const values = Array.from(token);
|
|
1913
|
+
if (values.some((byte) => !Number.isInteger(byte) || byte < 0 || byte > 255))
|
|
1914
|
+
throw new TypeError(`Tokenizer token ${id} is invalid.`);
|
|
1915
|
+
return Uint8Array.from(values);
|
|
1916
|
+
});
|
|
1917
|
+
const eosTokenId = Number(source.eosTokenId ?? source.eos_token_id);
|
|
1918
|
+
if (!Number.isInteger(eosTokenId) || eosTokenId < 0 || eosTokenId >= tokens.length) {
|
|
1919
|
+
throw new TypeError("A valid eos_token_id is required with tokenizer tokens.");
|
|
1920
|
+
}
|
|
1921
|
+
const configured = source.specialTokenIds ?? source.special_token_ids;
|
|
1922
|
+
const specialTokenIds = /* @__PURE__ */ new Set([eosTokenId]);
|
|
1923
|
+
if (Array.isArray(configured)) for (const id of configured) specialTokenIds.add(Number(id));
|
|
1924
|
+
return { tokens, eosTokenId, specialTokenIds };
|
|
1925
|
+
}
|
|
1926
|
+
function getTokenizerJson(source) {
|
|
1927
|
+
const value = source._tokenizerJSON ?? source.tokenizerJSON ?? source.tokenizer_json;
|
|
1928
|
+
return typeof value === "string" ? JSON.parse(value) : value;
|
|
1929
|
+
}
|
|
1930
|
+
function getVocabulary(source, tokenizerJson) {
|
|
1931
|
+
const modelVocabulary = field(field(tokenizerJson, "model"), "vocab");
|
|
1932
|
+
if (isRecord2(modelVocabulary)) return modelVocabulary;
|
|
1933
|
+
for (const name of ["get_vocab", "getVocab"]) {
|
|
1934
|
+
const method = source[name];
|
|
1935
|
+
if (typeof method !== "function") continue;
|
|
1936
|
+
const value = method.call(source, true);
|
|
1937
|
+
if (value instanceof Map) return Object.fromEntries(value);
|
|
1938
|
+
if (isRecord2(value)) return value;
|
|
1939
|
+
}
|
|
1940
|
+
return isRecord2(source.vocab) ? source.vocab : void 0;
|
|
1941
|
+
}
|
|
1942
|
+
function tokenBytesConverter(source, tokenizerJson) {
|
|
1943
|
+
const decoder2 = field(tokenizerJson, "decoder");
|
|
1944
|
+
if (field(decoder2, "type") === "ByteLevel") return byteLevelTokenBytes;
|
|
1945
|
+
const decode = source.decode;
|
|
1946
|
+
if (typeof decode === "function") {
|
|
1947
|
+
const byteFallback = hasComponent(decoder2, "ByteFallback");
|
|
1948
|
+
return (token, id) => {
|
|
1949
|
+
const fallback = byteFallback ? /^<0x([\da-f]{2})>$/i.exec(token) : null;
|
|
1950
|
+
if (fallback) return Uint8Array.of(Number.parseInt(fallback[1], 16));
|
|
1951
|
+
const decoded = decode.call(source, [id], {
|
|
1952
|
+
skip_special_tokens: false,
|
|
1953
|
+
clean_up_tokenization_spaces: false
|
|
1954
|
+
});
|
|
1955
|
+
if (typeof decoded !== "string") throw new TypeError(`Tokenizer.decode([${id}]) must return a string.`);
|
|
1956
|
+
return encoder3.encode(decoded);
|
|
1957
|
+
};
|
|
1958
|
+
}
|
|
1959
|
+
if (decoder2 !== void 0 && decoder2 !== null && field(decoder2, "type") !== "ByteLevel") {
|
|
1960
|
+
throw new TypeError("Tokenizer decoder semantics require a tokenizer with a decode() method.");
|
|
1961
|
+
}
|
|
1962
|
+
if (hasComponent(decoder2, "ByteLevel") || hasComponent(field(tokenizerJson, "pre_tokenizer"), "ByteLevel")) {
|
|
1963
|
+
return byteLevelTokenBytes;
|
|
1964
|
+
}
|
|
1965
|
+
const modelType = field(field(tokenizerJson, "model"), "type");
|
|
1966
|
+
const sentencePiece = modelType === "Unigram" || modelType === "SentencePiece";
|
|
1967
|
+
const prefix = field(field(tokenizerJson, "model"), "continuing_subword_prefix");
|
|
1968
|
+
return (token) => {
|
|
1969
|
+
const fallback = /^<0x([\da-f]{2})>$/i.exec(token);
|
|
1970
|
+
if (fallback) return Uint8Array.of(Number.parseInt(fallback[1], 16));
|
|
1971
|
+
if (sentencePiece || token.includes("\u2581")) return encoder3.encode(token.replaceAll("\u2581", " "));
|
|
1972
|
+
return encoder3.encode(
|
|
1973
|
+
typeof prefix === "string" && token.startsWith(prefix) ? token.slice(prefix.length) : token
|
|
1974
|
+
);
|
|
1975
|
+
};
|
|
1976
|
+
}
|
|
1977
|
+
function byteLevelTokenBytes(token) {
|
|
1978
|
+
const fallback = /^<0x([\da-f]{2})>$/i.exec(token);
|
|
1979
|
+
if (fallback) return Uint8Array.of(Number.parseInt(fallback[1], 16));
|
|
1980
|
+
const map = getByteLevelMap();
|
|
1981
|
+
const bytes = [];
|
|
1982
|
+
for (const character of token) {
|
|
1983
|
+
const byte = map.get(character);
|
|
1984
|
+
if (byte === void 0) bytes.push(...encoder3.encode(character));
|
|
1985
|
+
else bytes.push(byte);
|
|
1986
|
+
}
|
|
1987
|
+
return Uint8Array.from(bytes);
|
|
1988
|
+
}
|
|
1989
|
+
function getByteLevelMap() {
|
|
1990
|
+
if (byteLevelMap !== void 0) return byteLevelMap;
|
|
1991
|
+
const visible = /* @__PURE__ */ new Set();
|
|
1992
|
+
for (let code = 33; code <= 126; ++code) visible.add(code);
|
|
1993
|
+
for (let code = 161; code <= 172; ++code) visible.add(code);
|
|
1994
|
+
for (let code = 174; code <= 255; ++code) visible.add(code);
|
|
1995
|
+
let extra = 0;
|
|
1996
|
+
byteLevelMap = /* @__PURE__ */ new Map();
|
|
1997
|
+
for (let byte = 0; byte < 256; ++byte) {
|
|
1998
|
+
byteLevelMap.set(String.fromCharCode(visible.has(byte) ? byte : 256 + extra++), byte);
|
|
1999
|
+
}
|
|
2000
|
+
return byteLevelMap;
|
|
2001
|
+
}
|
|
2002
|
+
function tokenId(source, tokenizerJson, keys) {
|
|
2003
|
+
const vocabulary = getVocabulary(source, tokenizerJson);
|
|
2004
|
+
for (const key of keys) {
|
|
2005
|
+
const value = source[key] ?? field(tokenizerJson, key);
|
|
2006
|
+
if (Number.isInteger(value)) return Number(value);
|
|
2007
|
+
if (typeof value === "string" && Number.isInteger(vocabulary?.[value])) return Number(vocabulary[value]);
|
|
2008
|
+
}
|
|
2009
|
+
return void 0;
|
|
2010
|
+
}
|
|
2011
|
+
function hasComponent(value, type) {
|
|
2012
|
+
if (Array.isArray(value)) return value.some((item) => hasComponent(item, type));
|
|
2013
|
+
return isRecord2(value) && (value.type === type || Object.values(value).some((item) => hasComponent(item, type)));
|
|
2014
|
+
}
|
|
2015
|
+
function field(value, key) {
|
|
2016
|
+
return isRecord2(value) ? value[key] : void 0;
|
|
2017
|
+
}
|
|
2018
|
+
function asRecord(value, name) {
|
|
2019
|
+
if (typeof value !== "object" && typeof value !== "function" || value === null) {
|
|
2020
|
+
throw new TypeError(`${name} must be an object.`);
|
|
2021
|
+
}
|
|
2022
|
+
return value;
|
|
2023
|
+
}
|
|
2024
|
+
function isRecord2(value) {
|
|
2025
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
2026
|
+
}
|
|
2027
|
+
|
|
2028
|
+
// src/engine/constraint.ts
|
|
2029
|
+
var tokenizerCache = /* @__PURE__ */ new WeakMap();
|
|
2030
|
+
var JSON_OBJECT_SCHEMA = { type: "object" };
|
|
2031
|
+
function prepareTokenizer(tokenizerSource) {
|
|
2032
|
+
cachedTokenizer(tokenizerSource);
|
|
2033
|
+
}
|
|
2034
|
+
function createTokenConstraint(tokenizerSource, responseFormat) {
|
|
2035
|
+
const tokenizer = cachedTokenizer(tokenizerSource);
|
|
2036
|
+
const machine = createMachine(responseFormat, tokenizer);
|
|
2037
|
+
const maskCache = cacheFor(tokenizer, responseFormat);
|
|
2038
|
+
let state = machine.initial;
|
|
2039
|
+
const tokenStates = new Array(tokenizer.data.tokens.length);
|
|
2040
|
+
const tokenStamps = new Int32Array(tokenizer.data.tokens.length);
|
|
2041
|
+
let stamp = 0;
|
|
2042
|
+
let consecutiveWhitespace = 0;
|
|
2043
|
+
const tracksJsonWhitespace = responseFormat.type !== "regex";
|
|
2044
|
+
return {
|
|
2045
|
+
vocabSize: tokenizer.data.tokens.length,
|
|
2046
|
+
fillMask(target) {
|
|
2047
|
+
const words = Math.ceil(tokenizer.data.tokens.length / 32);
|
|
2048
|
+
if (target.length < words) throw new RangeError(`Mask target requires at least ${words} words.`);
|
|
2049
|
+
target.fill(0);
|
|
2050
|
+
stamp++;
|
|
2051
|
+
const cacheKey = machine.maskKey?.(state);
|
|
2052
|
+
const cachedMask = cacheKey === void 0 ? void 0 : maskCache?.get(cacheKey);
|
|
2053
|
+
if (cachedMask !== void 0) {
|
|
2054
|
+
target.set(cachedMask);
|
|
2055
|
+
return true;
|
|
2056
|
+
}
|
|
2057
|
+
let allowed = 0;
|
|
2058
|
+
if (machine.accepting(state)) {
|
|
2059
|
+
setBit(target, tokenizer.data.eosTokenId);
|
|
2060
|
+
allowed++;
|
|
2061
|
+
}
|
|
2062
|
+
const stringCapacity2 = machine.stringCapacity?.(state);
|
|
2063
|
+
if (stringCapacity2 !== void 0) {
|
|
2064
|
+
const safe = boundedStringMask(tokenizer, stringCapacity2);
|
|
2065
|
+
target.set(safe.mask);
|
|
2066
|
+
allowed += safe.count;
|
|
2067
|
+
}
|
|
2068
|
+
const nodes = [stringCapacity2 === void 0 ? tokenizer.trie : tokenizer.stringExceptionalTrie];
|
|
2069
|
+
const states = [state];
|
|
2070
|
+
while (nodes.length > 0) {
|
|
2071
|
+
const node = nodes.pop();
|
|
2072
|
+
const current = states.pop();
|
|
2073
|
+
for (const tokenId2 of node.tokenIds) {
|
|
2074
|
+
if (tokenizer.data.specialTokenIds.has(tokenId2)) continue;
|
|
2075
|
+
setBit(target, tokenId2);
|
|
2076
|
+
tokenStates[tokenId2] = current;
|
|
2077
|
+
tokenStamps[tokenId2] = stamp;
|
|
2078
|
+
allowed++;
|
|
2079
|
+
}
|
|
2080
|
+
for (let index = 0; index < node.childNodes.length; ++index) {
|
|
2081
|
+
const next = machine.transition(current, node.childBytes[index]);
|
|
2082
|
+
if (!machine.viable(next)) continue;
|
|
2083
|
+
nodes.push(node.childNodes[index]);
|
|
2084
|
+
states.push(next);
|
|
2085
|
+
}
|
|
2086
|
+
}
|
|
2087
|
+
if (allowed > 0 && cacheKey !== void 0) {
|
|
2088
|
+
maskCache?.set(cacheKey, target.subarray(0, words));
|
|
2089
|
+
}
|
|
2090
|
+
return allowed > 0;
|
|
2091
|
+
},
|
|
2092
|
+
commit(tokenId2) {
|
|
2093
|
+
if (!Number.isInteger(tokenId2) || tokenId2 < 0 || tokenId2 >= tokenizer.data.tokens.length) {
|
|
2094
|
+
throw new RangeError(`Token ${tokenId2} is outside the tokenizer vocabulary.`);
|
|
2095
|
+
}
|
|
2096
|
+
if (tokenId2 === tokenizer.data.eosTokenId) {
|
|
2097
|
+
if (!machine.accepting(state)) throw new Error(`Token ${tokenId2} does not satisfy the constraint.`);
|
|
2098
|
+
return true;
|
|
2099
|
+
}
|
|
2100
|
+
if (tokenizer.data.specialTokenIds.has(tokenId2)) {
|
|
2101
|
+
throw new Error(`Token ${tokenId2} does not satisfy the constraint.`);
|
|
2102
|
+
}
|
|
2103
|
+
let next;
|
|
2104
|
+
if (tokenStamps[tokenId2] === stamp && stamp > 0) {
|
|
2105
|
+
next = tokenStates[tokenId2];
|
|
2106
|
+
} else {
|
|
2107
|
+
next = state;
|
|
2108
|
+
for (const byte of tokenizer.data.tokens[tokenId2]) next = machine.transition(next, byte);
|
|
2109
|
+
}
|
|
2110
|
+
stamp++;
|
|
2111
|
+
if (!machine.viable(next)) throw new Error(`Token ${tokenId2} does not satisfy the constraint.`);
|
|
2112
|
+
consecutiveWhitespace = tracksJsonWhitespace && next === state && isJsonWhitespace(tokenizer.data.tokens[tokenId2]) ? consecutiveWhitespace + 1 : 0;
|
|
2113
|
+
state = next;
|
|
2114
|
+
return false;
|
|
2115
|
+
},
|
|
2116
|
+
repeatedWhitespace() {
|
|
2117
|
+
if (consecutiveWhitespace === 0) return void 0;
|
|
2118
|
+
return { tokenIds: tokenizer.whitespaceTokenIds, count: consecutiveWhitespace };
|
|
2119
|
+
}
|
|
2120
|
+
};
|
|
2121
|
+
}
|
|
2122
|
+
function createMachine(responseFormat, tokenizer) {
|
|
2123
|
+
if (responseFormat?.type === "regex") {
|
|
2124
|
+
if (typeof responseFormat.regex !== "string") throw new TypeError("response_format.regex must be a string.");
|
|
2125
|
+
return compileRegex(responseFormat.regex);
|
|
2126
|
+
}
|
|
2127
|
+
if (responseFormat?.type === "json_schema") {
|
|
2128
|
+
return compileJsonSchema(responseFormat.json_schema, tokenizer.maxTokenByteLength);
|
|
2129
|
+
}
|
|
2130
|
+
if (responseFormat?.type === "json_object") {
|
|
2131
|
+
return compileJsonSchema(JSON_OBJECT_SCHEMA, tokenizer.maxTokenByteLength);
|
|
2132
|
+
}
|
|
2133
|
+
throw new TypeError(`Unsupported response format: ${String(responseFormat?.type)}.`);
|
|
2134
|
+
}
|
|
2135
|
+
function cachedTokenizer(source) {
|
|
2136
|
+
let cached = tokenizerCache.get(source);
|
|
2137
|
+
if (cached === void 0) {
|
|
2138
|
+
const data = extractTokenizer(source);
|
|
2139
|
+
const stringExceptionalTokenIds = [];
|
|
2140
|
+
const whitespaceTokenIds = [];
|
|
2141
|
+
const stringSafeMask = new Uint32Array(Math.ceil(data.tokens.length / 32));
|
|
2142
|
+
const stringSafeLengths = new Uint32Array(data.tokens.length);
|
|
2143
|
+
let stringSafeCount = 0;
|
|
2144
|
+
let maxStringSafeLength = 0;
|
|
2145
|
+
let maxTokenByteLength = 0;
|
|
2146
|
+
for (let tokenId2 = 0; tokenId2 < data.tokens.length; ++tokenId2) {
|
|
2147
|
+
const special = data.specialTokenIds.has(tokenId2);
|
|
2148
|
+
if (!special && isJsonWhitespace(data.tokens[tokenId2])) whitespaceTokenIds.push(tokenId2);
|
|
2149
|
+
if (!special && data.tokens[tokenId2].length > maxTokenByteLength) {
|
|
2150
|
+
maxTokenByteLength = data.tokens[tokenId2].length;
|
|
2151
|
+
}
|
|
2152
|
+
const length = special ? void 0 : safeStringTokenLength(data.tokens[tokenId2]);
|
|
2153
|
+
if (length !== void 0) {
|
|
2154
|
+
stringSafeCount++;
|
|
2155
|
+
stringSafeLengths[tokenId2] = length;
|
|
2156
|
+
if (length > maxStringSafeLength) maxStringSafeLength = length;
|
|
2157
|
+
setBit(stringSafeMask, tokenId2);
|
|
2158
|
+
} else {
|
|
2159
|
+
stringExceptionalTokenIds.push(tokenId2);
|
|
2160
|
+
}
|
|
2161
|
+
}
|
|
2162
|
+
cached = {
|
|
2163
|
+
data,
|
|
2164
|
+
trie: createTrie(data.tokens),
|
|
2165
|
+
whitespaceTokenIds,
|
|
2166
|
+
stringExceptionalTrie: createTrie(data.tokens, stringExceptionalTokenIds),
|
|
2167
|
+
stringSafeMask,
|
|
2168
|
+
stringSafeCount,
|
|
2169
|
+
stringSafeLengths,
|
|
2170
|
+
maxStringSafeLength,
|
|
2171
|
+
maxTokenByteLength,
|
|
2172
|
+
boundedStringMasks: /* @__PURE__ */ new Map(),
|
|
2173
|
+
schemaMaskCaches: /* @__PURE__ */ new WeakMap(),
|
|
2174
|
+
booleanSchemaMaskCaches: [new MaskCache(), new MaskCache()],
|
|
2175
|
+
jsonObjectMaskCache: new MaskCache(),
|
|
2176
|
+
regexMaskCaches: /* @__PURE__ */ new Map()
|
|
2177
|
+
};
|
|
2178
|
+
tokenizerCache.set(source, cached);
|
|
2179
|
+
}
|
|
2180
|
+
return cached;
|
|
2181
|
+
}
|
|
2182
|
+
function cacheFor(tokenizer, responseFormat) {
|
|
2183
|
+
if (responseFormat.type === "regex") {
|
|
2184
|
+
let cache2 = tokenizer.regexMaskCaches.get(responseFormat.regex);
|
|
2185
|
+
if (cache2 === void 0) {
|
|
2186
|
+
cache2 = new MaskCache();
|
|
2187
|
+
if (tokenizer.regexMaskCaches.size >= 16) {
|
|
2188
|
+
tokenizer.regexMaskCaches.delete(tokenizer.regexMaskCaches.keys().next().value);
|
|
2189
|
+
}
|
|
2190
|
+
tokenizer.regexMaskCaches.set(responseFormat.regex, cache2);
|
|
2191
|
+
}
|
|
2192
|
+
return cache2;
|
|
2193
|
+
}
|
|
2194
|
+
if (responseFormat.type === "json_object") return tokenizer.jsonObjectMaskCache;
|
|
2195
|
+
const schema = responseFormat.json_schema;
|
|
2196
|
+
if (typeof schema === "boolean") return tokenizer.booleanSchemaMaskCaches[schema ? 1 : 0];
|
|
2197
|
+
let cache = tokenizer.schemaMaskCaches.get(schema);
|
|
2198
|
+
if (cache === void 0) {
|
|
2199
|
+
cache = new MaskCache();
|
|
2200
|
+
tokenizer.schemaMaskCaches.set(schema, cache);
|
|
2201
|
+
}
|
|
2202
|
+
return cache;
|
|
2203
|
+
}
|
|
2204
|
+
var MaskCache = class {
|
|
2205
|
+
masks = /* @__PURE__ */ new Map();
|
|
2206
|
+
words = 0;
|
|
2207
|
+
get(key) {
|
|
2208
|
+
const mask = this.masks.get(key);
|
|
2209
|
+
if (mask === void 0) return void 0;
|
|
2210
|
+
this.masks.delete(key);
|
|
2211
|
+
this.masks.set(key, mask);
|
|
2212
|
+
return mask;
|
|
2213
|
+
}
|
|
2214
|
+
set(key, source) {
|
|
2215
|
+
const mask = source.slice();
|
|
2216
|
+
const previous = this.masks.get(key);
|
|
2217
|
+
if (previous !== void 0) {
|
|
2218
|
+
this.words -= previous.length;
|
|
2219
|
+
this.masks.delete(key);
|
|
2220
|
+
}
|
|
2221
|
+
this.masks.set(key, mask);
|
|
2222
|
+
this.words += mask.length;
|
|
2223
|
+
while (this.masks.size > 256 || this.words > 1048576) {
|
|
2224
|
+
const oldestKey = this.masks.keys().next().value;
|
|
2225
|
+
const oldest = this.masks.get(oldestKey);
|
|
2226
|
+
this.masks.delete(oldestKey);
|
|
2227
|
+
this.words -= oldest.length;
|
|
2228
|
+
}
|
|
2229
|
+
}
|
|
2230
|
+
};
|
|
2231
|
+
function createTrie(tokens, tokenIds) {
|
|
2232
|
+
const root = { childBytes: [], childNodes: [], tokenIds: [] };
|
|
2233
|
+
const size = tokenIds === void 0 ? tokens.length : tokenIds.length;
|
|
2234
|
+
for (let index = 0; index < size; ++index) {
|
|
2235
|
+
const tokenId2 = tokenIds === void 0 ? index : tokenIds[index];
|
|
2236
|
+
const bytes = tokens[tokenId2];
|
|
2237
|
+
let node = root;
|
|
2238
|
+
for (let position = 0; position < bytes.length; ++position) {
|
|
2239
|
+
const byte = bytes[position];
|
|
2240
|
+
const childIndex = node.childBytes.indexOf(byte);
|
|
2241
|
+
if (childIndex === -1) {
|
|
2242
|
+
const child = { childBytes: [], childNodes: [], tokenIds: [] };
|
|
2243
|
+
node.childBytes.push(byte);
|
|
2244
|
+
node.childNodes.push(child);
|
|
2245
|
+
node = child;
|
|
2246
|
+
} else {
|
|
2247
|
+
node = node.childNodes[childIndex];
|
|
2248
|
+
}
|
|
2249
|
+
}
|
|
2250
|
+
node.tokenIds.push(tokenId2);
|
|
2251
|
+
}
|
|
2252
|
+
return root;
|
|
2253
|
+
}
|
|
2254
|
+
var safeStringDecoder = new TextDecoder("utf-8", { fatal: true });
|
|
2255
|
+
function safeStringTokenLength(bytes) {
|
|
2256
|
+
if (bytes.length === 0) return void 0;
|
|
2257
|
+
let length = 0;
|
|
2258
|
+
for (const byte of bytes) {
|
|
2259
|
+
if (byte < 32 || byte === 34 || byte === 92) return void 0;
|
|
2260
|
+
if ((byte & 192) !== 128) length++;
|
|
2261
|
+
}
|
|
2262
|
+
try {
|
|
2263
|
+
safeStringDecoder.decode(bytes);
|
|
2264
|
+
} catch {
|
|
2265
|
+
return void 0;
|
|
2266
|
+
}
|
|
2267
|
+
return length;
|
|
2268
|
+
}
|
|
2269
|
+
function boundedStringMask(tokenizer, capacity) {
|
|
2270
|
+
if (capacity >= tokenizer.maxStringSafeLength) {
|
|
2271
|
+
return { mask: tokenizer.stringSafeMask, count: tokenizer.stringSafeCount };
|
|
2272
|
+
}
|
|
2273
|
+
let cached = tokenizer.boundedStringMasks.get(capacity);
|
|
2274
|
+
if (cached !== void 0) return cached;
|
|
2275
|
+
const mask = new Uint32Array(Math.ceil(tokenizer.data.tokens.length / 32));
|
|
2276
|
+
let count = 0;
|
|
2277
|
+
for (let tokenId2 = 0; tokenId2 < tokenizer.stringSafeLengths.length; ++tokenId2) {
|
|
2278
|
+
const length = tokenizer.stringSafeLengths[tokenId2];
|
|
2279
|
+
if (length === 0 || length > capacity) continue;
|
|
2280
|
+
setBit(mask, tokenId2);
|
|
2281
|
+
count++;
|
|
2282
|
+
}
|
|
2283
|
+
cached = { mask, count };
|
|
2284
|
+
tokenizer.boundedStringMasks.set(capacity, cached);
|
|
2285
|
+
return cached;
|
|
2286
|
+
}
|
|
2287
|
+
function isJsonWhitespace(bytes) {
|
|
2288
|
+
return bytes.length > 0 && bytes.every((byte) => byte === 9 || byte === 10 || byte === 13 || byte === 32);
|
|
2289
|
+
}
|
|
2290
|
+
function setBit(mask, tokenId2) {
|
|
2291
|
+
mask[tokenId2 >>> 5] |= 1 << (tokenId2 & 31);
|
|
2292
|
+
}
|
|
2293
|
+
|
|
2294
|
+
// src/utils/mask.ts
|
|
2295
|
+
function applyMask(logits, mask, vocabSize) {
|
|
2296
|
+
const data = logits.data;
|
|
2297
|
+
const stride = logits.dims.at(-1);
|
|
2298
|
+
if (vocabSize > stride) {
|
|
2299
|
+
throw new Error(`Constraint vocabulary size ${vocabSize} exceeds logits vocabulary size ${stride}.`);
|
|
2300
|
+
}
|
|
2301
|
+
for (let offset = 0; offset < data.length; offset += stride) {
|
|
2302
|
+
const fullWords = vocabSize >>> 5;
|
|
2303
|
+
for (let word = 0; word < fullWords; ++word) {
|
|
2304
|
+
const bits = mask[word] | 0;
|
|
2305
|
+
if (bits === -1) continue;
|
|
2306
|
+
const start = offset + (word << 5);
|
|
2307
|
+
if (bits === 0) {
|
|
2308
|
+
data.fill(-Infinity, start, start + 32);
|
|
2309
|
+
} else {
|
|
2310
|
+
for (let bit = 0; bit < 32; ++bit) {
|
|
2311
|
+
if (!(bits & 1 << bit)) data[start + bit] = -Infinity;
|
|
2312
|
+
}
|
|
2313
|
+
}
|
|
2314
|
+
}
|
|
2315
|
+
for (let tokenId2 = fullWords << 5; tokenId2 < vocabSize; ++tokenId2) {
|
|
2316
|
+
if (!(mask[tokenId2 >>> 5] & 1 << (tokenId2 & 31))) data[offset + tokenId2] = -Infinity;
|
|
2317
|
+
}
|
|
2318
|
+
data.fill(-Infinity, offset + vocabSize, offset + stride);
|
|
2319
|
+
}
|
|
2320
|
+
}
|
|
2321
|
+
|
|
2322
|
+
// src/StructuredOutputProcessor.ts
|
|
2323
|
+
var WHITESPACE_REPETITION_PENALTY = 1.2;
|
|
2324
|
+
var MAX_CONSECUTIVE_WHITESPACE_TOKENS = 4;
|
|
2325
|
+
var StructuredOutputProcessor = class extends LogitsProcessorList {
|
|
2326
|
+
/**
|
|
2327
|
+
* Precomputes the tokenizer-derived data structures used by every
|
|
2328
|
+
* constraint. The first processor per tokenizer otherwise pays this cost
|
|
2329
|
+
* (hundreds of milliseconds for large vocabularies) in its constructor;
|
|
2330
|
+
* call this once after loading the model to pay it early instead.
|
|
2331
|
+
*/
|
|
2332
|
+
static warmup(tokenizer) {
|
|
2333
|
+
prepareTokenizer(tokenizer);
|
|
2334
|
+
}
|
|
2335
|
+
constructor(tokenizer, responseFormat) {
|
|
2336
|
+
super();
|
|
2337
|
+
const state = {
|
|
2338
|
+
constraint: createTokenConstraint(tokenizer, responseFormat)
|
|
2339
|
+
};
|
|
2340
|
+
this.push(new ConstraintLogitsProcessor(state));
|
|
2341
|
+
}
|
|
2342
|
+
};
|
|
2343
|
+
var ConstraintLogitsProcessor = class extends LogitsProcessor {
|
|
2344
|
+
constructor(state) {
|
|
2345
|
+
super();
|
|
2346
|
+
this.state = state;
|
|
2347
|
+
}
|
|
2348
|
+
_call(inputIds, logits) {
|
|
2349
|
+
assertSingleSequence(inputIds.length);
|
|
2350
|
+
const input = inputIds[0];
|
|
2351
|
+
const start = this.state.processedInputLength ?? input.length;
|
|
2352
|
+
for (let i = start; i < input.length; ++i) {
|
|
2353
|
+
if (this.state.constraint.commit(Number(input[i]))) {
|
|
2354
|
+
throw new Error(
|
|
2355
|
+
"StructuredOutputProcessor observed the tokenizer EOS token after generation continued. Ensure the model generation config uses the same eos_token_id as the tokenizer."
|
|
2356
|
+
);
|
|
2357
|
+
}
|
|
2358
|
+
}
|
|
2359
|
+
this.state.processedInputLength = input.length;
|
|
2360
|
+
const logitsVocabSize = logits.dims.at(-1);
|
|
2361
|
+
if (logitsVocabSize === void 0 || !Number.isInteger(logitsVocabSize) || logitsVocabSize <= 0) {
|
|
2362
|
+
throw new Error("StructuredOutputProcessor requires logits with a vocabulary dimension.");
|
|
2363
|
+
}
|
|
2364
|
+
const words = Math.ceil(logitsVocabSize / 32);
|
|
2365
|
+
if (this.state.mask?.length !== words) this.state.mask = new Uint32Array(words);
|
|
2366
|
+
if (!this.state.constraint.fillMask(this.state.mask)) {
|
|
2367
|
+
throw new Error("The constraint reached a dead end before producing a valid output.");
|
|
2368
|
+
}
|
|
2369
|
+
applyMask(logits, this.state.mask, this.state.constraint.vocabSize);
|
|
2370
|
+
const repeatedWhitespace = this.state.constraint.repeatedWhitespace();
|
|
2371
|
+
if (repeatedWhitespace !== void 0) {
|
|
2372
|
+
discourageRepeatedWhitespace(logits, repeatedWhitespace.tokenIds, repeatedWhitespace.count);
|
|
2373
|
+
}
|
|
2374
|
+
return logits;
|
|
2375
|
+
}
|
|
2376
|
+
};
|
|
2377
|
+
function assertSingleSequence(batchSize) {
|
|
2378
|
+
if (batchSize !== 1) {
|
|
2379
|
+
throw new Error(`StructuredOutputProcessor currently supports batch size 1; received ${batchSize}.`);
|
|
2380
|
+
}
|
|
2381
|
+
}
|
|
2382
|
+
function discourageRepeatedWhitespace(logits, tokenIds, count) {
|
|
2383
|
+
const data = logits.data;
|
|
2384
|
+
const stride = logits.dims.at(-1);
|
|
2385
|
+
const penalty = WHITESPACE_REPETITION_PENALTY ** count;
|
|
2386
|
+
for (let offset = 0; offset < data.length; offset += stride) {
|
|
2387
|
+
for (const tokenId2 of tokenIds) {
|
|
2388
|
+
const index = offset + tokenId2;
|
|
2389
|
+
if (count >= MAX_CONSECUTIVE_WHITESPACE_TOKENS) {
|
|
2390
|
+
data[index] = -Infinity;
|
|
2391
|
+
} else if (data[index] < 0) {
|
|
2392
|
+
data[index] *= penalty;
|
|
2393
|
+
} else {
|
|
2394
|
+
data[index] /= penalty;
|
|
2395
|
+
}
|
|
2396
|
+
}
|
|
2397
|
+
}
|
|
2398
|
+
}
|
|
2399
|
+
export {
|
|
2400
|
+
StructuredOutputProcessor
|
|
2401
|
+
};
|
|
2402
|
+
//# sourceMappingURL=index.js.map
|