@tailglow/cli 0.6.5 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -7
- package/package.json +2 -2
- package/src/commands.generated.js +593 -262
- package/src/docs.generated.js +32 -32
- package/src/help.js +7 -1
- package/src/import-command.js +647 -0
- package/src/import-file.js +181 -0
- package/src/import-options.js +24 -0
- package/src/import-worker.generated.js +1616 -0
- package/src/import.generated.d.ts +155 -0
- package/src/import.generated.js +4731 -0
- package/src/index.js +14 -2
|
@@ -0,0 +1,1616 @@
|
|
|
1
|
+
// @ts-nocheck
|
|
2
|
+
// Generated by scripts/build/import.build.js. Do not edit directly.
|
|
3
|
+
|
|
4
|
+
// packages/xml-parser/src/parser.js
|
|
5
|
+
var NAME_START = ":A-Z_a-z\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02FF\\u0370-\\u037D\\u037F-\\u1FFF\\u200C-\\u200D\\u2070-\\u218F\\u2C00-\\u2FEF\\u3001-\\uD7FF\\uF900-\\uFDCF\\uFDF0-\\uFFFD\\u{10000}-\\u{EFFFF}";
|
|
6
|
+
var NAME = new RegExp(`^[${NAME_START}][${NAME_START}\\-.0-9\\u00B7\\u0300-\\u036F\\u203F-\\u2040]*$`, "u");
|
|
7
|
+
var ASCII_NAME = new Uint8Array(128);
|
|
8
|
+
for (let code = 0;code < 128; code++) {
|
|
9
|
+
const char = String.fromCharCode(code);
|
|
10
|
+
if (/[:A-Z_a-z]/.test(char))
|
|
11
|
+
ASCII_NAME[code] = 3;
|
|
12
|
+
else if (/[-.0-9]/.test(char))
|
|
13
|
+
ASCII_NAME[code] = 2;
|
|
14
|
+
}
|
|
15
|
+
var SUSPECT_CHAR = new RegExp("[\\0-\\x08\\x0B\\x0C\\x0E-\\x1F\\uD800-\\uDFFF\\uFFFE\\uFFFF]");
|
|
16
|
+
var INVALID_CHAR = new RegExp("[\\0-\\x08\\x0B\\x0C\\x0E-\\x1F\\uFFFE\\uFFFF]|[\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|" + "(?:^|[^\\uD800-\\uDBFF])[\\uDC00-\\uDFFF]");
|
|
17
|
+
var SPACE = /^[ \t\r\n]*$/;
|
|
18
|
+
var TAG_DELIMITER = /["'>]/g;
|
|
19
|
+
var LINE_END = /\r\n?/g;
|
|
20
|
+
var ATTRIBUTE_SPACE = /\r\n?|[\t\n]/g;
|
|
21
|
+
var ENTITIES = Object.assign(Object.create(null), {
|
|
22
|
+
amp: "&",
|
|
23
|
+
lt: "<",
|
|
24
|
+
gt: ">",
|
|
25
|
+
apos: "'",
|
|
26
|
+
quot: '"'
|
|
27
|
+
});
|
|
28
|
+
var NAME_CACHE_SIZE = 64;
|
|
29
|
+
var DEFAULTS = {
|
|
30
|
+
maxDepth: 256,
|
|
31
|
+
maxTokenLength: 4 * 1024 * 1024,
|
|
32
|
+
maxNameLength: 1024,
|
|
33
|
+
maxAttributes: 4096
|
|
34
|
+
};
|
|
35
|
+
function normalizeLineEnds(value) {
|
|
36
|
+
return value.includes("\r") ? value.replace(LINE_END, `
|
|
37
|
+
`) : value;
|
|
38
|
+
}
|
|
39
|
+
function isXmlChar(cp) {
|
|
40
|
+
if (cp < 32)
|
|
41
|
+
return cp === 9 || cp === 10 || cp === 13;
|
|
42
|
+
return cp <= 55295 || cp >= 57344 && cp <= 65533 || cp >= 65536 && cp <= 1114111;
|
|
43
|
+
}
|
|
44
|
+
function decodeReference(source, start, end) {
|
|
45
|
+
if (source.charCodeAt(start) !== 35)
|
|
46
|
+
return ENTITIES[source.slice(start, end)] ?? null;
|
|
47
|
+
let i = start + 1;
|
|
48
|
+
const is_hex = source.charCodeAt(i) === 120;
|
|
49
|
+
if (is_hex)
|
|
50
|
+
i++;
|
|
51
|
+
if (i >= end)
|
|
52
|
+
return null;
|
|
53
|
+
let cp = 0;
|
|
54
|
+
for (;i < end; i++) {
|
|
55
|
+
const code = source.charCodeAt(i);
|
|
56
|
+
let digit = code - 48;
|
|
57
|
+
if (digit < 0 || digit > 9) {
|
|
58
|
+
const lower = code | 32;
|
|
59
|
+
if (!is_hex || lower < 97 || lower > 102)
|
|
60
|
+
return null;
|
|
61
|
+
digit = lower - 87;
|
|
62
|
+
}
|
|
63
|
+
cp = cp * (is_hex ? 16 : 10) + digit;
|
|
64
|
+
if (cp > 1114111)
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
return isXmlChar(cp) ? String.fromCodePoint(cp) : null;
|
|
68
|
+
}
|
|
69
|
+
function decodeReferences(value) {
|
|
70
|
+
let result = "";
|
|
71
|
+
let cursor = 0;
|
|
72
|
+
for (let amp = value.indexOf("&");amp >= 0; amp = value.indexOf("&", cursor)) {
|
|
73
|
+
const semicolon = value.indexOf(";", amp + 1);
|
|
74
|
+
if (semicolon < 0)
|
|
75
|
+
return null;
|
|
76
|
+
const decoded = decodeReference(value, amp + 1, semicolon);
|
|
77
|
+
if (decoded === null)
|
|
78
|
+
return null;
|
|
79
|
+
result += value.slice(cursor, amp) + decoded;
|
|
80
|
+
cursor = semicolon + 1;
|
|
81
|
+
}
|
|
82
|
+
return result + value.slice(cursor);
|
|
83
|
+
}
|
|
84
|
+
function endsAttributeName(code) {
|
|
85
|
+
return code === 61 || code === 32 || code === 9 || code === 10 || code === 13;
|
|
86
|
+
}
|
|
87
|
+
function skipSpace(source, i, limit) {
|
|
88
|
+
while (i < limit) {
|
|
89
|
+
const code = source.charCodeAt(i);
|
|
90
|
+
if (code !== 32 && code !== 9 && code !== 10 && code !== 13)
|
|
91
|
+
break;
|
|
92
|
+
i++;
|
|
93
|
+
}
|
|
94
|
+
return i;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
class XmlParseError extends Error {
|
|
98
|
+
constructor(message, code, offset) {
|
|
99
|
+
super(message);
|
|
100
|
+
this.name = "XmlParseError";
|
|
101
|
+
this.code = code;
|
|
102
|
+
this.offset = offset;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
class XmlStreamParser {
|
|
107
|
+
constructor(handlers = {}, options = {}) {
|
|
108
|
+
this.handlers = handlers;
|
|
109
|
+
this.options = { ...DEFAULTS, ...options, doctype: options.doctype ?? "prohibit" };
|
|
110
|
+
for (const key of Object.keys(DEFAULTS)) {
|
|
111
|
+
const value = this.options[key];
|
|
112
|
+
if (!Number.isInteger(value) || value <= 0)
|
|
113
|
+
throw new TypeError(`${key} must be a positive finite integer`);
|
|
114
|
+
}
|
|
115
|
+
if (!["prohibit", "ignore"].includes(this.options.doctype))
|
|
116
|
+
throw new TypeError('doctype must be "prohibit" or "ignore"');
|
|
117
|
+
this._offset = 0;
|
|
118
|
+
this._mode = "";
|
|
119
|
+
this._decoder = null;
|
|
120
|
+
this._surrogate = "";
|
|
121
|
+
this._ended = false;
|
|
122
|
+
this._failed = false;
|
|
123
|
+
this._busy = false;
|
|
124
|
+
this._stack = [];
|
|
125
|
+
this._root = false;
|
|
126
|
+
this._is_declaration_allowed = true;
|
|
127
|
+
this._doctype = false;
|
|
128
|
+
this._kind = "";
|
|
129
|
+
this._parts = [];
|
|
130
|
+
this._length = 0;
|
|
131
|
+
this._head = "";
|
|
132
|
+
this._tail = "";
|
|
133
|
+
this._quote = "";
|
|
134
|
+
this._brackets = 0;
|
|
135
|
+
this._dtdContext = "";
|
|
136
|
+
this._entityParts = null;
|
|
137
|
+
this._entityLength = 0;
|
|
138
|
+
this._textBrackets = 0;
|
|
139
|
+
this._textCR = false;
|
|
140
|
+
this._scannedName = "";
|
|
141
|
+
this._scannedAttributes = null;
|
|
142
|
+
this._scannedEmpty = false;
|
|
143
|
+
this._attributeNames = [];
|
|
144
|
+
}
|
|
145
|
+
get offset() {
|
|
146
|
+
return this._offset;
|
|
147
|
+
}
|
|
148
|
+
write(chunk) {
|
|
149
|
+
this._guard();
|
|
150
|
+
this._busy = true;
|
|
151
|
+
try {
|
|
152
|
+
this._input(chunk);
|
|
153
|
+
} catch (error) {
|
|
154
|
+
this._failed = true;
|
|
155
|
+
throw error;
|
|
156
|
+
} finally {
|
|
157
|
+
this._busy = false;
|
|
158
|
+
}
|
|
159
|
+
return this;
|
|
160
|
+
}
|
|
161
|
+
end(chunk) {
|
|
162
|
+
this._guard();
|
|
163
|
+
this._busy = true;
|
|
164
|
+
try {
|
|
165
|
+
if (chunk !== undefined)
|
|
166
|
+
this._input(chunk);
|
|
167
|
+
if (this._decoder) {
|
|
168
|
+
let final;
|
|
169
|
+
try {
|
|
170
|
+
final = this._decoder.decode();
|
|
171
|
+
} catch {
|
|
172
|
+
this._error("Truncated UTF-8 input", "UTF8");
|
|
173
|
+
}
|
|
174
|
+
if (final)
|
|
175
|
+
this._decoded(final);
|
|
176
|
+
}
|
|
177
|
+
if (this._surrogate)
|
|
178
|
+
this._error("Unpaired UTF-16 surrogate", "CHARACTER");
|
|
179
|
+
if (this._kind || this._entityParts || this._stack.length || !this._root)
|
|
180
|
+
this._error("Incomplete XML document", "EOF");
|
|
181
|
+
} catch (error) {
|
|
182
|
+
this._failed = true;
|
|
183
|
+
throw error;
|
|
184
|
+
} finally {
|
|
185
|
+
this._ended = true;
|
|
186
|
+
this._busy = false;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
_guard() {
|
|
190
|
+
if (this._ended || this._failed || this._busy) {
|
|
191
|
+
this._failed = true;
|
|
192
|
+
this._error("Parser is ended, failed, or already executing a callback", "STATE");
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
_error(message, code = "SYNTAX", offset = this._offset) {
|
|
196
|
+
throw new XmlParseError(message, code, offset);
|
|
197
|
+
}
|
|
198
|
+
_emit(name, ...args) {
|
|
199
|
+
this.handlers[name]?.(...args);
|
|
200
|
+
this._afterCallback();
|
|
201
|
+
}
|
|
202
|
+
_emitText(value, kind = "literal") {
|
|
203
|
+
this.handlers.text?.(value, kind);
|
|
204
|
+
this._afterCallback();
|
|
205
|
+
}
|
|
206
|
+
_afterCallback() {
|
|
207
|
+
if (this._failed)
|
|
208
|
+
this._error("Parser failed during callback", "STATE");
|
|
209
|
+
}
|
|
210
|
+
_input(chunk) {
|
|
211
|
+
const mode = typeof chunk === "string" ? "string" : chunk instanceof Uint8Array ? "bytes" : "";
|
|
212
|
+
if (!mode)
|
|
213
|
+
throw new TypeError("XML input must be a string or Uint8Array");
|
|
214
|
+
if (!chunk.length)
|
|
215
|
+
return;
|
|
216
|
+
if (this._mode && this._mode !== mode)
|
|
217
|
+
this._error("Cannot mix strings and UTF-8 bytes", "INPUT_MODE");
|
|
218
|
+
this._mode = mode;
|
|
219
|
+
if (mode === "bytes") {
|
|
220
|
+
this._decoder ??= new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
|
221
|
+
try {
|
|
222
|
+
chunk = this._decoder.decode(chunk, { stream: true });
|
|
223
|
+
} catch {
|
|
224
|
+
this._error("Invalid UTF-8 input", "UTF8");
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
this._decoded(chunk);
|
|
228
|
+
}
|
|
229
|
+
_decoded(value) {
|
|
230
|
+
if (!value)
|
|
231
|
+
return;
|
|
232
|
+
if (this._surrogate) {
|
|
233
|
+
value = this._surrogate + value;
|
|
234
|
+
this._surrogate = "";
|
|
235
|
+
}
|
|
236
|
+
const last = value.charCodeAt(value.length - 1);
|
|
237
|
+
if (last >= 55296 && last <= 56319) {
|
|
238
|
+
this._surrogate = value.slice(-1);
|
|
239
|
+
value = value.slice(0, -1);
|
|
240
|
+
}
|
|
241
|
+
if (SUSPECT_CHAR.test(value)) {
|
|
242
|
+
const invalid = INVALID_CHAR.exec(value);
|
|
243
|
+
if (invalid)
|
|
244
|
+
this._error("Invalid XML character", "CHARACTER", this._offset + invalid.index + invalid[0].length - 1);
|
|
245
|
+
}
|
|
246
|
+
let i = 0;
|
|
247
|
+
if (this._offset === 0 && value.charCodeAt(0) === 65279) {
|
|
248
|
+
i++;
|
|
249
|
+
this._offset++;
|
|
250
|
+
}
|
|
251
|
+
while (i < value.length) {
|
|
252
|
+
if (this._kind) {
|
|
253
|
+
i = this._continueToken(value, i);
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
const open = value.indexOf("<", i);
|
|
257
|
+
const end = open < 0 ? value.length : open;
|
|
258
|
+
if (end > i) {
|
|
259
|
+
this._text(value.slice(i, end));
|
|
260
|
+
this._offset += end - i;
|
|
261
|
+
i = end;
|
|
262
|
+
}
|
|
263
|
+
if (i === value.length)
|
|
264
|
+
break;
|
|
265
|
+
if (this._entityParts)
|
|
266
|
+
this._error("Unterminated entity reference", "ENTITY");
|
|
267
|
+
this._textBrackets = 0;
|
|
268
|
+
this._textCR = false;
|
|
269
|
+
const next = this._markup(value, i);
|
|
270
|
+
if (next >= 0) {
|
|
271
|
+
i = next;
|
|
272
|
+
continue;
|
|
273
|
+
}
|
|
274
|
+
this._kind = "prefix";
|
|
275
|
+
this._tokenStart = this._offset;
|
|
276
|
+
this._parts = [];
|
|
277
|
+
this._length = 0;
|
|
278
|
+
this._head = "";
|
|
279
|
+
this._tail = "";
|
|
280
|
+
this._quote = "";
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
_markup(value, open) {
|
|
284
|
+
const limit = Math.min(value.length, open + this.options.maxTokenLength);
|
|
285
|
+
const next = value.charCodeAt(open + 1);
|
|
286
|
+
let end;
|
|
287
|
+
if (next === 33) {
|
|
288
|
+
if (value.startsWith("<!--", open)) {
|
|
289
|
+
const close = value.indexOf("--", open + 4);
|
|
290
|
+
end = close + 3;
|
|
291
|
+
if (close < 0 || end > limit || value.charCodeAt(close + 2) !== 62)
|
|
292
|
+
return -1;
|
|
293
|
+
this._offset += end - open;
|
|
294
|
+
this._is_declaration_allowed = false;
|
|
295
|
+
this._comment(value, open + 4, close);
|
|
296
|
+
return end;
|
|
297
|
+
}
|
|
298
|
+
if (!this._stack.length || !value.startsWith("<![CDATA[", open))
|
|
299
|
+
return -1;
|
|
300
|
+
const close = value.indexOf("]]>", open + 9);
|
|
301
|
+
end = close + 3;
|
|
302
|
+
if (close < 0 || end > limit)
|
|
303
|
+
return -1;
|
|
304
|
+
this._offset += end - open;
|
|
305
|
+
this._is_declaration_allowed = false;
|
|
306
|
+
this._cdata(value, open + 9, close);
|
|
307
|
+
return end;
|
|
308
|
+
}
|
|
309
|
+
if (next === 63) {
|
|
310
|
+
const close = value.indexOf("?>", open + 1);
|
|
311
|
+
end = close + 2;
|
|
312
|
+
if (close < 0 || end > limit)
|
|
313
|
+
return -1;
|
|
314
|
+
this._offset += end - open;
|
|
315
|
+
this._token(value.slice(open, end), "pi");
|
|
316
|
+
return end;
|
|
317
|
+
}
|
|
318
|
+
end = this._scanTag(value, open, limit);
|
|
319
|
+
if (end < 0)
|
|
320
|
+
return -1;
|
|
321
|
+
this._offset += end - open;
|
|
322
|
+
this._is_declaration_allowed = false;
|
|
323
|
+
this._applyTag();
|
|
324
|
+
return end;
|
|
325
|
+
}
|
|
326
|
+
_scanTag(source, start, limit) {
|
|
327
|
+
let i = start + 1;
|
|
328
|
+
if (source.charCodeAt(i) === 47) {
|
|
329
|
+
const name = this._stack[this._stack.length - 1];
|
|
330
|
+
if (name === undefined || !source.startsWith(name, ++i))
|
|
331
|
+
return -1;
|
|
332
|
+
i = skipSpace(source, i + name.length, limit);
|
|
333
|
+
if (i >= limit || source.charCodeAt(i) !== 62)
|
|
334
|
+
return -1;
|
|
335
|
+
this._scannedName = name;
|
|
336
|
+
this._scannedAttributes = null;
|
|
337
|
+
return i + 1;
|
|
338
|
+
}
|
|
339
|
+
const nameEnd = this._nameEnd(source, i, limit);
|
|
340
|
+
if (nameEnd < 0)
|
|
341
|
+
return -1;
|
|
342
|
+
const attributes = Object.create(null);
|
|
343
|
+
let count = 0;
|
|
344
|
+
let is_empty = false;
|
|
345
|
+
i = nameEnd;
|
|
346
|
+
for (;; ) {
|
|
347
|
+
const before = i;
|
|
348
|
+
i = skipSpace(source, i, limit);
|
|
349
|
+
if (i >= limit)
|
|
350
|
+
return -1;
|
|
351
|
+
const code = source.charCodeAt(i);
|
|
352
|
+
if (code === 62)
|
|
353
|
+
break;
|
|
354
|
+
if (code === 47) {
|
|
355
|
+
if (i + 1 >= limit || source.charCodeAt(i + 1) !== 62)
|
|
356
|
+
return -1;
|
|
357
|
+
is_empty = true;
|
|
358
|
+
i++;
|
|
359
|
+
break;
|
|
360
|
+
}
|
|
361
|
+
if (i === before)
|
|
362
|
+
return -1;
|
|
363
|
+
let attribute = count < NAME_CACHE_SIZE ? this._attributeNames[count] : undefined;
|
|
364
|
+
let attribute_end = attribute === undefined ? -1 : i + attribute.length;
|
|
365
|
+
if (attribute_end < 0 || !endsAttributeName(source.charCodeAt(attribute_end)) || !source.startsWith(attribute, i)) {
|
|
366
|
+
attribute_end = this._nameEnd(source, i, limit);
|
|
367
|
+
if (attribute_end < 0)
|
|
368
|
+
return -1;
|
|
369
|
+
attribute = source.slice(i, attribute_end);
|
|
370
|
+
if (count < NAME_CACHE_SIZE)
|
|
371
|
+
this._attributeNames[count] = attribute;
|
|
372
|
+
}
|
|
373
|
+
if (attributes[attribute] !== undefined || ++count > this.options.maxAttributes)
|
|
374
|
+
return -1;
|
|
375
|
+
i = skipSpace(source, attribute_end, limit);
|
|
376
|
+
if (i >= limit || source.charCodeAt(i) !== 61)
|
|
377
|
+
return -1;
|
|
378
|
+
i = skipSpace(source, i + 1, limit);
|
|
379
|
+
if (i >= limit)
|
|
380
|
+
return -1;
|
|
381
|
+
const quote = source.charCodeAt(i);
|
|
382
|
+
if (quote !== 34 && quote !== 39)
|
|
383
|
+
return -1;
|
|
384
|
+
const value_start = ++i;
|
|
385
|
+
let has_spaces = false;
|
|
386
|
+
let has_references = false;
|
|
387
|
+
for (;i < limit; i++) {
|
|
388
|
+
const unit = source.charCodeAt(i);
|
|
389
|
+
if (unit === quote)
|
|
390
|
+
break;
|
|
391
|
+
if (unit <= 38) {
|
|
392
|
+
if (unit === 38)
|
|
393
|
+
has_references = true;
|
|
394
|
+
else if (unit === 9 || unit === 10 || unit === 13)
|
|
395
|
+
has_spaces = true;
|
|
396
|
+
} else if (unit === 60)
|
|
397
|
+
return -1;
|
|
398
|
+
}
|
|
399
|
+
if (i >= limit)
|
|
400
|
+
return -1;
|
|
401
|
+
let value = source.slice(value_start, i);
|
|
402
|
+
if (has_spaces)
|
|
403
|
+
value = value.replace(ATTRIBUTE_SPACE, " ");
|
|
404
|
+
if (has_references) {
|
|
405
|
+
value = decodeReferences(value);
|
|
406
|
+
if (value === null)
|
|
407
|
+
return -1;
|
|
408
|
+
}
|
|
409
|
+
attributes[attribute] = value;
|
|
410
|
+
i++;
|
|
411
|
+
}
|
|
412
|
+
this._scannedName = source.slice(start + 1, nameEnd);
|
|
413
|
+
this._scannedAttributes = attributes;
|
|
414
|
+
this._scannedEmpty = is_empty;
|
|
415
|
+
return i + 1;
|
|
416
|
+
}
|
|
417
|
+
_nameEnd(source, start, limit) {
|
|
418
|
+
if (start >= limit)
|
|
419
|
+
return -1;
|
|
420
|
+
let code = source.charCodeAt(start);
|
|
421
|
+
let is_ascii = code < 128;
|
|
422
|
+
if (is_ascii && !(ASCII_NAME[code] & 1))
|
|
423
|
+
return -1;
|
|
424
|
+
let i = start + 1;
|
|
425
|
+
for (;i < limit; i++) {
|
|
426
|
+
code = source.charCodeAt(i);
|
|
427
|
+
if (code >= 128)
|
|
428
|
+
is_ascii = false;
|
|
429
|
+
else if (!(ASCII_NAME[code] & 2))
|
|
430
|
+
break;
|
|
431
|
+
}
|
|
432
|
+
if (i >= limit || i - start > this.options.maxNameLength)
|
|
433
|
+
return -1;
|
|
434
|
+
if (!is_ascii && !NAME.test(source.slice(start, i)))
|
|
435
|
+
return -1;
|
|
436
|
+
return i;
|
|
437
|
+
}
|
|
438
|
+
_applyTag() {
|
|
439
|
+
const attributes = this._scannedAttributes;
|
|
440
|
+
if (attributes === null) {
|
|
441
|
+
this._closeElement(this._scannedName);
|
|
442
|
+
return;
|
|
443
|
+
}
|
|
444
|
+
this._scannedAttributes = null;
|
|
445
|
+
this._openElement(this._scannedName, attributes, this._scannedEmpty);
|
|
446
|
+
}
|
|
447
|
+
_openElement(name, attributes, is_empty) {
|
|
448
|
+
if (!this._stack.length && this._root)
|
|
449
|
+
this._error("Multiple document elements");
|
|
450
|
+
if (this._stack.length + 1 > this.options.maxDepth)
|
|
451
|
+
this._error("Element exceeds maxDepth", "LIMIT");
|
|
452
|
+
this._root = true;
|
|
453
|
+
if (!is_empty)
|
|
454
|
+
this._stack.push(name);
|
|
455
|
+
this.handlers.startTag?.(name, attributes);
|
|
456
|
+
this._afterCallback();
|
|
457
|
+
if (is_empty) {
|
|
458
|
+
this.handlers.endTag?.(name);
|
|
459
|
+
this._afterCallback();
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
_closeElement(name) {
|
|
463
|
+
if (this._stack.pop() !== name)
|
|
464
|
+
this._error("Mismatched end tag");
|
|
465
|
+
this.handlers.endTag?.(name);
|
|
466
|
+
this._afterCallback();
|
|
467
|
+
}
|
|
468
|
+
_comment(source, start, end) {
|
|
469
|
+
const handler = this.handlers.comment;
|
|
470
|
+
if (handler != null)
|
|
471
|
+
Reflect.apply(handler, this.handlers, [normalizeLineEnds(source.slice(start, end))]);
|
|
472
|
+
this._afterCallback();
|
|
473
|
+
}
|
|
474
|
+
_cdata(source, start, end) {
|
|
475
|
+
if (end > start)
|
|
476
|
+
this._emitText(normalizeLineEnds(source.slice(start, end)), "cdata");
|
|
477
|
+
}
|
|
478
|
+
_continueToken(value, i) {
|
|
479
|
+
const start = i;
|
|
480
|
+
let is_done = false;
|
|
481
|
+
while (i < value.length && !is_done) {
|
|
482
|
+
const kind = this._kind;
|
|
483
|
+
if (kind === "comment" || kind === "cdata" || kind === "pi") {
|
|
484
|
+
const end = this._sectionEnd(value, i);
|
|
485
|
+
is_done = end >= 0;
|
|
486
|
+
const next = is_done ? end : value.length;
|
|
487
|
+
this._consumeSection(next - i);
|
|
488
|
+
i = next;
|
|
489
|
+
continue;
|
|
490
|
+
}
|
|
491
|
+
if (kind === "tag") {
|
|
492
|
+
let next;
|
|
493
|
+
if (this._quote) {
|
|
494
|
+
next = value.indexOf(this._quote, i);
|
|
495
|
+
if (next >= 0)
|
|
496
|
+
this._quote = "";
|
|
497
|
+
} else {
|
|
498
|
+
TAG_DELIMITER.lastIndex = i;
|
|
499
|
+
const delimiter = TAG_DELIMITER.exec(value);
|
|
500
|
+
next = delimiter ? delimiter.index : -1;
|
|
501
|
+
if (delimiter) {
|
|
502
|
+
if (delimiter[0] === ">")
|
|
503
|
+
is_done = true;
|
|
504
|
+
else
|
|
505
|
+
this._quote = delimiter[0];
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
next = next < 0 ? value.length : next + 1;
|
|
509
|
+
this._length += next - i;
|
|
510
|
+
this._offset += next - i;
|
|
511
|
+
i = next;
|
|
512
|
+
if (this._length > this.options.maxTokenLength)
|
|
513
|
+
this._error("XML token exceeds maxTokenLength", "LIMIT");
|
|
514
|
+
continue;
|
|
515
|
+
}
|
|
516
|
+
const c = value[i++];
|
|
517
|
+
this._offset++;
|
|
518
|
+
if (++this._length > this.options.maxTokenLength)
|
|
519
|
+
this._error("XML token exceeds maxTokenLength", "LIMIT");
|
|
520
|
+
is_done = this._frame(c);
|
|
521
|
+
}
|
|
522
|
+
this._parts.push(value.slice(start, i));
|
|
523
|
+
if (is_done) {
|
|
524
|
+
const token = this._parts.length === 1 ? this._parts[0] : this._parts.join("");
|
|
525
|
+
const kind = this._kind;
|
|
526
|
+
this._kind = "";
|
|
527
|
+
this._parts = [];
|
|
528
|
+
this._token(token, kind);
|
|
529
|
+
}
|
|
530
|
+
return i;
|
|
531
|
+
}
|
|
532
|
+
_sectionEnd(value, i) {
|
|
533
|
+
const tail = this._tail;
|
|
534
|
+
const text = tail ? tail + value.slice(i) : value;
|
|
535
|
+
const from = tail ? 0 : i;
|
|
536
|
+
const shift = tail ? i - tail.length : 0;
|
|
537
|
+
let end = -1;
|
|
538
|
+
if (this._kind === "comment") {
|
|
539
|
+
const hyphens = text.indexOf("--", from);
|
|
540
|
+
if (hyphens >= 0 && hyphens + 2 < text.length) {
|
|
541
|
+
end = hyphens + 3 + shift;
|
|
542
|
+
if (text.charCodeAt(hyphens + 2) !== 62) {
|
|
543
|
+
this._consumeSection(end - i);
|
|
544
|
+
this._error("Double hyphen in comment");
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
} else {
|
|
548
|
+
const terminator = this._kind === "cdata" ? "]]>" : "?>";
|
|
549
|
+
const found = text.indexOf(terminator, from);
|
|
550
|
+
if (found >= 0)
|
|
551
|
+
end = found + terminator.length + shift;
|
|
552
|
+
}
|
|
553
|
+
if (end < 0)
|
|
554
|
+
this._tail = text.slice(Math.max(from, text.length - 2));
|
|
555
|
+
return end;
|
|
556
|
+
}
|
|
557
|
+
_consumeSection(count) {
|
|
558
|
+
const allowed = this.options.maxTokenLength - this._length;
|
|
559
|
+
if (count > allowed)
|
|
560
|
+
this._error("XML token exceeds maxTokenLength", "LIMIT", this._offset + allowed + 1);
|
|
561
|
+
this._length += count;
|
|
562
|
+
this._offset += count;
|
|
563
|
+
}
|
|
564
|
+
_frame(c) {
|
|
565
|
+
const tail = this._tail;
|
|
566
|
+
this._tail = (tail + c).slice(-3);
|
|
567
|
+
if (this._kind === "prefix") {
|
|
568
|
+
this._head += c;
|
|
569
|
+
const h = this._head;
|
|
570
|
+
if (h.length === 2 && c !== "!") {
|
|
571
|
+
this._kind = c === "?" ? "pi" : "tag";
|
|
572
|
+
return c === ">";
|
|
573
|
+
}
|
|
574
|
+
if (h === "<!--") {
|
|
575
|
+
this._kind = "comment";
|
|
576
|
+
this._tail = "";
|
|
577
|
+
} else if (h === "<![CDATA[")
|
|
578
|
+
this._kind = "cdata";
|
|
579
|
+
else if (h === "<!DOCTYPE") {
|
|
580
|
+
if (this.options.doctype === "prohibit")
|
|
581
|
+
this._error("DOCTYPE is prohibited", "DOCTYPE");
|
|
582
|
+
this._kind = "doctype";
|
|
583
|
+
this._brackets = 0;
|
|
584
|
+
this._dtdContext = "";
|
|
585
|
+
} else if (!["<!--", "<![CDATA[", "<!DOCTYPE"].some((prefix) => prefix.startsWith(h)) && h.length > 1) {
|
|
586
|
+
this._error("Invalid markup declaration");
|
|
587
|
+
}
|
|
588
|
+
return false;
|
|
589
|
+
}
|
|
590
|
+
if (this._dtdContext === "comment") {
|
|
591
|
+
if (tail.endsWith("--") && c !== ">")
|
|
592
|
+
this._error("Double hyphen in DTD comment");
|
|
593
|
+
if (this._tail === "-->")
|
|
594
|
+
this._dtdContext = "";
|
|
595
|
+
return false;
|
|
596
|
+
}
|
|
597
|
+
if (this._dtdContext === "pi") {
|
|
598
|
+
if (this._tail.endsWith("?>"))
|
|
599
|
+
this._dtdContext = "";
|
|
600
|
+
return false;
|
|
601
|
+
}
|
|
602
|
+
if (!this._quote && tail === "<!-" && c === "-") {
|
|
603
|
+
this._dtdContext = "comment";
|
|
604
|
+
this._tail = "";
|
|
605
|
+
return false;
|
|
606
|
+
}
|
|
607
|
+
if (!this._quote && tail.endsWith("<") && c === "?") {
|
|
608
|
+
this._dtdContext = "pi";
|
|
609
|
+
return false;
|
|
610
|
+
}
|
|
611
|
+
if (this._quote) {
|
|
612
|
+
if (c === this._quote)
|
|
613
|
+
this._quote = "";
|
|
614
|
+
return false;
|
|
615
|
+
}
|
|
616
|
+
if (c === '"' || c === "'") {
|
|
617
|
+
this._quote = c;
|
|
618
|
+
return false;
|
|
619
|
+
}
|
|
620
|
+
if (c === "[")
|
|
621
|
+
this._brackets++;
|
|
622
|
+
if (c === "]" && --this._brackets < 0)
|
|
623
|
+
this._error("Unmatched DOCTYPE bracket");
|
|
624
|
+
return c === ">" && this._brackets === 0;
|
|
625
|
+
}
|
|
626
|
+
_text(value) {
|
|
627
|
+
this._is_declaration_allowed = false;
|
|
628
|
+
if (!this._stack.length) {
|
|
629
|
+
if (!SPACE.test(value))
|
|
630
|
+
this._error("Non-whitespace outside document element");
|
|
631
|
+
this._literal(value);
|
|
632
|
+
return;
|
|
633
|
+
}
|
|
634
|
+
let i = 0;
|
|
635
|
+
if (this._entityParts) {
|
|
636
|
+
const semicolon = value.indexOf(";");
|
|
637
|
+
const end = semicolon < 0 ? value.length : semicolon;
|
|
638
|
+
this._entityLength += end;
|
|
639
|
+
if (this._entityLength + 2 > this.options.maxTokenLength)
|
|
640
|
+
this._error("Entity exceeds maxTokenLength", "LIMIT");
|
|
641
|
+
this._entityParts.push(value.slice(0, end));
|
|
642
|
+
if (semicolon < 0)
|
|
643
|
+
return;
|
|
644
|
+
const name = this._entityParts.join("");
|
|
645
|
+
this._entityParts = null;
|
|
646
|
+
this._reference(this._entity(name));
|
|
647
|
+
i = semicolon + 1;
|
|
648
|
+
}
|
|
649
|
+
while (i < value.length) {
|
|
650
|
+
const amp = value.indexOf("&", i);
|
|
651
|
+
if (amp < 0) {
|
|
652
|
+
this._literal(i ? value.slice(i) : value);
|
|
653
|
+
return;
|
|
654
|
+
}
|
|
655
|
+
if (amp > i)
|
|
656
|
+
this._literal(value.slice(i, amp));
|
|
657
|
+
i = amp + 1;
|
|
658
|
+
if (i === value.length) {
|
|
659
|
+
this._entityParts = [];
|
|
660
|
+
this._entityLength = 0;
|
|
661
|
+
return;
|
|
662
|
+
}
|
|
663
|
+
const semicolon = value.indexOf(";", i);
|
|
664
|
+
const end = semicolon < 0 ? value.length : semicolon;
|
|
665
|
+
if (end - i + 2 > this.options.maxTokenLength)
|
|
666
|
+
this._error("Entity exceeds maxTokenLength", "LIMIT");
|
|
667
|
+
if (semicolon < 0) {
|
|
668
|
+
this._entityParts = [value.slice(i)];
|
|
669
|
+
this._entityLength = end - i;
|
|
670
|
+
return;
|
|
671
|
+
}
|
|
672
|
+
this._reference(decodeReference(value, i, semicolon) ?? this._entity(value.slice(i, semicolon)));
|
|
673
|
+
i = semicolon + 1;
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
_reference(value) {
|
|
677
|
+
this._textBrackets = 0;
|
|
678
|
+
this._textCR = false;
|
|
679
|
+
this._emitText(value, "reference");
|
|
680
|
+
}
|
|
681
|
+
_literal(value) {
|
|
682
|
+
const length = value.length;
|
|
683
|
+
const first = value.charCodeAt(0);
|
|
684
|
+
const brackets = this._textBrackets;
|
|
685
|
+
const is_straddling = brackets > 0 && (brackets === 2 && first === 62 || first === 93 && value.charCodeAt(1) === 62);
|
|
686
|
+
if (is_straddling || value.includes("]]>"))
|
|
687
|
+
this._error("CDATA close delimiter in text");
|
|
688
|
+
let trailing = 0;
|
|
689
|
+
while (trailing < 2 && trailing < length && value.charCodeAt(length - 1 - trailing) === 93)
|
|
690
|
+
trailing++;
|
|
691
|
+
this._textBrackets = trailing === length ? Math.min(2, brackets + trailing) : trailing;
|
|
692
|
+
const has_pending_cr = this._textCR;
|
|
693
|
+
this._textCR = value.charCodeAt(length - 1) === 13;
|
|
694
|
+
if (has_pending_cr && first === 10)
|
|
695
|
+
value = value.slice(1);
|
|
696
|
+
value = normalizeLineEnds(value);
|
|
697
|
+
if (value)
|
|
698
|
+
this._emitText(value);
|
|
699
|
+
}
|
|
700
|
+
_entity(name) {
|
|
701
|
+
const value = decodeReference(name, 0, name.length);
|
|
702
|
+
if (value !== null)
|
|
703
|
+
return value;
|
|
704
|
+
if (!/^#(?:[0-9]+|x[0-9a-fA-F]+)$/.test(name))
|
|
705
|
+
this._error("Unknown or malformed entity reference", "ENTITY");
|
|
706
|
+
const cp = name[1] === "x" ? Number.parseInt(name.slice(2), 16) : Number(name.slice(1));
|
|
707
|
+
if (!Number.isSafeInteger(cp) || cp > 1114111 || cp < 0)
|
|
708
|
+
this._error("Invalid character reference", "ENTITY");
|
|
709
|
+
this._error("Invalid XML character reference", "CHARACTER");
|
|
710
|
+
}
|
|
711
|
+
_name(name) {
|
|
712
|
+
if (name.length > this.options.maxNameLength)
|
|
713
|
+
this._error("Name exceeds maxNameLength", "LIMIT");
|
|
714
|
+
if (!NAME.test(name))
|
|
715
|
+
this._error("Invalid XML name");
|
|
716
|
+
return name;
|
|
717
|
+
}
|
|
718
|
+
_token(raw, kind) {
|
|
719
|
+
const is_declaration_allowed = this._is_declaration_allowed;
|
|
720
|
+
this._is_declaration_allowed = false;
|
|
721
|
+
if (kind === "tag") {
|
|
722
|
+
if (this._scanTag(raw, 0, raw.length) === raw.length)
|
|
723
|
+
this._applyTag();
|
|
724
|
+
else
|
|
725
|
+
this._tag(normalizeLineEnds(raw));
|
|
726
|
+
return;
|
|
727
|
+
}
|
|
728
|
+
if (kind === "comment") {
|
|
729
|
+
this._comment(raw, 4, raw.length - 3);
|
|
730
|
+
return;
|
|
731
|
+
}
|
|
732
|
+
if (kind === "cdata") {
|
|
733
|
+
if (!this._stack.length)
|
|
734
|
+
this._error("CDATA outside document element");
|
|
735
|
+
this._cdata(raw, 9, raw.length - 3);
|
|
736
|
+
return;
|
|
737
|
+
}
|
|
738
|
+
const token = normalizeLineEnds(raw);
|
|
739
|
+
if (kind === "pi") {
|
|
740
|
+
const match = /^<\?([^ \t\n?]+)(?:[ \t\n]+([\s\S]*))?\?>$/.exec(token);
|
|
741
|
+
if (!match)
|
|
742
|
+
this._error("Malformed processing instruction");
|
|
743
|
+
const target = this._name(match[1]);
|
|
744
|
+
if (target.toLowerCase() === "xml") {
|
|
745
|
+
if (target !== "xml" || !is_declaration_allowed)
|
|
746
|
+
this._error("Misplaced XML declaration");
|
|
747
|
+
const declaration = /^<\?xml[ \t\n]+version[ \t\n]*=[ \t\n]*(['"])1\.0\1(?:[ \t\n]+encoding[ \t\n]*=[ \t\n]*(['"])([A-Za-z][A-Za-z0-9._-]*)\2)?(?:[ \t\n]+standalone[ \t\n]*=[ \t\n]*(['"])(?:yes|no)\4)?[ \t\n]*\?>$/.exec(token);
|
|
748
|
+
if (!declaration)
|
|
749
|
+
this._error("Unsupported or malformed XML declaration", "DECLARATION");
|
|
750
|
+
if (this._mode === "bytes" && declaration[3] && declaration[3].toLowerCase() !== "utf-8")
|
|
751
|
+
this._error("Only UTF-8 byte encoding is supported", "ENCODING");
|
|
752
|
+
} else
|
|
753
|
+
this._emit("processingInstruction", target, match[2] ?? "");
|
|
754
|
+
return;
|
|
755
|
+
}
|
|
756
|
+
this._parseDoctype(token, raw);
|
|
757
|
+
}
|
|
758
|
+
_parseDoctype(token, raw) {
|
|
759
|
+
if (this._doctype || this._root)
|
|
760
|
+
this._error("Misplaced or duplicate DOCTYPE", "DOCTYPE");
|
|
761
|
+
const header = /^<!DOCTYPE[ \t\n]+([^ \t\n\[>]+)([\s\S]*)>$/.exec(token);
|
|
762
|
+
if (!header)
|
|
763
|
+
this._error("Malformed DOCTYPE", "DOCTYPE");
|
|
764
|
+
this._name(header[1]);
|
|
765
|
+
let rest = header[2];
|
|
766
|
+
const external = /^[ \t\n]+(?:SYSTEM[ \t\n]+(?:"[^"]*"|'[^']*')|PUBLIC[ \t\n]+(?:"[\r\na-zA-Z0-9 \-'()+,./:=?;!*#@$_%]*"|'[\r\na-zA-Z0-9 \-()+,./:=?;!*#@$_%]*')[ \t\n]+(?:"[^"]*"|'[^']*'))/.exec(rest);
|
|
767
|
+
if (external)
|
|
768
|
+
rest = rest.slice(external[0].length);
|
|
769
|
+
const start = skipSpace(rest, 0, rest.length);
|
|
770
|
+
let end = rest.length;
|
|
771
|
+
while (end > start) {
|
|
772
|
+
const code = rest.charCodeAt(end - 1);
|
|
773
|
+
if (code !== 32 && code !== 9 && code !== 10 && code !== 13)
|
|
774
|
+
break;
|
|
775
|
+
end--;
|
|
776
|
+
}
|
|
777
|
+
rest = rest.slice(start, end);
|
|
778
|
+
if (rest && !(rest.startsWith("[") && rest.endsWith("]")))
|
|
779
|
+
this._error("Malformed DOCTYPE framing", "DOCTYPE");
|
|
780
|
+
this._doctype = true;
|
|
781
|
+
this._emit("doctype", raw);
|
|
782
|
+
}
|
|
783
|
+
_tag(token) {
|
|
784
|
+
if (token.startsWith("</")) {
|
|
785
|
+
const match = /^<\/([^ \t\n>]+)[ \t\n]*>$/.exec(token);
|
|
786
|
+
if (!match)
|
|
787
|
+
this._error("Malformed end tag");
|
|
788
|
+
this._closeElement(this._name(match[1]));
|
|
789
|
+
return;
|
|
790
|
+
}
|
|
791
|
+
const is_empty = token.endsWith("/>");
|
|
792
|
+
const end = token.length - (is_empty ? 2 : 1);
|
|
793
|
+
let i = 1;
|
|
794
|
+
while (i < end && !/[ \t\n]/.test(token[i]))
|
|
795
|
+
i++;
|
|
796
|
+
const name = this._name(token.slice(1, i));
|
|
797
|
+
const attrs = Object.create(null);
|
|
798
|
+
let count = 0;
|
|
799
|
+
while (i < end) {
|
|
800
|
+
const before = i;
|
|
801
|
+
while (i < end && /[ \t\n]/.test(token[i]))
|
|
802
|
+
i++;
|
|
803
|
+
if (i === end)
|
|
804
|
+
break;
|
|
805
|
+
if (i === before)
|
|
806
|
+
this._error("Missing whitespace before attribute");
|
|
807
|
+
const start = i;
|
|
808
|
+
while (i < end && !/[ \t\n=]/.test(token[i]))
|
|
809
|
+
i++;
|
|
810
|
+
const attr = this._name(token.slice(start, i));
|
|
811
|
+
if (Object.hasOwn(attrs, attr))
|
|
812
|
+
this._error("Duplicate attribute");
|
|
813
|
+
if (++count > this.options.maxAttributes)
|
|
814
|
+
this._error("Too many attributes", "LIMIT");
|
|
815
|
+
while (i < end && /[ \t\n]/.test(token[i]))
|
|
816
|
+
i++;
|
|
817
|
+
if (token[i++] !== "=")
|
|
818
|
+
this._error("Expected attribute equals sign");
|
|
819
|
+
while (i < end && /[ \t\n]/.test(token[i]))
|
|
820
|
+
i++;
|
|
821
|
+
const quote = token[i++];
|
|
822
|
+
if (quote !== '"' && quote !== "'")
|
|
823
|
+
this._error("Expected quoted attribute value");
|
|
824
|
+
const close = token.indexOf(quote, i);
|
|
825
|
+
if (close < 0 || close >= end)
|
|
826
|
+
this._error("Unterminated attribute value");
|
|
827
|
+
let value = token.slice(i, close);
|
|
828
|
+
if (value.includes("<"))
|
|
829
|
+
this._error("Literal less-than in attribute");
|
|
830
|
+
value = value.replace(ATTRIBUTE_SPACE, " ");
|
|
831
|
+
if (value.includes("&")) {
|
|
832
|
+
let cursor = 0;
|
|
833
|
+
const pieces = [];
|
|
834
|
+
while (cursor < value.length) {
|
|
835
|
+
const amp = value.indexOf("&", cursor);
|
|
836
|
+
if (amp < 0) {
|
|
837
|
+
pieces.push(value.slice(cursor));
|
|
838
|
+
break;
|
|
839
|
+
}
|
|
840
|
+
pieces.push(value.slice(cursor, amp));
|
|
841
|
+
const semi = value.indexOf(";", amp + 1);
|
|
842
|
+
if (semi < 0)
|
|
843
|
+
this._error("Unterminated attribute entity", "ENTITY");
|
|
844
|
+
pieces.push(this._entity(value.slice(amp + 1, semi)));
|
|
845
|
+
cursor = semi + 1;
|
|
846
|
+
}
|
|
847
|
+
value = pieces.join("");
|
|
848
|
+
}
|
|
849
|
+
attrs[attr] = value;
|
|
850
|
+
i = close + 1;
|
|
851
|
+
}
|
|
852
|
+
this._openElement(name, attrs, is_empty);
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
// packages/xml-parser/src/whitespace.js
|
|
856
|
+
function createWhitespaceFilter(mode = "preserve") {
|
|
857
|
+
if (mode === "preserve")
|
|
858
|
+
return;
|
|
859
|
+
if (mode !== "strip-indentation")
|
|
860
|
+
throw new TypeError("Unsupported XML whitespace mode");
|
|
861
|
+
const stack = [];
|
|
862
|
+
return {
|
|
863
|
+
enter(attributes) {
|
|
864
|
+
const space = attributes["xml:space"];
|
|
865
|
+
stack.push({
|
|
866
|
+
is_preserved: space === "preserve" || space !== "default" && Boolean(stack.at(-1)?.is_preserved),
|
|
867
|
+
has_content: false,
|
|
868
|
+
has_protected_text: false
|
|
869
|
+
});
|
|
870
|
+
},
|
|
871
|
+
text(value, kind) {
|
|
872
|
+
const frame = stack.at(-1);
|
|
873
|
+
if (!frame)
|
|
874
|
+
return;
|
|
875
|
+
frame.has_content ||= /[^ \t\r\n]/.test(value);
|
|
876
|
+
frame.has_protected_text ||= kind !== "literal";
|
|
877
|
+
},
|
|
878
|
+
leave(record) {
|
|
879
|
+
const frame = stack.pop();
|
|
880
|
+
if (!record || !frame || frame.is_preserved || frame.has_content || frame.has_protected_text)
|
|
881
|
+
return;
|
|
882
|
+
if (!record.children.some((child) => typeof child !== "string"))
|
|
883
|
+
return;
|
|
884
|
+
record.children = record.children.filter((child) => typeof child !== "string" || !/^[ \t\r\n]*[\r\n][ \t\r\n]*$/.test(child));
|
|
885
|
+
}
|
|
886
|
+
};
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
// packages/xml-parser/src/streams.js
|
|
890
|
+
var SLICE_SIZE = 16 * 1024;
|
|
891
|
+
// packages/xml-parser/src/parallel/segment.js
|
|
892
|
+
var encoder;
|
|
893
|
+
function describeXmlError(error) {
|
|
894
|
+
const value = error ?? {};
|
|
895
|
+
const description = {
|
|
896
|
+
name: typeof value.name === "string" ? value.name : "Error",
|
|
897
|
+
message: typeof value.message === "string" ? value.message : String(error)
|
|
898
|
+
};
|
|
899
|
+
if (typeof value.code === "string")
|
|
900
|
+
description.code = value.code;
|
|
901
|
+
if (typeof value.offset === "number")
|
|
902
|
+
description.offset = value.offset;
|
|
903
|
+
return description;
|
|
904
|
+
}
|
|
905
|
+
function createXmlSegmentProcessor(config, state, sink, describeRecordError) {
|
|
906
|
+
const whitespace = createWhitespaceFilter(config.whitespace);
|
|
907
|
+
const max_chars = config.max_record_chars;
|
|
908
|
+
const max_nodes = config.max_record_nodes;
|
|
909
|
+
function checkLimits(count, node_count) {
|
|
910
|
+
if (count > max_chars)
|
|
911
|
+
throw new RangeError("maxRecordChars exceeded");
|
|
912
|
+
if (node_count > max_nodes)
|
|
913
|
+
throw new RangeError("maxRecordNodes exceeded");
|
|
914
|
+
}
|
|
915
|
+
let ready = [];
|
|
916
|
+
const stack = [];
|
|
917
|
+
let chars = 0;
|
|
918
|
+
let nodes = 0;
|
|
919
|
+
let root_text = "";
|
|
920
|
+
let is_root_text_protected = false;
|
|
921
|
+
let root_chars = state.root_chars;
|
|
922
|
+
let root_nodes = state.root_nodes;
|
|
923
|
+
let has_element_children = state.has_element_children;
|
|
924
|
+
const kept = [];
|
|
925
|
+
const root_check = { chars: -1, nodes: -1 };
|
|
926
|
+
let is_root_limit = false;
|
|
927
|
+
function checkRoot(count, node_count) {
|
|
928
|
+
root_check.chars = Math.max(root_check.chars, count - state.root_chars);
|
|
929
|
+
root_check.nodes = Math.max(root_check.nodes, node_count - state.root_nodes);
|
|
930
|
+
try {
|
|
931
|
+
checkLimits(count, node_count);
|
|
932
|
+
} catch (error) {
|
|
933
|
+
is_root_limit = true;
|
|
934
|
+
throw error;
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
function finishRootText() {
|
|
938
|
+
if (root_text && (!has_element_children || is_root_text_protected || /[^ \t\r\n]/.test(root_text))) {
|
|
939
|
+
kept.push(root_text);
|
|
940
|
+
root_chars += root_text.length;
|
|
941
|
+
root_nodes++;
|
|
942
|
+
}
|
|
943
|
+
root_text = "";
|
|
944
|
+
is_root_text_protected = false;
|
|
945
|
+
}
|
|
946
|
+
function addChildText(value) {
|
|
947
|
+
const children = stack[stack.length - 1].children;
|
|
948
|
+
const last = children.length - 1;
|
|
949
|
+
const is_merging = typeof children[last] === "string";
|
|
950
|
+
chars += value.length;
|
|
951
|
+
if (!is_merging)
|
|
952
|
+
nodes++;
|
|
953
|
+
checkLimits(chars, nodes);
|
|
954
|
+
if (is_merging)
|
|
955
|
+
children[last] += value;
|
|
956
|
+
else
|
|
957
|
+
children.push(value);
|
|
958
|
+
}
|
|
959
|
+
function openRecord(name, attributes) {
|
|
960
|
+
let length = name.length;
|
|
961
|
+
for (const key of Object.keys(attributes))
|
|
962
|
+
length += key.length + attributes[key].length;
|
|
963
|
+
chars += length;
|
|
964
|
+
nodes++;
|
|
965
|
+
checkLimits(chars, nodes);
|
|
966
|
+
const record = { name, attributes, children: [] };
|
|
967
|
+
if (stack.length)
|
|
968
|
+
stack[stack.length - 1].children.push(record);
|
|
969
|
+
stack.push(record);
|
|
970
|
+
}
|
|
971
|
+
function closeRecord() {
|
|
972
|
+
const record = stack.pop();
|
|
973
|
+
whitespace?.leave(record);
|
|
974
|
+
if (!stack.length)
|
|
975
|
+
ready.push(record);
|
|
976
|
+
}
|
|
977
|
+
let depth = 0;
|
|
978
|
+
const path = [];
|
|
979
|
+
const select_path = config.select_path;
|
|
980
|
+
let handlers;
|
|
981
|
+
if (!select_path) {
|
|
982
|
+
handlers = {
|
|
983
|
+
startTag(name, attributes) {
|
|
984
|
+
depth++;
|
|
985
|
+
whitespace?.enter(attributes);
|
|
986
|
+
if (depth === 1)
|
|
987
|
+
return;
|
|
988
|
+
if (depth === 2) {
|
|
989
|
+
has_element_children = true;
|
|
990
|
+
finishRootText();
|
|
991
|
+
chars = 0;
|
|
992
|
+
nodes = 0;
|
|
993
|
+
}
|
|
994
|
+
openRecord(name, attributes);
|
|
995
|
+
},
|
|
996
|
+
endTag() {
|
|
997
|
+
if (depth === 1) {
|
|
998
|
+
finishRootText();
|
|
999
|
+
whitespace?.leave();
|
|
1000
|
+
} else
|
|
1001
|
+
closeRecord();
|
|
1002
|
+
depth--;
|
|
1003
|
+
},
|
|
1004
|
+
text(value, kind) {
|
|
1005
|
+
whitespace?.text(value, kind);
|
|
1006
|
+
if (!value.length)
|
|
1007
|
+
return;
|
|
1008
|
+
if (depth === 1) {
|
|
1009
|
+
checkRoot(root_chars + root_text.length + value.length, root_nodes + 1);
|
|
1010
|
+
root_text += value;
|
|
1011
|
+
is_root_text_protected ||= kind !== "literal";
|
|
1012
|
+
} else if (stack.length)
|
|
1013
|
+
addChildText(value);
|
|
1014
|
+
}
|
|
1015
|
+
};
|
|
1016
|
+
} else {
|
|
1017
|
+
handlers = {
|
|
1018
|
+
startTag(name, attributes) {
|
|
1019
|
+
path.push(name);
|
|
1020
|
+
whitespace?.enter(attributes);
|
|
1021
|
+
if (!stack.length) {
|
|
1022
|
+
if (path.length !== select_path.length)
|
|
1023
|
+
return;
|
|
1024
|
+
for (let index = 0;index < path.length; index++)
|
|
1025
|
+
if (path[index] !== select_path[index])
|
|
1026
|
+
return;
|
|
1027
|
+
chars = 0;
|
|
1028
|
+
nodes = 0;
|
|
1029
|
+
}
|
|
1030
|
+
openRecord(name, attributes);
|
|
1031
|
+
},
|
|
1032
|
+
endTag() {
|
|
1033
|
+
if (stack.length)
|
|
1034
|
+
closeRecord();
|
|
1035
|
+
else
|
|
1036
|
+
whitespace?.leave();
|
|
1037
|
+
path.pop();
|
|
1038
|
+
},
|
|
1039
|
+
text(value, kind) {
|
|
1040
|
+
whitespace?.text(value, kind);
|
|
1041
|
+
if (stack.length && value.length)
|
|
1042
|
+
addChildText(value);
|
|
1043
|
+
}
|
|
1044
|
+
};
|
|
1045
|
+
}
|
|
1046
|
+
const parser = new XmlStreamParser(handlers, config.parser_options);
|
|
1047
|
+
parser.write(config.root_tag);
|
|
1048
|
+
const base_offset = parser.offset;
|
|
1049
|
+
let position = state.start;
|
|
1050
|
+
let last_write_offset = 0;
|
|
1051
|
+
let length = 0;
|
|
1052
|
+
let count = 0;
|
|
1053
|
+
let failure = null;
|
|
1054
|
+
function drain(slice) {
|
|
1055
|
+
const completed = ready;
|
|
1056
|
+
ready = [];
|
|
1057
|
+
for (const record of completed) {
|
|
1058
|
+
try {
|
|
1059
|
+
sink.add(record);
|
|
1060
|
+
} catch (error) {
|
|
1061
|
+
failure = { phase: "record", slice, error: describeRecordError(error) };
|
|
1062
|
+
return false;
|
|
1063
|
+
}
|
|
1064
|
+
count++;
|
|
1065
|
+
}
|
|
1066
|
+
return true;
|
|
1067
|
+
}
|
|
1068
|
+
function failWrite(phase, slice, error) {
|
|
1069
|
+
const description = describeXmlError(error);
|
|
1070
|
+
if (description.offset !== undefined)
|
|
1071
|
+
description.offset -= base_offset;
|
|
1072
|
+
failure = { phase, slice, error: description };
|
|
1073
|
+
if (is_root_limit)
|
|
1074
|
+
failure.is_root_limit = true;
|
|
1075
|
+
}
|
|
1076
|
+
return {
|
|
1077
|
+
push(bytes) {
|
|
1078
|
+
let offset = 0;
|
|
1079
|
+
while (offset < bytes.length && !failure) {
|
|
1080
|
+
const slice = Math.floor(position / SLICE_SIZE);
|
|
1081
|
+
const boundary = (slice + 1) * SLICE_SIZE;
|
|
1082
|
+
const size = Math.min(bytes.length - offset, boundary - position);
|
|
1083
|
+
last_write_offset = parser.offset - base_offset;
|
|
1084
|
+
try {
|
|
1085
|
+
parser.write(bytes.subarray(offset, offset + size));
|
|
1086
|
+
} catch (error) {
|
|
1087
|
+
failWrite("write", slice, error);
|
|
1088
|
+
return false;
|
|
1089
|
+
}
|
|
1090
|
+
offset += size;
|
|
1091
|
+
position += size;
|
|
1092
|
+
if (position === boundary && !drain(slice))
|
|
1093
|
+
return false;
|
|
1094
|
+
}
|
|
1095
|
+
return !failure;
|
|
1096
|
+
},
|
|
1097
|
+
close() {
|
|
1098
|
+
if (failure)
|
|
1099
|
+
return false;
|
|
1100
|
+
length = parser.offset - base_offset;
|
|
1101
|
+
const slice = Math.floor((position - 1) / SLICE_SIZE);
|
|
1102
|
+
const is_drained = position % SLICE_SIZE === 0 || drain(slice);
|
|
1103
|
+
try {
|
|
1104
|
+
if (state.is_final) {
|
|
1105
|
+
if (!is_drained)
|
|
1106
|
+
return false;
|
|
1107
|
+
parser.end();
|
|
1108
|
+
return drain(slice);
|
|
1109
|
+
}
|
|
1110
|
+
if (root_text)
|
|
1111
|
+
throw new Error("The segment ended inside root text.");
|
|
1112
|
+
parser.end((encoder ??= new TextEncoder).encode(`</${config.root_name}>`));
|
|
1113
|
+
} catch (error) {
|
|
1114
|
+
failWrite(state.is_final ? "end" : "boundary", slice, error);
|
|
1115
|
+
return false;
|
|
1116
|
+
}
|
|
1117
|
+
return is_drained;
|
|
1118
|
+
},
|
|
1119
|
+
get failure() {
|
|
1120
|
+
return failure;
|
|
1121
|
+
},
|
|
1122
|
+
get length() {
|
|
1123
|
+
return length;
|
|
1124
|
+
},
|
|
1125
|
+
get last_write_offset() {
|
|
1126
|
+
return last_write_offset;
|
|
1127
|
+
},
|
|
1128
|
+
take() {
|
|
1129
|
+
const output = { output: sink.take(), count };
|
|
1130
|
+
count = 0;
|
|
1131
|
+
return output;
|
|
1132
|
+
},
|
|
1133
|
+
summary() {
|
|
1134
|
+
return {
|
|
1135
|
+
kept,
|
|
1136
|
+
has_element_children,
|
|
1137
|
+
root_check: root_check.chars >= 0 ? root_check : null
|
|
1138
|
+
};
|
|
1139
|
+
}
|
|
1140
|
+
};
|
|
1141
|
+
}
|
|
1142
|
+
function processXmlSegment(config, state, bytes, task) {
|
|
1143
|
+
const processor = createXmlSegmentProcessor(config, state, task.createSink(config.task_options), task.describeError ?? describeXmlError);
|
|
1144
|
+
processor.push(bytes);
|
|
1145
|
+
const body = processor.take();
|
|
1146
|
+
processor.close();
|
|
1147
|
+
return {
|
|
1148
|
+
body,
|
|
1149
|
+
tail: processor.take(),
|
|
1150
|
+
failure: processor.failure,
|
|
1151
|
+
length: processor.length,
|
|
1152
|
+
last_write_offset: processor.last_write_offset,
|
|
1153
|
+
summary: processor.summary()
|
|
1154
|
+
};
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
// packages/xml-parser/src/parallel/split.js
|
|
1158
|
+
var MAX_PROLOG_BYTES = 16 * 1024 * 1024;
|
|
1159
|
+
|
|
1160
|
+
// packages/xml-parser/src/parallel/worker.js
|
|
1161
|
+
var XML_WORKER_PROTOCOL = 2;
|
|
1162
|
+
function serveXmlSegmentWorker(scope, task) {
|
|
1163
|
+
let config = null;
|
|
1164
|
+
scope.addEventListener("message", (event) => {
|
|
1165
|
+
const data = event.data;
|
|
1166
|
+
if (data?.type === "hello") {
|
|
1167
|
+
const is_match = data.protocol === XML_WORKER_PROTOCOL && data.task === task.name;
|
|
1168
|
+
scope.postMessage({ type: is_match ? "ready" : "refused", protocol: XML_WORKER_PROTOCOL, task: task.name }, []);
|
|
1169
|
+
return;
|
|
1170
|
+
}
|
|
1171
|
+
if (data?.type === "config") {
|
|
1172
|
+
config = data.config;
|
|
1173
|
+
return;
|
|
1174
|
+
}
|
|
1175
|
+
if (data?.type !== "segment")
|
|
1176
|
+
return;
|
|
1177
|
+
let message;
|
|
1178
|
+
let transfer = [];
|
|
1179
|
+
try {
|
|
1180
|
+
if (!config)
|
|
1181
|
+
throw new Error("The XML worker received a segment before its configuration.");
|
|
1182
|
+
const result = processXmlSegment(config, data.state, data.bytes, task);
|
|
1183
|
+
if (task.transfer)
|
|
1184
|
+
transfer = [...task.transfer(result.body.output), ...task.transfer(result.tail.output)];
|
|
1185
|
+
message = { type: "segment", seq: data.seq, result };
|
|
1186
|
+
} catch (error) {
|
|
1187
|
+
message = { type: "error", seq: data.seq, error: describeXmlError(error) };
|
|
1188
|
+
}
|
|
1189
|
+
scope.postMessage(message, transfer);
|
|
1190
|
+
});
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
// packages/xml-parser/src/parallel/coordinator.js
|
|
1194
|
+
var MAX_DEFAULT_SEGMENT_BYTES = 1024 * 1024;
|
|
1195
|
+
var MIN_DEFAULT_SEGMENT_BYTES = 64 * 1024;
|
|
1196
|
+
var DEFAULT_READ_BYTES = 4 * 1024 * 1024;
|
|
1197
|
+
var DEFAULT_MAX_SEGMENT_BYTES = 64 * 1024 * 1024;
|
|
1198
|
+
// packages/xml-parser/node.js
|
|
1199
|
+
import { parentPort, Worker } from "node:worker_threads";
|
|
1200
|
+
function toTransferList(transfer) {
|
|
1201
|
+
return transfer;
|
|
1202
|
+
}
|
|
1203
|
+
function getNodeXmlWorkerScope() {
|
|
1204
|
+
const port = parentPort;
|
|
1205
|
+
if (!port)
|
|
1206
|
+
throw new Error("getNodeXmlWorkerScope must run inside a worker_threads worker.");
|
|
1207
|
+
return {
|
|
1208
|
+
postMessage(message, transfer) {
|
|
1209
|
+
port.postMessage(message, toTransferList(transfer));
|
|
1210
|
+
},
|
|
1211
|
+
addEventListener(type, listener) {
|
|
1212
|
+
if (type === "message")
|
|
1213
|
+
port.on("message", (data) => listener({ data }));
|
|
1214
|
+
}
|
|
1215
|
+
};
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
// packages/import/src/constants.js
|
|
1219
|
+
var IMPORT_BATCH_BYTES = 1024 * 1024;
|
|
1220
|
+
var IMPORT_JSON_BYTES = 50 * 1024 * 1024;
|
|
1221
|
+
var IMPORT_CHUNK_BYTES = 16 * 1024;
|
|
1222
|
+
var IMPORT_SNIFF_BYTES = 256 * 1024;
|
|
1223
|
+
var IMPORT_SAMPLE_BYTES = 2 * 1024 * 1024;
|
|
1224
|
+
var IMPORT_XML_SEGMENT_MAX_BYTES = 2 * IMPORT_BATCH_BYTES;
|
|
1225
|
+
var IMPORT_XML_READ_BYTES = 256 * 1024;
|
|
1226
|
+
var IMPORT_WORKER_MIN_BYTES = 32 * 1024 * 1024;
|
|
1227
|
+
var IMPORT_FORMATS = Object.freeze({
|
|
1228
|
+
xml: "xml",
|
|
1229
|
+
jsonl: "jsonl",
|
|
1230
|
+
json: "json",
|
|
1231
|
+
csv: "csv",
|
|
1232
|
+
tsv: "tsv"
|
|
1233
|
+
});
|
|
1234
|
+
var IMPORT_REJECTED_STATUSES = Object.freeze([
|
|
1235
|
+
400,
|
|
1236
|
+
401,
|
|
1237
|
+
403,
|
|
1238
|
+
404,
|
|
1239
|
+
405,
|
|
1240
|
+
409,
|
|
1241
|
+
410,
|
|
1242
|
+
413,
|
|
1243
|
+
415,
|
|
1244
|
+
422,
|
|
1245
|
+
429
|
|
1246
|
+
]);
|
|
1247
|
+
|
|
1248
|
+
// packages/import/src/errors.js
|
|
1249
|
+
class ImportRowError extends Error {
|
|
1250
|
+
constructor(index, detail) {
|
|
1251
|
+
super(`Row ${index}${detail}`);
|
|
1252
|
+
this.index = index;
|
|
1253
|
+
this.detail = detail;
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
// packages/import/src/timestamp.js
|
|
1258
|
+
function getImportField(record, path) {
|
|
1259
|
+
if (!path.startsWith("/"))
|
|
1260
|
+
return Object.hasOwn(record, path) ? record[path] : undefined;
|
|
1261
|
+
let value = record;
|
|
1262
|
+
for (const raw of path.slice(1).split("/")) {
|
|
1263
|
+
if (/~(?:[^01]|$)/.test(raw))
|
|
1264
|
+
return;
|
|
1265
|
+
const key = raw.replace(/~1/g, "/").replace(/~0/g, "~");
|
|
1266
|
+
if (value === null || typeof value !== "object" || !Object.hasOwn(value, key))
|
|
1267
|
+
return;
|
|
1268
|
+
value = value[key];
|
|
1269
|
+
}
|
|
1270
|
+
return value;
|
|
1271
|
+
}
|
|
1272
|
+
function parseImportTimestamp(value) {
|
|
1273
|
+
if (typeof value === "number" || typeof value === "string" && /^\d{10}(?:\d{3})?$/.test(value)) {
|
|
1274
|
+
const number = Number(value);
|
|
1275
|
+
if (!Number.isSafeInteger(number) || number < 1e9)
|
|
1276
|
+
return null;
|
|
1277
|
+
const digits = String(number).length;
|
|
1278
|
+
if (digits !== 10 && digits !== 13)
|
|
1279
|
+
return null;
|
|
1280
|
+
const date = new Date(digits === 10 ? number * 1000 : number);
|
|
1281
|
+
return Number.isNaN(date.valueOf()) ? null : date.toISOString();
|
|
1282
|
+
}
|
|
1283
|
+
if (typeof value !== "string")
|
|
1284
|
+
return null;
|
|
1285
|
+
const match = /^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,3}))? ?(Z|[+-]\d{2}:?\d{2})$/.exec(value);
|
|
1286
|
+
if (!match)
|
|
1287
|
+
return null;
|
|
1288
|
+
const [
|
|
1289
|
+
,
|
|
1290
|
+
year_text,
|
|
1291
|
+
month_text,
|
|
1292
|
+
day_text,
|
|
1293
|
+
hour_text,
|
|
1294
|
+
minute_text,
|
|
1295
|
+
second_text,
|
|
1296
|
+
fraction = "",
|
|
1297
|
+
zone
|
|
1298
|
+
] = match;
|
|
1299
|
+
const [year, month, day, hour, minute, second] = [
|
|
1300
|
+
year_text,
|
|
1301
|
+
month_text,
|
|
1302
|
+
day_text,
|
|
1303
|
+
hour_text,
|
|
1304
|
+
minute_text,
|
|
1305
|
+
second_text
|
|
1306
|
+
].map(Number);
|
|
1307
|
+
if (year < 1 || month < 1 || month > 12 || day < 1 || hour > 23 || minute > 59 || second > 59)
|
|
1308
|
+
return null;
|
|
1309
|
+
const date = new Date(0);
|
|
1310
|
+
date.setUTCFullYear(year, month - 1, day);
|
|
1311
|
+
date.setUTCHours(hour, minute, second, Number(fraction.padEnd(3, "0")));
|
|
1312
|
+
if (date.getUTCFullYear() !== year || date.getUTCMonth() !== month - 1 || date.getUTCDate() !== day)
|
|
1313
|
+
return null;
|
|
1314
|
+
let offset = 0;
|
|
1315
|
+
if (zone !== "Z") {
|
|
1316
|
+
const digits = zone.slice(1).replace(":", "");
|
|
1317
|
+
const hours = Number(digits.slice(0, 2));
|
|
1318
|
+
const minutes = Number(digits.slice(2));
|
|
1319
|
+
if (hours > 14 || minutes > 59 || hours === 14 && minutes !== 0 || /^-00:?00$/.test(zone))
|
|
1320
|
+
return null;
|
|
1321
|
+
offset = (hours * 60 + minutes) * (zone[0] === "-" ? -1 : 1);
|
|
1322
|
+
}
|
|
1323
|
+
return new Date(date.valueOf() - offset * 60000).toISOString();
|
|
1324
|
+
}
|
|
1325
|
+
function isAmbiguousTimestamp(value) {
|
|
1326
|
+
return typeof value === "string" && (/^\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?)?$/.test(value) || /-00:?00$/.test(value));
|
|
1327
|
+
}
|
|
1328
|
+
function getRowTimestamp(row, timestamp_field) {
|
|
1329
|
+
if (!timestamp_field)
|
|
1330
|
+
return null;
|
|
1331
|
+
const value = getImportField(row.record, timestamp_field);
|
|
1332
|
+
const timestamp = parseImportTimestamp(value);
|
|
1333
|
+
if (!timestamp)
|
|
1334
|
+
throw new ImportRowError(row.index, `: timestamp field ${JSON.stringify(timestamp_field)} is missing, invalid or timezone-ambiguous. Choose an explicit timezone or a different field.`);
|
|
1335
|
+
const milliseconds = Date.parse(timestamp);
|
|
1336
|
+
if (milliseconds < 0)
|
|
1337
|
+
throw new ImportRowError(row.index, `: timestamp field ${JSON.stringify(timestamp_field)} is before 1970-01-01T00:00:00Z and cannot be used as an import timestamp.`);
|
|
1338
|
+
if (milliseconds > Date.now())
|
|
1339
|
+
throw new ImportRowError(row.index, `: timestamp field ${JSON.stringify(timestamp_field)} is in the future and cannot be used as an import timestamp.`);
|
|
1340
|
+
return timestamp;
|
|
1341
|
+
}
|
|
1342
|
+
|
|
1343
|
+
// packages/import/src/batch.js
|
|
1344
|
+
var encoder2 = new TextEncoder;
|
|
1345
|
+
function serializeImportRow(row) {
|
|
1346
|
+
const line = JSON.stringify(row.record);
|
|
1347
|
+
if (line === undefined || line.length + 1 > IMPORT_BATCH_BYTES)
|
|
1348
|
+
throw new ImportRowError(row.index, " exceeds the 1 MiB record limit.");
|
|
1349
|
+
const bytes = encoder2.encode(line).byteLength + 1;
|
|
1350
|
+
if (bytes > IMPORT_BATCH_BYTES)
|
|
1351
|
+
throw new ImportRowError(row.index, " exceeds the 1 MiB record limit.");
|
|
1352
|
+
return { row, line, bytes };
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
// packages/import/src/row.js
|
|
1356
|
+
function createImportRow(record, index) {
|
|
1357
|
+
if (!record || typeof record !== "object" || Array.isArray(record))
|
|
1358
|
+
throw new ImportRowError(index, " must be a JSON object.");
|
|
1359
|
+
const row = { index, record };
|
|
1360
|
+
serializeImportRow(row);
|
|
1361
|
+
return row;
|
|
1362
|
+
}
|
|
1363
|
+
|
|
1364
|
+
// packages/import/src/scan-profile.js
|
|
1365
|
+
var IMPORT_PROFILE_FIELDS = 256;
|
|
1366
|
+
var MAX_DEPTH = 16;
|
|
1367
|
+
var MAX_PATH_CHARS = 4096;
|
|
1368
|
+
var MAX_VISITED_FIELDS = 2048;
|
|
1369
|
+
var PREVIEW_ROWS = 12;
|
|
1370
|
+
function createScanProfile({ timestamp_field, max_fields = IMPORT_PROFILE_FIELDS } = {}) {
|
|
1371
|
+
const fields = new Map;
|
|
1372
|
+
const preview = [];
|
|
1373
|
+
let rows = 0;
|
|
1374
|
+
let max_row_bytes = 0;
|
|
1375
|
+
let has_row_limits = false;
|
|
1376
|
+
let has_long_paths = false;
|
|
1377
|
+
let has_field_limit = false;
|
|
1378
|
+
function profile(record) {
|
|
1379
|
+
let visited = 0;
|
|
1380
|
+
function visit(object, prefix, depth) {
|
|
1381
|
+
for (const key of Object.keys(object)) {
|
|
1382
|
+
if (++visited > MAX_VISITED_FIELDS) {
|
|
1383
|
+
has_row_limits = true;
|
|
1384
|
+
return;
|
|
1385
|
+
}
|
|
1386
|
+
const path = prefix + "/" + key.replace(/~/g, "~0").replace(/\//g, "~1");
|
|
1387
|
+
if (path.length > MAX_PATH_CHARS) {
|
|
1388
|
+
has_long_paths = true;
|
|
1389
|
+
continue;
|
|
1390
|
+
}
|
|
1391
|
+
const value = object[key];
|
|
1392
|
+
let entry = fields.get(path);
|
|
1393
|
+
if (!entry) {
|
|
1394
|
+
if (fields.size >= max_fields) {
|
|
1395
|
+
has_field_limit = true;
|
|
1396
|
+
continue;
|
|
1397
|
+
}
|
|
1398
|
+
entry = {
|
|
1399
|
+
field: {
|
|
1400
|
+
path,
|
|
1401
|
+
label: path.slice(1).replace(/~1/g, "/").replace(/~0/g, "~"),
|
|
1402
|
+
types: [],
|
|
1403
|
+
present: 0,
|
|
1404
|
+
missing: 0
|
|
1405
|
+
},
|
|
1406
|
+
candidate: {
|
|
1407
|
+
path,
|
|
1408
|
+
label: key,
|
|
1409
|
+
valid: 0,
|
|
1410
|
+
invalid: 0,
|
|
1411
|
+
missing: 0,
|
|
1412
|
+
ambiguous: 0,
|
|
1413
|
+
min: null,
|
|
1414
|
+
max: null
|
|
1415
|
+
},
|
|
1416
|
+
has_evidence: /(?:date|time|(?:^|_)at$|timestamp)/i.test(key)
|
|
1417
|
+
};
|
|
1418
|
+
fields.set(path, entry);
|
|
1419
|
+
}
|
|
1420
|
+
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
1421
|
+
if (depth >= MAX_DEPTH)
|
|
1422
|
+
has_row_limits = true;
|
|
1423
|
+
else
|
|
1424
|
+
visit(value, path, depth + 1);
|
|
1425
|
+
}
|
|
1426
|
+
}
|
|
1427
|
+
}
|
|
1428
|
+
visit(record, "", 1);
|
|
1429
|
+
for (const entry of fields.values()) {
|
|
1430
|
+
const value = getImportField(record, entry.field.path);
|
|
1431
|
+
if (value === undefined)
|
|
1432
|
+
continue;
|
|
1433
|
+
entry.field.present++;
|
|
1434
|
+
let type = typeof value;
|
|
1435
|
+
if (value === null)
|
|
1436
|
+
type = "null";
|
|
1437
|
+
else if (Array.isArray(value))
|
|
1438
|
+
type = "array";
|
|
1439
|
+
if (!entry.field.types.includes(type))
|
|
1440
|
+
entry.field.types.push(type);
|
|
1441
|
+
if (value !== undefined && value !== null && value !== "") {
|
|
1442
|
+
const timestamp = parseImportTimestamp(value);
|
|
1443
|
+
if (typeof value === "string" && /^\d{4}-\d{2}-\d{2}(?:[T ]|$)/.test(value))
|
|
1444
|
+
entry.has_evidence = true;
|
|
1445
|
+
if (timestamp) {
|
|
1446
|
+
entry.candidate.valid++;
|
|
1447
|
+
if (!entry.candidate.min || timestamp < entry.candidate.min)
|
|
1448
|
+
entry.candidate.min = timestamp;
|
|
1449
|
+
if (!entry.candidate.max || timestamp > entry.candidate.max)
|
|
1450
|
+
entry.candidate.max = timestamp;
|
|
1451
|
+
} else if (isAmbiguousTimestamp(value))
|
|
1452
|
+
entry.candidate.ambiguous++;
|
|
1453
|
+
else
|
|
1454
|
+
entry.candidate.invalid++;
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
return {
|
|
1459
|
+
get rows() {
|
|
1460
|
+
return rows;
|
|
1461
|
+
},
|
|
1462
|
+
get max_row_bytes() {
|
|
1463
|
+
return max_row_bytes;
|
|
1464
|
+
},
|
|
1465
|
+
get preview() {
|
|
1466
|
+
return preview;
|
|
1467
|
+
},
|
|
1468
|
+
get is_truncated() {
|
|
1469
|
+
return has_row_limits || has_long_paths || has_field_limit;
|
|
1470
|
+
},
|
|
1471
|
+
add(row) {
|
|
1472
|
+
const entry = serializeImportRow(row);
|
|
1473
|
+
getRowTimestamp(row, timestamp_field);
|
|
1474
|
+
rows++;
|
|
1475
|
+
max_row_bytes = Math.max(max_row_bytes, entry.bytes);
|
|
1476
|
+
if (preview.length < PREVIEW_ROWS)
|
|
1477
|
+
preview.push(row);
|
|
1478
|
+
profile(row.record);
|
|
1479
|
+
},
|
|
1480
|
+
merge(partial) {
|
|
1481
|
+
for (let index = 0;index < partial.preview.length && preview.length < PREVIEW_ROWS; index++)
|
|
1482
|
+
preview.push({ index: rows + index + 1, record: partial.preview[index] });
|
|
1483
|
+
rows += partial.rows;
|
|
1484
|
+
max_row_bytes = Math.max(max_row_bytes, partial.max_row_bytes);
|
|
1485
|
+
has_row_limits ||= partial.has_row_limits;
|
|
1486
|
+
has_long_paths ||= partial.has_long_paths;
|
|
1487
|
+
has_field_limit ||= partial.has_field_limit;
|
|
1488
|
+
for (const local of partial.entries) {
|
|
1489
|
+
const path = local.field.path;
|
|
1490
|
+
let entry = fields.get(path);
|
|
1491
|
+
if (!entry) {
|
|
1492
|
+
if (fields.size >= max_fields) {
|
|
1493
|
+
has_field_limit = true;
|
|
1494
|
+
continue;
|
|
1495
|
+
}
|
|
1496
|
+
entry = {
|
|
1497
|
+
field: { ...local.field, types: [], present: 0, missing: 0 },
|
|
1498
|
+
candidate: {
|
|
1499
|
+
...local.candidate,
|
|
1500
|
+
valid: 0,
|
|
1501
|
+
invalid: 0,
|
|
1502
|
+
ambiguous: 0,
|
|
1503
|
+
min: null,
|
|
1504
|
+
max: null
|
|
1505
|
+
},
|
|
1506
|
+
has_evidence: false
|
|
1507
|
+
};
|
|
1508
|
+
fields.set(path, entry);
|
|
1509
|
+
}
|
|
1510
|
+
entry.field.present += local.field.present;
|
|
1511
|
+
for (const type of local.field.types)
|
|
1512
|
+
if (!entry.field.types.includes(type))
|
|
1513
|
+
entry.field.types.push(type);
|
|
1514
|
+
const candidate = entry.candidate;
|
|
1515
|
+
candidate.valid += local.candidate.valid;
|
|
1516
|
+
candidate.invalid += local.candidate.invalid;
|
|
1517
|
+
candidate.ambiguous += local.candidate.ambiguous;
|
|
1518
|
+
if (local.candidate.min && (!candidate.min || local.candidate.min < candidate.min))
|
|
1519
|
+
candidate.min = local.candidate.min;
|
|
1520
|
+
if (local.candidate.max && (!candidate.max || local.candidate.max > candidate.max))
|
|
1521
|
+
candidate.max = local.candidate.max;
|
|
1522
|
+
entry.has_evidence ||= local.has_evidence;
|
|
1523
|
+
}
|
|
1524
|
+
},
|
|
1525
|
+
partial() {
|
|
1526
|
+
return {
|
|
1527
|
+
rows,
|
|
1528
|
+
max_row_bytes,
|
|
1529
|
+
preview: preview.map((row) => row.record),
|
|
1530
|
+
entries: Array.from(fields.values()),
|
|
1531
|
+
has_row_limits,
|
|
1532
|
+
has_long_paths,
|
|
1533
|
+
has_field_limit
|
|
1534
|
+
};
|
|
1535
|
+
},
|
|
1536
|
+
finish(report) {
|
|
1537
|
+
report.row_count = rows;
|
|
1538
|
+
report.max_row_bytes = max_row_bytes;
|
|
1539
|
+
report.preview = preview;
|
|
1540
|
+
for (const { field, candidate, has_evidence } of fields.values()) {
|
|
1541
|
+
field.missing = rows - field.present;
|
|
1542
|
+
report.fields.push(field);
|
|
1543
|
+
if (has_evidence) {
|
|
1544
|
+
candidate.missing = rows - candidate.valid - candidate.invalid - candidate.ambiguous;
|
|
1545
|
+
report.timestamp_candidates.push(candidate);
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
}
|
|
1549
|
+
};
|
|
1550
|
+
}
|
|
1551
|
+
|
|
1552
|
+
// packages/import/src/xml-task.js
|
|
1553
|
+
var PARTIAL_PROFILE_FIELDS = 1024;
|
|
1554
|
+
var IMPORT_XML_OPTIONS = Object.freeze({
|
|
1555
|
+
whitespace: "strip-indentation",
|
|
1556
|
+
parserOptions: Object.freeze({ doctype: "ignore" }),
|
|
1557
|
+
maxRecordChars: IMPORT_BATCH_BYTES,
|
|
1558
|
+
maxRecordNodes: 1e5
|
|
1559
|
+
});
|
|
1560
|
+
var IMPORT_XML_TASK = {
|
|
1561
|
+
name: "tailglow-import-xml",
|
|
1562
|
+
createSink(options) {
|
|
1563
|
+
let index = 0;
|
|
1564
|
+
if (options.kind === "rows") {
|
|
1565
|
+
let records = [];
|
|
1566
|
+
return {
|
|
1567
|
+
add(record) {
|
|
1568
|
+
createImportRow(record, ++index);
|
|
1569
|
+
records.push(record);
|
|
1570
|
+
},
|
|
1571
|
+
take() {
|
|
1572
|
+
const output = { records };
|
|
1573
|
+
records = [];
|
|
1574
|
+
return output;
|
|
1575
|
+
}
|
|
1576
|
+
};
|
|
1577
|
+
}
|
|
1578
|
+
const createProfile = () => createScanProfile({
|
|
1579
|
+
timestamp_field: options.timestamp_field,
|
|
1580
|
+
max_fields: PARTIAL_PROFILE_FIELDS
|
|
1581
|
+
});
|
|
1582
|
+
let profile = createProfile();
|
|
1583
|
+
return {
|
|
1584
|
+
add(record) {
|
|
1585
|
+
profile.add(createImportRow(record, ++index));
|
|
1586
|
+
},
|
|
1587
|
+
take() {
|
|
1588
|
+
const partial = profile.partial();
|
|
1589
|
+
profile = createProfile();
|
|
1590
|
+
return partial;
|
|
1591
|
+
}
|
|
1592
|
+
};
|
|
1593
|
+
},
|
|
1594
|
+
describeError(error) {
|
|
1595
|
+
if (error instanceof ImportRowError)
|
|
1596
|
+
return { detail: error.detail };
|
|
1597
|
+
const value = error ?? {};
|
|
1598
|
+
return {
|
|
1599
|
+
name: typeof value.name === "string" ? value.name : "Error",
|
|
1600
|
+
message: typeof value.message === "string" ? value.message : String(error)
|
|
1601
|
+
};
|
|
1602
|
+
},
|
|
1603
|
+
restoreError(description, record_number) {
|
|
1604
|
+
const value = description;
|
|
1605
|
+
if (typeof value.detail === "string")
|
|
1606
|
+
return new ImportRowError(record_number, value.detail);
|
|
1607
|
+
if (value.name === "RangeError")
|
|
1608
|
+
return new RangeError(value.message);
|
|
1609
|
+
if (value.name === "TypeError")
|
|
1610
|
+
return new TypeError(value.message);
|
|
1611
|
+
return new Error(value.message);
|
|
1612
|
+
}
|
|
1613
|
+
};
|
|
1614
|
+
|
|
1615
|
+
// packages/import/src/worker.node.js
|
|
1616
|
+
serveXmlSegmentWorker(getNodeXmlWorkerScope(), IMPORT_XML_TASK);
|