@felan-ai/ext-web-access 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +0 -8
- package/README.md +126 -62
- package/dist/boundary.d.ts +1 -3
- package/dist/boundary.d.ts.map +1 -1
- package/dist/boundary.js +1 -19
- package/dist/boundary.js.map +1 -1
- package/dist/config.d.ts +4 -9
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +82 -36
- package/dist/config.js.map +1 -1
- package/dist/content-find.d.ts +19 -10
- package/dist/content-find.d.ts.map +1 -1
- package/dist/content-find.js +104 -70
- package/dist/content-find.js.map +1 -1
- package/dist/credentials.d.ts.map +1 -1
- package/dist/credentials.js +12 -6
- package/dist/credentials.js.map +1 -1
- package/dist/extract.d.ts +7 -7
- package/dist/extract.d.ts.map +1 -1
- package/dist/extract.js +237 -84
- package/dist/extract.js.map +1 -1
- package/dist/http.d.ts +3 -3
- package/dist/http.d.ts.map +1 -1
- package/dist/http.js +19 -5
- package/dist/http.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +444 -460
- package/dist/index.js.map +1 -1
- package/dist/pdf-service.d.ts +22 -0
- package/dist/pdf-service.d.ts.map +1 -0
- package/dist/pdf-service.js +61 -0
- package/dist/pdf-service.js.map +1 -0
- package/dist/providers.d.ts.map +1 -1
- package/dist/providers.js +191 -171
- package/dist/providers.js.map +1 -1
- package/dist/ssrf.d.ts +1 -0
- package/dist/ssrf.d.ts.map +1 -1
- package/dist/ssrf.js +36 -6
- package/dist/ssrf.js.map +1 -1
- package/dist/types.d.ts +6 -76
- package/dist/types.d.ts.map +1 -1
- package/dist/url.d.ts +2 -0
- package/dist/url.d.ts.map +1 -0
- package/dist/url.js +15 -0
- package/dist/url.js.map +1 -0
- package/package.json +7 -8
- package/dist/github.d.ts +0 -13
- package/dist/github.d.ts.map +0 -1
- package/dist/github.js +0 -166
- package/dist/github.js.map +0 -1
- package/dist/source-check.d.ts +0 -21
- package/dist/source-check.d.ts.map +0 -1
- package/dist/source-check.js +0 -136
- package/dist/source-check.js.map +0 -1
- package/dist/storage.d.ts +0 -19
- package/dist/storage.d.ts.map +0 -1
- package/dist/storage.js +0 -458
- package/dist/storage.js.map +0 -1
package/dist/extract.js
CHANGED
|
@@ -1,80 +1,213 @@
|
|
|
1
1
|
import { Readability } from '@mozilla/readability';
|
|
2
2
|
import { parseHTML } from 'linkedom';
|
|
3
3
|
import TurndownService from 'turndown';
|
|
4
|
-
import {
|
|
5
|
-
import { extractGitHubRepository, parseGitHubUrl } from './github.js';
|
|
4
|
+
import { pdfSettings } from './config.js';
|
|
6
5
|
import { combinedSignal, readResponseBytes } from './http.js';
|
|
6
|
+
import { isPdfConversionResult, requestPdfConversion, sanitizedPdfConversionError, unavailablePdfConversionError, } from './pdf-service.js';
|
|
7
7
|
import { fetchRemoteUrl, ssrfSettings, validateRemoteUrl } from './ssrf.js';
|
|
8
8
|
const MAX_HTTP_RESPONSE_BYTES = 5 * 1024 * 1024;
|
|
9
|
-
const
|
|
9
|
+
const MAX_LLMS_TXT_RESPONSE_BYTES = 1024 * 1024;
|
|
10
10
|
const MAX_EXTRACTED_CHARACTERS = 750_000;
|
|
11
|
-
const DEFAULT_PDF_MAX_SIZE_MB = 20;
|
|
12
|
-
const MAX_PDF_MAX_SIZE_MB = 50;
|
|
13
|
-
const DEFAULT_PDF_MAX_PAGES = 100;
|
|
14
11
|
const HTTP_TIMEOUT_MS = 30_000;
|
|
15
|
-
|
|
12
|
+
const PDF_SIGNATURE = /^%PDF-[0-9]\.[0-9]/u;
|
|
13
|
+
export async function extractContent(rawUrl, config, pdfEvents, signal, timeoutMs = HTTP_TIMEOUT_MS, options = {}) {
|
|
16
14
|
const settings = ssrfSettings(config);
|
|
17
|
-
const
|
|
18
|
-
const
|
|
19
|
-
|
|
20
|
-
return extractGitHubRepository(url.toString(), github, runtime, config, signal, options.forceClone);
|
|
21
|
-
}
|
|
22
|
-
const response = await fetchRemoteUrl(url, {
|
|
15
|
+
const requestSignal = combinedSignal(signal, timeoutMs);
|
|
16
|
+
const validatedUrl = await validateRemoteUrl(rawUrl, settings, { signal: requestSignal });
|
|
17
|
+
const response = await fetchRemoteUrl(validatedUrl, {
|
|
23
18
|
headers: {
|
|
24
|
-
Accept: 'text/html,application/xhtml+xml,application/json,
|
|
19
|
+
Accept: 'text/html,application/xhtml+xml,application/json,application/pdf,text/plain;q=0.9,text/*;q=0.8',
|
|
25
20
|
'User-Agent': 'Felan-Web-Access/0.1',
|
|
26
21
|
},
|
|
27
|
-
signal:
|
|
22
|
+
signal: requestSignal,
|
|
28
23
|
}, settings);
|
|
24
|
+
const responseUrl = response.url ? new URL(response.url) : validatedUrl;
|
|
29
25
|
if (!response.ok) {
|
|
30
|
-
|
|
31
|
-
}
|
|
32
|
-
const contentType = response.headers.get('content-type')?.split(';')[0]?.trim().toLowerCase() ?? '';
|
|
33
|
-
const contentLength = Number(response.headers.get('content-length'));
|
|
34
|
-
if (contentType.startsWith('image/') && Number.isFinite(contentLength) && contentLength > MAX_IMAGE_BYTES) {
|
|
35
|
-
throw new Error(`Image exceeds the ${MAX_IMAGE_BYTES}-byte limit`);
|
|
36
|
-
}
|
|
37
|
-
const pdfMaximumBytes = positiveNumber(config.pdf?.maxSizeMB, DEFAULT_PDF_MAX_SIZE_MB, MAX_PDF_MAX_SIZE_MB) * 1024 * 1024;
|
|
38
|
-
if (isPdf(url, contentType) && Number.isFinite(contentLength) && contentLength > pdfMaximumBytes) {
|
|
39
|
-
throw new Error(`PDF exceeds the configured ${pdfMaximumBytes}-byte limit`);
|
|
40
|
-
}
|
|
41
|
-
const maximumBytes = contentType.startsWith('image/')
|
|
42
|
-
? MAX_IMAGE_BYTES
|
|
43
|
-
: isPdf(url, contentType) ? pdfMaximumBytes : MAX_HTTP_RESPONSE_BYTES;
|
|
44
|
-
const bytes = await readResponseBytes(response, maximumBytes);
|
|
45
|
-
if (contentType.startsWith('image/')) {
|
|
26
|
+
await response.body?.cancel();
|
|
46
27
|
return {
|
|
47
|
-
url:
|
|
48
|
-
title:
|
|
49
|
-
content:
|
|
50
|
-
error:
|
|
51
|
-
contentType,
|
|
52
|
-
image: { data: Buffer.from(bytes).toString('base64'), mimeType: contentType },
|
|
28
|
+
url: responseUrl.toString(),
|
|
29
|
+
title: responseUrl.hostname,
|
|
30
|
+
content: '',
|
|
31
|
+
error: `HTTP ${response.status}`,
|
|
53
32
|
};
|
|
54
33
|
}
|
|
55
|
-
|
|
56
|
-
|
|
34
|
+
const contentType = response.headers.get('content-type')?.split(';')[0]?.trim().toLowerCase() ?? '';
|
|
35
|
+
if (!isSupportedContentType(contentType) && contentType !== 'application/octet-stream') {
|
|
36
|
+
await response.body?.cancel();
|
|
37
|
+
throw new Error(`Unsupported content type: ${contentType || 'unknown'}`);
|
|
38
|
+
}
|
|
39
|
+
const declaredPdf = contentType === 'application/pdf' || contentType === 'application/octet-stream';
|
|
40
|
+
if (declaredPdf) {
|
|
41
|
+
const limits = pdfSettings(config);
|
|
42
|
+
try {
|
|
43
|
+
return await extractDeclaredPdf(response, responseUrl, contentType, limits, pdfEvents, signal, requestSignal);
|
|
44
|
+
}
|
|
45
|
+
catch (error) {
|
|
46
|
+
await response.body?.cancel().catch(() => undefined);
|
|
47
|
+
throw error;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
const maximumBytes = MAX_HTTP_RESPONSE_BYTES;
|
|
51
|
+
const contentLengthHeader = response.headers.get('content-length');
|
|
52
|
+
const contentLength = contentLengthHeader !== null && /^\d+$/u.test(contentLengthHeader)
|
|
53
|
+
? Number(contentLengthHeader)
|
|
54
|
+
: undefined;
|
|
55
|
+
if (contentLength !== undefined && (!Number.isSafeInteger(contentLength) || contentLength > maximumBytes)) {
|
|
56
|
+
await response.body?.cancel();
|
|
57
|
+
throw new Error(`Response exceeds the ${maximumBytes}-byte limit`);
|
|
58
|
+
}
|
|
59
|
+
const bytes = await readResponseBytes(response, maximumBytes, requestSignal);
|
|
60
|
+
const pdfSignature = hasPdfSignature(bytes);
|
|
61
|
+
if (pdfSignature) {
|
|
62
|
+
throw new Error(`PDF file signature does not match content type: ${contentType || 'unknown'}`);
|
|
63
|
+
}
|
|
57
64
|
const text = new TextDecoder().decode(bytes);
|
|
58
|
-
if (
|
|
59
|
-
|
|
65
|
+
if (isHtml(contentType, text)) {
|
|
66
|
+
if (!options.ignoreLlmsTxt
|
|
67
|
+
&& !isDirectLlmsTxt(validatedUrl)
|
|
68
|
+
&& !isDirectLlmsTxt(responseUrl)
|
|
69
|
+
&& shouldProbeLlmsTxt(responseUrl, contentType, text)) {
|
|
70
|
+
const replacement = await getLlmsTxtReplacement(responseUrl, settings, options.llmsTxtProbes ?? new Map(), signal, timeoutMs);
|
|
71
|
+
if (replacement)
|
|
72
|
+
return replacement;
|
|
73
|
+
}
|
|
74
|
+
if (signal?.aborted)
|
|
75
|
+
throw signal.reason ?? new Error('Web content fetch was cancelled');
|
|
76
|
+
return extractHtml(text, responseUrl, contentType || 'text/html');
|
|
60
77
|
}
|
|
61
|
-
if (contentType
|
|
62
|
-
return
|
|
63
|
-
if (contentType.includes('json') || looksLikeJson(text))
|
|
64
|
-
return extractJson(text, url, contentType);
|
|
78
|
+
if (isJson(contentType, text))
|
|
79
|
+
return extractJson(text, responseUrl, contentType || 'application/json');
|
|
65
80
|
return boundedExtracted({
|
|
66
|
-
url:
|
|
67
|
-
title: fileTitle(
|
|
81
|
+
url: responseUrl.toString(),
|
|
82
|
+
title: fileTitle(responseUrl, responseUrl.hostname),
|
|
68
83
|
content: text,
|
|
69
84
|
error: null,
|
|
70
85
|
contentType: contentType || 'text/plain',
|
|
71
86
|
});
|
|
72
87
|
}
|
|
73
|
-
|
|
88
|
+
async function getLlmsTxtReplacement(responseUrl, settings, probes, callerSignal, timeoutMs) {
|
|
89
|
+
const origin = responseUrl.origin;
|
|
90
|
+
let probe = probes.get(origin);
|
|
91
|
+
if (!probe) {
|
|
92
|
+
probe = probeLlmsTxt(new URL('/llms.txt', responseUrl.origin), settings, callerSignal, combinedSignal(callerSignal, timeoutMs));
|
|
93
|
+
probes.set(origin, probe);
|
|
94
|
+
}
|
|
95
|
+
const replacement = await probe;
|
|
96
|
+
if (callerSignal?.aborted)
|
|
97
|
+
throw callerSignal.reason ?? new Error('Web content fetch was cancelled');
|
|
98
|
+
return replacement;
|
|
99
|
+
}
|
|
100
|
+
async function probeLlmsTxt(llmsUrl, settings, callerSignal, requestSignal) {
|
|
101
|
+
try {
|
|
102
|
+
const validatedUrl = await validateRemoteUrl(llmsUrl, settings, { signal: requestSignal });
|
|
103
|
+
const response = await fetchRemoteUrl(validatedUrl, {
|
|
104
|
+
headers: {
|
|
105
|
+
Accept: 'text/plain,text/markdown,text/*;q=0.9',
|
|
106
|
+
'User-Agent': 'Felan-Web-Access/0.1',
|
|
107
|
+
},
|
|
108
|
+
signal: requestSignal,
|
|
109
|
+
}, settings, { allowCrossOriginRedirects: false });
|
|
110
|
+
const responseUrl = response.url ? new URL(response.url) : validatedUrl;
|
|
111
|
+
if (responseUrl.origin !== llmsUrl.origin) {
|
|
112
|
+
await response.body?.cancel();
|
|
113
|
+
return undefined;
|
|
114
|
+
}
|
|
115
|
+
if (!response.ok) {
|
|
116
|
+
await response.body?.cancel();
|
|
117
|
+
return undefined;
|
|
118
|
+
}
|
|
119
|
+
const contentType = normalizedContentType(response);
|
|
120
|
+
if (!isLlmsTxtContentType(contentType)) {
|
|
121
|
+
await response.body?.cancel();
|
|
122
|
+
return undefined;
|
|
123
|
+
}
|
|
124
|
+
const contentLength = declaredContentLength(response);
|
|
125
|
+
if (contentLength !== undefined
|
|
126
|
+
&& (!Number.isSafeInteger(contentLength) || contentLength > MAX_LLMS_TXT_RESPONSE_BYTES)) {
|
|
127
|
+
await response.body?.cancel();
|
|
128
|
+
return undefined;
|
|
129
|
+
}
|
|
130
|
+
const bytes = await readResponseBytes(response, MAX_LLMS_TXT_RESPONSE_BYTES, requestSignal);
|
|
131
|
+
if (isBinaryText(bytes))
|
|
132
|
+
return undefined;
|
|
133
|
+
let content;
|
|
134
|
+
try {
|
|
135
|
+
content = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
return undefined;
|
|
139
|
+
}
|
|
140
|
+
if (!content.trim() || isHtml(contentType, content))
|
|
141
|
+
return undefined;
|
|
142
|
+
return boundedExtracted({
|
|
143
|
+
url: responseUrl.toString(),
|
|
144
|
+
title: fileTitle(responseUrl, 'llms.txt'),
|
|
145
|
+
content,
|
|
146
|
+
error: null,
|
|
147
|
+
contentType: contentType || 'text/plain',
|
|
148
|
+
llmsTxtReplacement: true,
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
catch (error) {
|
|
152
|
+
if (callerSignal?.aborted)
|
|
153
|
+
throw callerSignal.reason ?? error;
|
|
154
|
+
return undefined;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
async function extractDeclaredPdf(response, responseUrl, contentType, limits, pdfEvents, callerSignal, requestSignal) {
|
|
158
|
+
const contentLengthHeader = response.headers.get('content-length');
|
|
159
|
+
const contentLength = contentLengthHeader !== null && /^\d+$/u.test(contentLengthHeader)
|
|
160
|
+
? Number(contentLengthHeader)
|
|
161
|
+
: undefined;
|
|
162
|
+
if (contentLength !== undefined && (!Number.isSafeInteger(contentLength) || contentLength > limits.maximumBytes)) {
|
|
163
|
+
await response.body?.cancel();
|
|
164
|
+
throw new Error(`PDF exceeds the configured ${limits.maximumBytes}-byte limit`);
|
|
165
|
+
}
|
|
166
|
+
let bytes;
|
|
167
|
+
try {
|
|
168
|
+
bytes = await readResponseBytes(response, limits.maximumBytes, requestSignal);
|
|
169
|
+
}
|
|
170
|
+
catch (error) {
|
|
171
|
+
if (error instanceof Error && error.message.startsWith('Response exceeds the ')) {
|
|
172
|
+
throw new Error(`PDF exceeds the configured ${limits.maximumBytes}-byte limit`);
|
|
173
|
+
}
|
|
174
|
+
throw error;
|
|
175
|
+
}
|
|
176
|
+
if (!hasPdfSignature(bytes)) {
|
|
177
|
+
if (contentType === 'application/pdf') {
|
|
178
|
+
throw new Error('PDF content type does not match the file signature');
|
|
179
|
+
}
|
|
180
|
+
throw new Error('Unsupported binary content: application/octet-stream');
|
|
181
|
+
}
|
|
182
|
+
if (callerSignal?.aborted)
|
|
183
|
+
throw callerSignal.reason ?? new Error('PDF conversion was cancelled');
|
|
184
|
+
const conversion = requestPdfConversion(pdfEvents, bytes, callerSignal);
|
|
185
|
+
if (!conversion)
|
|
186
|
+
throw new Error(unavailablePdfConversionError());
|
|
187
|
+
try {
|
|
188
|
+
const converted = await conversion;
|
|
189
|
+
if (!isPdfConversionResult(converted))
|
|
190
|
+
throw new Error('invalid conversion result');
|
|
191
|
+
return boundedExtracted({
|
|
192
|
+
url: responseUrl.toString(),
|
|
193
|
+
title: fileTitle(responseUrl, 'document.pdf'),
|
|
194
|
+
content: converted.markdown,
|
|
195
|
+
error: null,
|
|
196
|
+
contentType,
|
|
197
|
+
converter: 'MarkItDown',
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
catch (error) {
|
|
201
|
+
throw new Error(sanitizedPdfConversionError(error, callerSignal));
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
export async function fetchWithConcurrency(urls, concurrency, fetchOne, signal) {
|
|
74
205
|
const results = new Array(urls.length);
|
|
75
206
|
let next = 0;
|
|
76
207
|
const workers = Array.from({ length: Math.min(concurrency, urls.length) }, async () => {
|
|
77
208
|
while (next < urls.length) {
|
|
209
|
+
if (signal?.aborted)
|
|
210
|
+
throw signal.reason ?? new Error('Web content fetch was cancelled');
|
|
78
211
|
const index = next;
|
|
79
212
|
next += 1;
|
|
80
213
|
const url = urls[index];
|
|
@@ -82,13 +215,16 @@ export async function fetchWithConcurrency(urls, concurrency, fetchOne, captureE
|
|
|
82
215
|
results[index] = await fetchOne(url);
|
|
83
216
|
}
|
|
84
217
|
catch (error) {
|
|
85
|
-
if (
|
|
86
|
-
throw error;
|
|
218
|
+
if (signal?.aborted)
|
|
219
|
+
throw signal.reason ?? error;
|
|
87
220
|
results[index] = { url, title: '', content: '', error: errorMessage(error) };
|
|
88
221
|
}
|
|
89
222
|
}
|
|
90
223
|
});
|
|
91
|
-
await Promise.
|
|
224
|
+
const settled = await Promise.allSettled(workers);
|
|
225
|
+
const failure = settled.find((result) => result.status === 'rejected');
|
|
226
|
+
if (failure)
|
|
227
|
+
throw failure.reason;
|
|
92
228
|
return results;
|
|
93
229
|
}
|
|
94
230
|
function extractHtml(html, url, contentType) {
|
|
@@ -96,7 +232,9 @@ function extractHtml(html, url, contentType) {
|
|
|
96
232
|
const article = new Readability(document, { charThreshold: 20 }).parse();
|
|
97
233
|
const turndown = new TurndownService({ codeBlockStyle: 'fenced', headingStyle: 'atx' });
|
|
98
234
|
const title = article?.title?.trim() || document.title?.trim() || fileTitle(url, url.hostname);
|
|
99
|
-
const content = article?.content
|
|
235
|
+
const content = article?.content
|
|
236
|
+
? turndown.turndown(article.content)
|
|
237
|
+
: turndown.turndown(document.body?.innerHTML ?? html);
|
|
100
238
|
return boundedExtracted({ url: url.toString(), title, content: content.trim(), error: null, contentType });
|
|
101
239
|
}
|
|
102
240
|
function extractJson(text, url, contentType) {
|
|
@@ -105,38 +243,14 @@ function extractJson(text, url, contentType) {
|
|
|
105
243
|
content = JSON.stringify(JSON.parse(text), null, 2);
|
|
106
244
|
}
|
|
107
245
|
catch {
|
|
108
|
-
//
|
|
109
|
-
}
|
|
110
|
-
return boundedExtracted({ url: url.toString(), title: fileTitle(url, url.hostname), content, error: null, contentType });
|
|
111
|
-
}
|
|
112
|
-
async function extractPdf(bytes, url, config, contentType) {
|
|
113
|
-
if (typeof Promise.try !== 'function') {
|
|
114
|
-
const { default: promiseTry } = await import('promise.try');
|
|
115
|
-
promiseTry.shim();
|
|
116
|
-
}
|
|
117
|
-
const [{ getDocumentProxy }, pdfjs] = await Promise.all([import('unpdf'), import('unpdf/pdfjs')]);
|
|
118
|
-
const verbosity = pdfjs.VerbosityLevel.ERRORS;
|
|
119
|
-
const pdf = await getDocumentProxy(bytes, { verbosity });
|
|
120
|
-
const metadata = await pdf.getMetadata();
|
|
121
|
-
const info = metadata.info && typeof metadata.info === 'object' ? metadata.info : {};
|
|
122
|
-
const title = typeof info.Title === 'string' && info.Title.trim() ? info.Title.trim() : fileTitle(url, 'document');
|
|
123
|
-
const maximumPages = positiveInteger(config.pdf?.maxPages, DEFAULT_PDF_MAX_PAGES, 500);
|
|
124
|
-
const pageCount = Math.min(pdf.numPages, maximumPages);
|
|
125
|
-
const sections = [`# ${title}`, '', `Source: ${url.toString()}`, `Pages: ${pdf.numPages}`, ''];
|
|
126
|
-
for (let pageNumber = 1; pageNumber <= pageCount; pageNumber += 1) {
|
|
127
|
-
const page = await pdf.getPage(pageNumber);
|
|
128
|
-
const textContent = await page.getTextContent();
|
|
129
|
-
const pageText = textContent.items.map((item) => typeof item.str === 'string' ? item.str : '').join(' ').replace(/\s+/gu, ' ').trim();
|
|
130
|
-
if (pageText)
|
|
131
|
-
sections.push(`<!-- Page ${pageNumber} -->`, '', pageText, '');
|
|
246
|
+
// Preserve malformed JSON as bounded text so callers can still match it.
|
|
132
247
|
}
|
|
133
248
|
return boundedExtracted({
|
|
134
249
|
url: url.toString(),
|
|
135
|
-
title,
|
|
136
|
-
content
|
|
250
|
+
title: fileTitle(url, url.hostname),
|
|
251
|
+
content,
|
|
137
252
|
error: null,
|
|
138
253
|
contentType,
|
|
139
|
-
...(pdf.numPages > maximumPages ? { truncated: true } : {}),
|
|
140
254
|
});
|
|
141
255
|
}
|
|
142
256
|
function boundedExtracted(value) {
|
|
@@ -148,13 +262,52 @@ function boundedExtracted(value) {
|
|
|
148
262
|
truncated: true,
|
|
149
263
|
};
|
|
150
264
|
}
|
|
151
|
-
function
|
|
152
|
-
return contentType === '
|
|
265
|
+
function isSupportedContentType(contentType) {
|
|
266
|
+
return contentType === ''
|
|
267
|
+
|| contentType.startsWith('text/')
|
|
268
|
+
|| contentType === 'application/pdf'
|
|
269
|
+
|| contentType === 'application/json'
|
|
270
|
+
|| contentType.endsWith('+json')
|
|
271
|
+
|| contentType === 'application/xhtml+xml';
|
|
272
|
+
}
|
|
273
|
+
function normalizedContentType(response) {
|
|
274
|
+
return response.headers.get('content-type')?.split(';')[0]?.trim().toLowerCase() ?? '';
|
|
275
|
+
}
|
|
276
|
+
function declaredContentLength(response) {
|
|
277
|
+
const value = response.headers.get('content-length');
|
|
278
|
+
return value !== null && /^\d+$/u.test(value) ? Number(value) : undefined;
|
|
279
|
+
}
|
|
280
|
+
function isDirectLlmsTxt(url) {
|
|
281
|
+
return url.pathname.toLowerCase() === '/llms.txt';
|
|
282
|
+
}
|
|
283
|
+
function shouldProbeLlmsTxt(url, contentType, text) {
|
|
284
|
+
if (isDirectLlmsTxt(url))
|
|
285
|
+
return false;
|
|
286
|
+
if (contentType.includes('html'))
|
|
287
|
+
return true;
|
|
288
|
+
return contentType === '' && isHtml(contentType, text);
|
|
289
|
+
}
|
|
290
|
+
function isLlmsTxtContentType(contentType) {
|
|
291
|
+
return contentType === '' || (contentType.startsWith('text/') && !contentType.includes('html'));
|
|
292
|
+
}
|
|
293
|
+
function isBinaryText(bytes) {
|
|
294
|
+
for (const byte of bytes) {
|
|
295
|
+
if (byte === 0 || (byte < 0x20 && byte !== 0x09 && byte !== 0x0a && byte !== 0x0d))
|
|
296
|
+
return true;
|
|
297
|
+
}
|
|
298
|
+
return false;
|
|
299
|
+
}
|
|
300
|
+
function hasPdfSignature(bytes) {
|
|
301
|
+
if (bytes.byteLength < 8)
|
|
302
|
+
return false;
|
|
303
|
+
return PDF_SIGNATURE.test(new TextDecoder('ascii').decode(bytes.subarray(0, 8)));
|
|
153
304
|
}
|
|
154
|
-
function
|
|
155
|
-
return /<!doctype html|<html[\s>]|<body[\s>]/iu.test(text.slice(0, 2_000));
|
|
305
|
+
function isHtml(contentType, text) {
|
|
306
|
+
return contentType.includes('html') || /<!doctype html|<html[\s>]|<body[\s>]/iu.test(text.slice(0, 2_000));
|
|
156
307
|
}
|
|
157
|
-
function
|
|
308
|
+
function isJson(contentType, text) {
|
|
309
|
+
if (contentType.includes('json'))
|
|
310
|
+
return true;
|
|
158
311
|
const trimmed = text.trimStart();
|
|
159
312
|
return trimmed.startsWith('{') || trimmed.startsWith('[');
|
|
160
313
|
}
|
package/dist/extract.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract.js","sourceRoot":"","sources":["../src/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,eAAe,MAAM,UAAU,CAAC;AAEvC,OAAO,EAAE,eAAe,EAAE,cAAc,EAAwB,MAAM,aAAa,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAG5E,MAAM,uBAAuB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAChD,MAAM,eAAe,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AACxC,MAAM,wBAAwB,GAAG,OAAO,CAAC;AACzC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AACnC,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAC/B,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,MAAM,eAAe,GAAG,MAAM,CAAC;AAQ/B,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,OAAqB,EACrB,MAAuB,EACvB,MAAoB,EACpB,UAA0B,EAAE;IAE5B,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC1F,IAAI,MAAM,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QACrC,OAAO,uBAAuB,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IACtG,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE;QACzC,OAAO,EAAE;YACP,MAAM,EAAE,qGAAqG;YAC7G,YAAY,EAAE,sBAAsB;SACrC;QACD,MAAM,EAAE,cAAc,CAAC,MAAM,EAAE,eAAe,CAAC;KAChD,EAAE,QAAQ,CAAC,CAAC;IACb,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;IACrG,CAAC;IACD,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;IACpG,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACrE,IAAI,WAAW,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,aAAa,GAAG,eAAe,EAAE,CAAC;QAC1G,MAAM,IAAI,KAAK,CAAC,qBAAqB,eAAe,aAAa,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,eAAe,GAAG,cAAc,CAAC,MAAM,CAAC,GAAG,EAAE,SAAS,EAAE,uBAAuB,EAAE,mBAAmB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;IAC1H,IAAI,KAAK,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,aAAa,GAAG,eAAe,EAAE,CAAC;QACjG,MAAM,IAAI,KAAK,CAAC,8BAA8B,eAAe,aAAa,CAAC,CAAC;IAC9E,CAAC;IACD,MAAM,YAAY,GAAG,WAAW,CAAC,UAAU,CAAC,QAAQ,CAAC;QACnD,CAAC,CAAC,eAAe;QACjB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,uBAAuB,CAAC;IACxE,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAE9D,IAAI,WAAW,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE;YACnB,KAAK,EAAE,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC;YAC9B,OAAO,EAAE,kBAAkB,WAAW,KAAK,KAAK,CAAC,UAAU,SAAS;YACpE,KAAK,EAAE,IAAI;YACX,WAAW;YACX,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE;SAC9E,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,CAAC,GAAG,EAAE,WAAW,CAAC;QAAE,OAAO,UAAU,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IAEhF,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7C,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QAC3B,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;IAC/G,CAAC;IACD,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC;IACpG,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC;QAAE,OAAO,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC;IACpG,OAAO,gBAAgB,CAAC;QACtB,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE;QACnB,KAAK,EAAE,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC;QACnC,OAAO,EAAE,IAAI;QACb,KAAK,EAAE,IAAI;QACX,WAAW,EAAE,WAAW,IAAI,YAAY;KACzC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAc,EACd,WAAmB,EACnB,QAAoD,EACpD,aAAa,GAAG,IAAI;IAEpB,MAAM,OAAO,GAAG,IAAI,KAAK,CAAmB,IAAI,CAAC,MAAM,CAAC,CAAC;IACzD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QACpF,OAAO,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,IAAI,CAAC;YACnB,IAAI,IAAI,CAAC,CAAC;YACV,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAE,CAAC;YACzB,IAAI,CAAC;gBACH,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,CAAC;YACvC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,aAAa;oBAAE,MAAM,KAAK,CAAC;gBAChC,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/E,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,GAAQ,EAAE,WAAmB;IAC9D,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,QAA+B,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IAChG,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,cAAc,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IACxF,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC/F,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,IAAI,IAAI,CAAC,CAAC;IAC5H,OAAO,gBAAgB,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;AAC7G,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,GAAQ,EAAE,WAAmB;IAC9D,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,IAAI,CAAC;QACH,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,8CAA8C;IAChD,CAAC;IACD,OAAO,gBAAgB,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;AAC3H,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,KAAiB,EAAE,GAAQ,EAAE,MAAuB,EAAE,WAAmB;IACjG,IAAI,OAAQ,OAAkD,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;QAClF,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QAC5D,UAAU,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IACD,MAAM,CAAC,EAAE,gBAAgB,EAAE,EAAE,KAAK,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAClG,MAAM,SAAS,GAAI,KAA+D,CAAC,cAAc,CAAC,MAAM,CAAC;IACzG,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,IAAI,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,IAA+B,CAAC,CAAC,CAAC,EAAE,CAAC;IAChH,MAAM,KAAK,GAAG,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IACnH,MAAM,YAAY,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,qBAAqB,EAAE,GAAG,CAAC,CAAC;IACvF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACvD,MAAM,QAAQ,GAAG,CAAC,KAAK,KAAK,EAAE,EAAE,EAAE,EAAE,WAAW,GAAG,CAAC,QAAQ,EAAE,EAAE,EAAE,UAAU,GAAG,CAAC,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IAC/F,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,IAAI,SAAS,EAAE,UAAU,IAAI,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAChD,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAa,EAAE,EAAE,CAAC,OAAQ,IAA0B,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAE,IAAwB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3L,IAAI,QAAQ;YAAE,QAAQ,CAAC,IAAI,CAAC,aAAa,UAAU,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,gBAAgB,CAAC;QACtB,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE;QACnB,KAAK;QACL,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE;QACnC,KAAK,EAAE,IAAI;QACX,WAAW;QACX,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5D,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAuB;IAC/C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,IAAI,wBAAwB;QAAE,OAAO,KAAK,CAAC;IACnE,OAAO;QACL,GAAG,KAAK;QACR,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,wBAAwB,CAAC;QACzD,SAAS,EAAE,IAAI;KAChB,CAAC;AACJ,CAAC;AAED,SAAS,KAAK,CAAC,GAAQ,EAAE,WAAmB;IAC1C,OAAO,WAAW,KAAK,iBAAiB,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAC1F,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,wCAAwC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;IACjC,OAAO,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,SAAS,CAAC,GAAQ,EAAE,QAAgB;IAC3C,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,OAAO,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC1D,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC"}
|
|
1
|
+
{"version":3,"file":"extract.js","sourceRoot":"","sources":["../src/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,WAAW,EAAwB,MAAM,aAAa,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC9D,OAAO,EACL,qBAAqB,EACrB,oBAAoB,EACpB,2BAA2B,EAC3B,6BAA6B,GAE9B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAG5E,MAAM,uBAAuB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAChD,MAAM,2BAA2B,GAAG,IAAI,GAAG,IAAI,CAAC;AAChD,MAAM,wBAAwB,GAAG,OAAO,CAAC;AACzC,MAAM,eAAe,GAAG,MAAM,CAAC;AAC/B,MAAM,aAAa,GAAG,qBAAqB,CAAC;AAS5C,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,MAAuB,EACvB,SAA+B,EAC/B,MAAoB,EACpB,SAAS,GAAG,eAAe,EAC3B,UAAiC,EAAE;IAEnC,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,aAAa,GAAG,cAAc,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACxD,MAAM,YAAY,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;IAC1F,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,YAAY,EAAE;QAClD,OAAO,EAAE;YACP,MAAM,EAAE,gGAAgG;YACxG,YAAY,EAAE,sBAAsB;SACrC;QACD,MAAM,EAAE,aAAa;KACtB,EAAE,QAAQ,CAAC,CAAC;IACb,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;IACxE,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAC9B,OAAO;YACL,GAAG,EAAE,WAAW,CAAC,QAAQ,EAAE;YAC3B,KAAK,EAAE,WAAW,CAAC,QAAQ;YAC3B,OAAO,EAAE,EAAE;YACX,KAAK,EAAE,QAAQ,QAAQ,CAAC,MAAM,EAAE;SACjC,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;IACpG,IAAI,CAAC,sBAAsB,CAAC,WAAW,CAAC,IAAI,WAAW,KAAK,0BAA0B,EAAE,CAAC;QACvF,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,6BAA6B,WAAW,IAAI,SAAS,EAAE,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,WAAW,GAAG,WAAW,KAAK,iBAAiB,IAAI,WAAW,KAAK,0BAA0B,CAAC;IACpG,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QACnC,IAAI,CAAC;YACH,OAAO,MAAM,kBAAkB,CAC7B,QAAQ,EACR,WAAW,EACX,WAAW,EACX,MAAM,EACN,SAAS,EACT,MAAM,EACN,aAAa,CACd,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACrD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,uBAAuB,CAAC;IAC7C,MAAM,mBAAmB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACnE,MAAM,aAAa,GAAG,mBAAmB,KAAK,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC;QACtF,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC;QAC7B,CAAC,CAAC,SAAS,CAAC;IACd,IAAI,aAAa,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,IAAI,aAAa,GAAG,YAAY,CAAC,EAAE,CAAC;QAC1G,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,wBAAwB,YAAY,aAAa,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,QAAQ,EAAE,YAAY,EAAE,aAAa,CAAC,CAAC;IAE7E,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IAC5C,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,mDAAmD,WAAW,IAAI,SAAS,EAAE,CAAC,CAAC;IACjG,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7C,IAAI,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,CAAC;QAC9B,IAAI,CAAC,OAAO,CAAC,aAAa;eACrB,CAAC,eAAe,CAAC,YAAY,CAAC;eAC9B,CAAC,eAAe,CAAC,WAAW,CAAC;eAC7B,kBAAkB,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,CAAC,EAAE,CAAC;YACxD,MAAM,WAAW,GAAG,MAAM,qBAAqB,CAC7C,WAAW,EACX,QAAQ,EACR,OAAO,CAAC,aAAa,IAAI,IAAI,GAAG,EAAE,EAClC,MAAM,EACN,SAAS,CACV,CAAC;YACF,IAAI,WAAW;gBAAE,OAAO,WAAW,CAAC;QACtC,CAAC;QACD,IAAI,MAAM,EAAE,OAAO;YAAE,MAAM,MAAM,CAAC,MAAM,IAAI,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACzF,OAAO,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,WAAW,IAAI,WAAW,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC;QAAE,OAAO,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,WAAW,IAAI,kBAAkB,CAAC,CAAC;IACxG,OAAO,gBAAgB,CAAC;QACtB,GAAG,EAAE,WAAW,CAAC,QAAQ,EAAE;QAC3B,KAAK,EAAE,SAAS,CAAC,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC;QACnD,OAAO,EAAE,IAAI;QACb,KAAK,EAAE,IAAI;QACX,WAAW,EAAE,WAAW,IAAI,YAAY;KACzC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,WAAgB,EAChB,QAAyC,EACzC,MAAuB,EACvB,YAAqC,EACrC,SAAiB;IAEjB,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;IAClC,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC/B,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,YAAY,CAClB,IAAI,GAAG,CAAC,WAAW,EAAE,WAAW,CAAC,MAAM,CAAC,EACxC,QAAQ,EACR,YAAY,EACZ,cAAc,CAAC,YAAY,EAAE,SAAS,CAAC,CACxC,CAAC;QACF,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;IACD,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC;IAChC,IAAI,YAAY,EAAE,OAAO;QAAE,MAAM,YAAY,CAAC,MAAM,IAAI,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrG,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,OAAY,EACZ,QAAyC,EACzC,YAAqC,EACrC,aAA0B;IAE1B,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,iBAAiB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAC3F,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,YAAY,EAAE;YAClD,OAAO,EAAE;gBACP,MAAM,EAAE,uCAAuC;gBAC/C,YAAY,EAAE,sBAAsB;aACrC;YACD,MAAM,EAAE,aAAa;SACtB,EAAE,QAAQ,EAAE,EAAE,yBAAyB,EAAE,KAAK,EAAE,CAAC,CAAC;QACnD,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;QACxE,IAAI,WAAW,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;YAC1C,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;YAC9B,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;YAC9B,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,WAAW,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,EAAE,CAAC;YACvC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;YAC9B,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,aAAa,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QACtD,IAAI,aAAa,KAAK,SAAS;eAC1B,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,IAAI,aAAa,GAAG,2BAA2B,CAAC,EAAE,CAAC;YAC3F,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;YAC9B,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,QAAQ,EAAE,2BAA2B,EAAE,aAAa,CAAC,CAAC;QAC5F,IAAI,YAAY,CAAC,KAAK,CAAC;YAAE,OAAO,SAAS,CAAC;QAC1C,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACH,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACpE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC;YAAE,OAAO,SAAS,CAAC;QACtE,OAAO,gBAAgB,CAAC;YACtB,GAAG,EAAE,WAAW,CAAC,QAAQ,EAAE;YAC3B,KAAK,EAAE,SAAS,CAAC,WAAW,EAAE,UAAU,CAAC;YACzC,OAAO;YACP,KAAK,EAAE,IAAI;YACX,WAAW,EAAE,WAAW,IAAI,YAAY;YACxC,kBAAkB,EAAE,IAAI;SACzB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,YAAY,EAAE,OAAO;YAAE,MAAM,YAAY,CAAC,MAAM,IAAI,KAAK,CAAC;QAC9D,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC/B,QAAkB,EAClB,WAAgB,EAChB,WAAmB,EACnB,MAAsC,EACtC,SAA0C,EAC1C,YAAqC,EACrC,aAA0B;IAE1B,MAAM,mBAAmB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACnE,MAAM,aAAa,GAAG,mBAAmB,KAAK,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC;QACtF,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC;QAC7B,CAAC,CAAC,SAAS,CAAC;IACd,IAAI,aAAa,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,IAAI,aAAa,GAAG,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;QACjH,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,8BAA8B,MAAM,CAAC,YAAY,aAAa,CAAC,CAAC;IAClF,CAAC;IAED,IAAI,KAAiB,CAAC;IACtB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAChF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,uBAAuB,CAAC,EAAE,CAAC;YAChF,MAAM,IAAI,KAAK,CAAC,8BAA8B,MAAM,CAAC,YAAY,aAAa,CAAC,CAAC;QAClF,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5B,IAAI,WAAW,KAAK,iBAAiB,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACxE,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IACD,IAAI,YAAY,EAAE,OAAO;QAAE,MAAM,YAAY,CAAC,MAAM,IAAI,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAClG,MAAM,UAAU,GAAG,oBAAoB,CAAC,SAAS,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;IACxE,IAAI,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,EAAE,CAAC,CAAC;IAClE,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC;QACnC,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QACpF,OAAO,gBAAgB,CAAC;YACtB,GAAG,EAAE,WAAW,CAAC,QAAQ,EAAE;YAC3B,KAAK,EAAE,SAAS,CAAC,WAAW,EAAE,cAAc,CAAC;YAC7C,OAAO,EAAE,SAAS,CAAC,QAAQ;YAC3B,KAAK,EAAE,IAAI;YACX,WAAW;YACX,SAAS,EAAE,YAAY;SACxB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC;IACpE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAc,EACd,WAAmB,EACnB,QAAoD,EACpD,MAAoB;IAEpB,MAAM,OAAO,GAAG,IAAI,KAAK,CAAmB,IAAI,CAAC,MAAM,CAAC,CAAC;IACzD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QACpF,OAAO,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,MAAM,CAAC,MAAM,IAAI,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;YACzF,MAAM,KAAK,GAAG,IAAI,CAAC;YACnB,IAAI,IAAI,CAAC,CAAC;YACV,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAE,CAAC;YACzB,IAAI,CAAC;gBACH,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,CAAC;YACvC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,MAAM,EAAE,OAAO;oBAAE,MAAM,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC;gBAClD,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/E,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAmC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;IACxG,IAAI,OAAO;QAAE,MAAM,OAAO,CAAC,MAAM,CAAC;IAClC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,GAAQ,EAAE,WAAmB;IAC9D,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,QAA+B,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IAChG,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,cAAc,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IACxF,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC/F,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO;QAC9B,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC;QACpC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,IAAI,IAAI,CAAC,CAAC;IACxD,OAAO,gBAAgB,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;AAC7G,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,GAAQ,EAAE,WAAmB;IAC9D,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,IAAI,CAAC;QACH,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,yEAAyE;IAC3E,CAAC;IACD,OAAO,gBAAgB,CAAC;QACtB,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE;QACnB,KAAK,EAAE,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC;QACnC,OAAO;QACP,KAAK,EAAE,IAAI;QACX,WAAW;KACZ,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAuB;IAC/C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,IAAI,wBAAwB;QAAE,OAAO,KAAK,CAAC;IACnE,OAAO;QACL,GAAG,KAAK;QACR,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,wBAAwB,CAAC;QACzD,SAAS,EAAE,IAAI;KAChB,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,WAAmB;IACjD,OAAO,WAAW,KAAK,EAAE;WACpB,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC;WAC/B,WAAW,KAAK,iBAAiB;WACjC,WAAW,KAAK,kBAAkB;WAClC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC;WAC7B,WAAW,KAAK,uBAAuB,CAAC;AAC/C,CAAC;AAED,SAAS,qBAAqB,CAAC,QAAkB;IAC/C,OAAO,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;AACzF,CAAC;AAED,SAAS,qBAAqB,CAAC,QAAkB;IAC/C,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACrD,OAAO,KAAK,KAAK,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC5E,CAAC;AAED,SAAS,eAAe,CAAC,GAAQ;IAC/B,OAAO,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,WAAW,CAAC;AACpD,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAQ,EAAE,WAAmB,EAAE,IAAY;IACrE,IAAI,eAAe,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACvC,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9C,OAAO,WAAW,KAAK,EAAE,IAAI,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,oBAAoB,CAAC,WAAmB;IAC/C,OAAO,WAAW,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;AAClG,CAAC;AAED,SAAS,YAAY,CAAC,KAAiB;IACrC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;IAClG,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,eAAe,CAAC,KAAiB;IACxC,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACvC,OAAO,aAAa,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AACnF,CAAC;AAED,SAAS,MAAM,CAAC,WAAmB,EAAE,IAAY;IAC/C,OAAO,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,wCAAwC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AAC7G,CAAC;AAED,SAAS,MAAM,CAAC,WAAmB,EAAE,IAAY;IAC/C,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;IACjC,OAAO,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,SAAS,CAAC,GAAQ,EAAE,QAAgB;IAC3C,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,OAAO,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC1D,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC"}
|
package/dist/http.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export declare function readResponseBytes(response: Response, maximumBytes: number): Promise<Uint8Array>;
|
|
2
|
-
export declare function readResponseText(response: Response, maximumBytes: number): Promise<string>;
|
|
3
|
-
export declare function readJsonResponse<T>(response: Response, maximumBytes: number, label: string): Promise<T>;
|
|
1
|
+
export declare function readResponseBytes(response: Response, maximumBytes: number, signal?: AbortSignal): Promise<Uint8Array>;
|
|
2
|
+
export declare function readResponseText(response: Response, maximumBytes: number, signal?: AbortSignal): Promise<string>;
|
|
3
|
+
export declare function readJsonResponse<T>(response: Response, maximumBytes: number, label: string, signal?: AbortSignal): Promise<T>;
|
|
4
4
|
export declare function combinedSignal(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal;
|
|
5
5
|
//# sourceMappingURL=http.d.ts.map
|
package/dist/http.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAAA,wBAAsB,iBAAiB,
|
|
1
|
+
{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAAA,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,QAAQ,EAClB,YAAY,EAAE,MAAM,EACpB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,UAAU,CAAC,CAqCrB;AAED,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,QAAQ,EAClB,YAAY,EAAE,MAAM,EACpB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,MAAM,CAAC,CAEjB;AAED,wBAAsB,gBAAgB,CAAC,CAAC,EACtC,QAAQ,EAAE,QAAQ,EAClB,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,CAAC,CAAC,CAOZ;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,WAAW,CAG9F"}
|
package/dist/http.js
CHANGED
|
@@ -1,12 +1,20 @@
|
|
|
1
|
-
export async function readResponseBytes(response, maximumBytes) {
|
|
1
|
+
export async function readResponseBytes(response, maximumBytes, signal) {
|
|
2
2
|
const reader = response.body?.getReader();
|
|
3
3
|
if (!reader)
|
|
4
4
|
return new Uint8Array();
|
|
5
5
|
const chunks = [];
|
|
6
6
|
let total = 0;
|
|
7
|
+
const abort = () => { void reader.cancel(signal?.reason).catch(() => undefined); };
|
|
8
|
+
signal?.addEventListener('abort', abort, { once: true });
|
|
7
9
|
try {
|
|
10
|
+
if (signal?.aborted) {
|
|
11
|
+
await reader.cancel(signal.reason).catch(() => undefined);
|
|
12
|
+
throw signal.reason ?? new Error('Response read was cancelled');
|
|
13
|
+
}
|
|
8
14
|
while (true) {
|
|
9
15
|
const { done, value } = await reader.read();
|
|
16
|
+
if (signal?.aborted)
|
|
17
|
+
throw signal.reason ?? new Error('Response read was cancelled');
|
|
10
18
|
if (done)
|
|
11
19
|
break;
|
|
12
20
|
total += value.byteLength;
|
|
@@ -17,7 +25,13 @@ export async function readResponseBytes(response, maximumBytes) {
|
|
|
17
25
|
chunks.push(value);
|
|
18
26
|
}
|
|
19
27
|
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
if (signal?.aborted)
|
|
30
|
+
throw signal.reason ?? error;
|
|
31
|
+
throw error;
|
|
32
|
+
}
|
|
20
33
|
finally {
|
|
34
|
+
signal?.removeEventListener('abort', abort);
|
|
21
35
|
reader.releaseLock();
|
|
22
36
|
}
|
|
23
37
|
const bytes = new Uint8Array(total);
|
|
@@ -28,11 +42,11 @@ export async function readResponseBytes(response, maximumBytes) {
|
|
|
28
42
|
}
|
|
29
43
|
return bytes;
|
|
30
44
|
}
|
|
31
|
-
export async function readResponseText(response, maximumBytes) {
|
|
32
|
-
return new TextDecoder().decode(await readResponseBytes(response, maximumBytes));
|
|
45
|
+
export async function readResponseText(response, maximumBytes, signal) {
|
|
46
|
+
return new TextDecoder().decode(await readResponseBytes(response, maximumBytes, signal));
|
|
33
47
|
}
|
|
34
|
-
export async function readJsonResponse(response, maximumBytes, label) {
|
|
35
|
-
const text = await readResponseText(response, maximumBytes);
|
|
48
|
+
export async function readJsonResponse(response, maximumBytes, label, signal) {
|
|
49
|
+
const text = await readResponseText(response, maximumBytes, signal);
|
|
36
50
|
try {
|
|
37
51
|
return JSON.parse(text);
|
|
38
52
|
}
|
package/dist/http.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http.js","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,KAAK,UAAU,iBAAiB,
|
|
1
|
+
{"version":3,"file":"http.js","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,QAAkB,EAClB,YAAoB,EACpB,MAAoB;IAEpB,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;IAC1C,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,UAAU,EAAE,CAAC;IACrC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,KAAK,GAAG,GAAG,EAAE,GAAG,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,IAAI,CAAC;QACH,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;YACpB,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC1D,MAAM,MAAM,CAAC,MAAM,IAAI,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,MAAM,CAAC,MAAM,IAAI,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;YACrF,IAAI,IAAI;gBAAE,MAAM;YAChB,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;YAC1B,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;gBACzB,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,wBAAwB,YAAY,aAAa,CAAC,CAAC;YACrE,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,MAAM,EAAE,OAAO;YAAE,MAAM,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC;QAClD,MAAM,KAAK,CAAC;IACd,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC5C,MAAM,CAAC,WAAW,EAAE,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IACpC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;IAC7B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,QAAkB,EAClB,YAAoB,EACpB,MAAoB;IAEpB,OAAO,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,MAAM,iBAAiB,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,QAAkB,EAClB,YAAoB,EACpB,KAAa,EACb,MAAoB;IAEpB,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IACpE,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAM,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,wBAAwB,CAAC,CAAC;IACpD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAA+B,EAAE,SAAiB;IAC/E,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/C,OAAO,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;AAC/D,CAAC"}
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAgH3D,QAAA,MAAM,kBAAkB,EAAE,cAyGzB,CAAC;AA2XF,OAAO,EAAE,iBAAiB,EAAE,2BAA2B,EAAE,MAAM,aAAa,CAAC;AAC7E,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACnD,eAAe,kBAAkB,CAAC"}
|