@j0hanz/fetch-url-mcp 1.12.6 → 1.12.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/http/auth.d.ts +2 -2
- package/dist/http/auth.d.ts.map +1 -1
- package/dist/http/auth.js +15 -16
- package/dist/http/index.d.ts +6 -0
- package/dist/http/index.d.ts.map +1 -0
- package/dist/http/index.js +5 -0
- package/dist/http/native.d.ts +73 -0
- package/dist/http/native.d.ts.map +1 -1
- package/dist/http/native.js +585 -62
- package/dist/http/rate-limit.d.ts +1 -1
- package/dist/http/rate-limit.d.ts.map +1 -1
- package/dist/http/rate-limit.js +5 -6
- package/dist/index.d.ts +17 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +69 -8
- package/dist/lib/config.js +2 -2
- package/dist/lib/core.d.ts +56 -4
- package/dist/lib/core.d.ts.map +1 -1
- package/dist/lib/core.js +162 -11
- package/dist/lib/error/classes.d.ts +19 -0
- package/dist/lib/error/classes.d.ts.map +1 -0
- package/dist/lib/error/classes.js +107 -0
- package/dist/lib/error/classify.d.ts +4 -0
- package/dist/lib/error/classify.d.ts.map +1 -0
- package/dist/lib/error/classify.js +154 -0
- package/dist/lib/error/codes.d.ts +23 -0
- package/dist/lib/error/codes.d.ts.map +1 -0
- package/dist/lib/error/codes.js +22 -0
- package/dist/lib/error/index.d.ts +6 -0
- package/dist/lib/error/index.d.ts.map +1 -0
- package/dist/lib/error/index.js +5 -0
- package/dist/lib/{error-messages.d.ts → error/messages.d.ts} +2 -2
- package/dist/lib/error/messages.d.ts.map +1 -0
- package/dist/lib/{error-messages.js → error/messages.js} +13 -13
- package/dist/lib/{tool-errors.d.ts → error/payload.d.ts} +7 -13
- package/dist/lib/error/payload.d.ts.map +1 -0
- package/dist/lib/error/payload.js +108 -0
- package/dist/lib/mcp-interop.d.ts +1 -0
- package/dist/lib/mcp-interop.d.ts.map +1 -1
- package/dist/lib/mcp-interop.js +17 -9
- package/dist/lib/net/http.d.ts.map +1 -0
- package/dist/lib/{http.js → net/http.js} +11 -14
- package/dist/lib/net/index.d.ts +4 -0
- package/dist/lib/net/index.d.ts.map +1 -0
- package/dist/lib/net/index.js +3 -0
- package/dist/lib/{fetch-pipeline.d.ts → net/pipeline.d.ts} +3 -3
- package/dist/lib/net/pipeline.d.ts.map +1 -0
- package/dist/lib/{fetch-pipeline.js → net/pipeline.js} +7 -9
- package/dist/lib/{url.d.ts → net/url.d.ts} +2 -2
- package/dist/lib/net/url.d.ts.map +1 -0
- package/dist/lib/{url.js → net/url.js} +6 -8
- package/dist/lib/utils.d.ts +3 -18
- package/dist/lib/utils.d.ts.map +1 -1
- package/dist/lib/utils.js +33 -105
- package/dist/resources/index.d.ts.map +1 -1
- package/dist/resources/index.js +6 -3
- package/dist/schemas.d.ts +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +12 -14
- package/dist/tasks/index.d.ts +2 -0
- package/dist/tasks/index.d.ts.map +1 -0
- package/dist/tasks/index.js +1 -0
- package/dist/tasks/manager.d.ts +123 -1
- package/dist/tasks/manager.d.ts.map +1 -1
- package/dist/tasks/manager.js +753 -18
- package/dist/tools/{fetch-url.d.ts → index.d.ts} +4 -5
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/{fetch-url.js → index.js} +14 -31
- package/dist/transform/index.d.ts +279 -0
- package/dist/transform/index.d.ts.map +1 -0
- package/dist/transform/index.js +5234 -0
- package/package.json +2 -2
- package/dist/cli.d.ts +0 -19
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js +0 -65
- package/dist/http/health.d.ts +0 -8
- package/dist/http/health.d.ts.map +0 -1
- package/dist/http/health.js +0 -152
- package/dist/http/helpers.d.ts +0 -68
- package/dist/http/helpers.d.ts.map +0 -1
- package/dist/http/helpers.js +0 -404
- package/dist/lib/error-codes.d.ts +0 -11
- package/dist/lib/error-codes.d.ts.map +0 -1
- package/dist/lib/error-codes.js +0 -15
- package/dist/lib/error-messages.d.ts.map +0 -1
- package/dist/lib/fetch-pipeline.d.ts.map +0 -1
- package/dist/lib/http.d.ts.map +0 -1
- package/dist/lib/logger-names.d.ts +0 -14
- package/dist/lib/logger-names.d.ts.map +0 -1
- package/dist/lib/logger-names.js +0 -13
- package/dist/lib/session.d.ts +0 -44
- package/dist/lib/session.d.ts.map +0 -1
- package/dist/lib/session.js +0 -137
- package/dist/lib/tool-errors.d.ts.map +0 -1
- package/dist/lib/tool-errors.js +0 -252
- package/dist/lib/url.d.ts.map +0 -1
- package/dist/lib/zod.d.ts +0 -3
- package/dist/lib/zod.d.ts.map +0 -1
- package/dist/lib/zod.js +0 -27
- package/dist/tasks/call-contract.d.ts +0 -25
- package/dist/tasks/call-contract.d.ts.map +0 -1
- package/dist/tasks/call-contract.js +0 -59
- package/dist/tasks/execution.d.ts +0 -16
- package/dist/tasks/execution.d.ts.map +0 -1
- package/dist/tasks/execution.js +0 -241
- package/dist/tasks/handlers.d.ts +0 -11
- package/dist/tasks/handlers.d.ts.map +0 -1
- package/dist/tasks/handlers.js +0 -157
- package/dist/tasks/owner.d.ts +0 -43
- package/dist/tasks/owner.d.ts.map +0 -1
- package/dist/tasks/owner.js +0 -144
- package/dist/tasks/registry.d.ts +0 -20
- package/dist/tasks/registry.d.ts.map +0 -1
- package/dist/tasks/registry.js +0 -40
- package/dist/tasks/waiters.d.ts +0 -27
- package/dist/tasks/waiters.d.ts.map +0 -1
- package/dist/tasks/waiters.js +0 -114
- package/dist/tools/fetch-url.d.ts.map +0 -1
- package/dist/transform/dom-prep.d.ts +0 -16
- package/dist/transform/dom-prep.d.ts.map +0 -1
- package/dist/transform/dom-prep.js +0 -1287
- package/dist/transform/html-translators.d.ts +0 -5
- package/dist/transform/html-translators.d.ts.map +0 -1
- package/dist/transform/html-translators.js +0 -697
- package/dist/transform/markdown-cleanup.d.ts +0 -10
- package/dist/transform/markdown-cleanup.d.ts.map +0 -1
- package/dist/transform/markdown-cleanup.js +0 -542
- package/dist/transform/metadata.d.ts +0 -18
- package/dist/transform/metadata.d.ts.map +0 -1
- package/dist/transform/metadata.js +0 -462
- package/dist/transform/next-flight.d.ts +0 -2
- package/dist/transform/next-flight.d.ts.map +0 -1
- package/dist/transform/next-flight.js +0 -374
- package/dist/transform/shared.d.ts +0 -8
- package/dist/transform/shared.d.ts.map +0 -1
- package/dist/transform/shared.js +0 -137
- package/dist/transform/transform.d.ts +0 -38
- package/dist/transform/transform.d.ts.map +0 -1
- package/dist/transform/transform.js +0 -1041
- package/dist/transform/types.d.ts +0 -124
- package/dist/transform/types.d.ts.map +0 -1
- package/dist/transform/types.js +0 -5
- package/dist/transform/worker-pool.d.ts +0 -76
- package/dist/transform/worker-pool.d.ts.map +0 -1
- package/dist/transform/worker-pool.js +0 -725
- /package/dist/lib/{http.d.ts → net/http.d.ts} +0 -0
|
@@ -1,374 +0,0 @@
|
|
|
1
|
-
import { detectLanguageFromCode } from './html-translators.js';
|
|
2
|
-
const NEXT_FLIGHT_PAYLOAD_RE = /self\.__next_f\.push\(\[1,"((?:\\.|[^"\\])*)"\]\)<\/script>/gs;
|
|
3
|
-
const TEMPLATE_ASSIGNMENT_RE = /([A-Za-z_$][\w$]*)=`([\s\S]*?)`;/g;
|
|
4
|
-
const FLIGHT_INSTALL_RE = /commands:\{cli:"([^"]+)",npm:"([^"]+)",yarn:"([^"]+)",pnpm:"([^"]+)",bun:"([^"]+)"\}/;
|
|
5
|
-
const FLIGHT_IMPORT_RE = /commands:\{main:'([^']+)',individual:'([^']+)'\}/;
|
|
6
|
-
const FLIGHT_DEMO_RE = /title:"((?:\\.|[^"\\])*)",files:([A-Za-z_$][\w$]*)\.([A-Za-z_$][\w$]*)/g;
|
|
7
|
-
const FLIGHT_API_RE = /children:"([^"]+)"\}\),`\\n`,\(0,e\.jsx\)\(o,\{data:\[([\s\S]*?)\]\}\)/g;
|
|
8
|
-
const FLIGHT_API_ROW_RE = /attribute:"((?:\\.|[^"\\])*)",type:"((?:\\.|[^"\\])*)",description:"((?:\\.|[^"\\])*)",default:"((?:\\.|[^"\\])*)"/g;
|
|
9
|
-
const FLIGHT_MERMAID_SECTION_RE = /_jsx\(Heading,\{\s*level:"[1-6]",\s*id:"[^"]+",\s*children:"((?:\\.|[^"\\])*)"\s*\}\)(?:(?!_jsx\(Heading,\{)[\s\S]){0,12000}?_jsx\(Mermaid,\{\s*chart:"((?:\\.|[^"\\])*)"\s*\}\)/g;
|
|
10
|
-
function decodeHtmlEntities(value) {
|
|
11
|
-
return value
|
|
12
|
-
.replace(/'|'/g, "'")
|
|
13
|
-
.replace(/"/g, '"')
|
|
14
|
-
.replace(/&/g, '&')
|
|
15
|
-
.replace(/</g, '<')
|
|
16
|
-
.replace(/>/g, '>');
|
|
17
|
-
}
|
|
18
|
-
function decodeJsonStringLiteral(value) {
|
|
19
|
-
const decoded = JSON.parse(`"${value}"`);
|
|
20
|
-
return typeof decoded === 'string' ? decoded : undefined;
|
|
21
|
-
}
|
|
22
|
-
function decodeFlightStringValue(value) {
|
|
23
|
-
try {
|
|
24
|
-
return decodeJsonStringLiteral(value) ?? decodeHtmlEntities(value);
|
|
25
|
-
}
|
|
26
|
-
catch {
|
|
27
|
-
return decodeHtmlEntities(value);
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
function decodeNextFlightPayloads(html) {
|
|
31
|
-
const payloads = [];
|
|
32
|
-
for (const match of html.matchAll(NEXT_FLIGHT_PAYLOAD_RE)) {
|
|
33
|
-
const rawPayload = match[1];
|
|
34
|
-
if (!rawPayload)
|
|
35
|
-
continue;
|
|
36
|
-
try {
|
|
37
|
-
const decodedPayload = decodeJsonStringLiteral(rawPayload);
|
|
38
|
-
if (decodedPayload)
|
|
39
|
-
payloads.push(decodedPayload);
|
|
40
|
-
}
|
|
41
|
-
catch {
|
|
42
|
-
// Ignore malformed payload fragments and continue with the rest.
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
return payloads;
|
|
46
|
-
}
|
|
47
|
-
function parseObjectEntries(body) {
|
|
48
|
-
const entries = new Map();
|
|
49
|
-
const segments = [];
|
|
50
|
-
let currentSegment = '';
|
|
51
|
-
let inString = false;
|
|
52
|
-
let escapeNext = false;
|
|
53
|
-
let nestLevel = 0;
|
|
54
|
-
for (const char of body) {
|
|
55
|
-
if (escapeNext) {
|
|
56
|
-
escapeNext = false;
|
|
57
|
-
currentSegment += char;
|
|
58
|
-
continue;
|
|
59
|
-
}
|
|
60
|
-
if (char === '\\') {
|
|
61
|
-
escapeNext = true;
|
|
62
|
-
currentSegment += char;
|
|
63
|
-
continue;
|
|
64
|
-
}
|
|
65
|
-
if (char === '"' || char === "'") {
|
|
66
|
-
inString = !inString;
|
|
67
|
-
}
|
|
68
|
-
if (!inString) {
|
|
69
|
-
if (char === '{' || char === '[' || char === '(')
|
|
70
|
-
nestLevel++;
|
|
71
|
-
else if (char === '}' || char === ']' || char === ')')
|
|
72
|
-
nestLevel--;
|
|
73
|
-
}
|
|
74
|
-
if (char === ',' && !inString && nestLevel === 0) {
|
|
75
|
-
segments.push(currentSegment);
|
|
76
|
-
currentSegment = '';
|
|
77
|
-
}
|
|
78
|
-
else {
|
|
79
|
-
currentSegment += char;
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
if (currentSegment)
|
|
83
|
-
segments.push(currentSegment);
|
|
84
|
-
for (const part of segments) {
|
|
85
|
-
const entryMatch = /(?:"((?:\\.|[^"\\])*)"|([A-Za-z_$][\w$]*)):([A-Za-z_$][\w$]*)$/.exec(part.trim());
|
|
86
|
-
const key = entryMatch?.[1] ?? entryMatch?.[2];
|
|
87
|
-
const value = entryMatch?.[3];
|
|
88
|
-
if (key && value)
|
|
89
|
-
entries.set(key, value);
|
|
90
|
-
}
|
|
91
|
-
return entries;
|
|
92
|
-
}
|
|
93
|
-
function parseFlightObjectRefs(text) {
|
|
94
|
-
const templateMap = new Map();
|
|
95
|
-
const aliasMap = new Map();
|
|
96
|
-
const objectMaps = new Map();
|
|
97
|
-
for (const match of text.matchAll(TEMPLATE_ASSIGNMENT_RE)) {
|
|
98
|
-
const name = match[1];
|
|
99
|
-
const code = match[2];
|
|
100
|
-
if (name && code)
|
|
101
|
-
templateMap.set(name, decodeHtmlEntities(code));
|
|
102
|
-
}
|
|
103
|
-
const regex = /([A-Za-z_$][\w$]*)=\{/g;
|
|
104
|
-
let match;
|
|
105
|
-
while ((match = regex.exec(text)) !== null) {
|
|
106
|
-
const objectName = match[1];
|
|
107
|
-
if (!objectName)
|
|
108
|
-
continue;
|
|
109
|
-
const start = regex.lastIndex;
|
|
110
|
-
let inString = false;
|
|
111
|
-
let escapeLevel = false;
|
|
112
|
-
let depth = 1;
|
|
113
|
-
let end = -1;
|
|
114
|
-
for (let i = start; i < text.length; i++) {
|
|
115
|
-
const char = text[i];
|
|
116
|
-
if (escapeLevel) {
|
|
117
|
-
escapeLevel = false;
|
|
118
|
-
continue;
|
|
119
|
-
}
|
|
120
|
-
if (char === '\\') {
|
|
121
|
-
escapeLevel = true;
|
|
122
|
-
continue;
|
|
123
|
-
}
|
|
124
|
-
if (char === '"' || char === "'") {
|
|
125
|
-
inString = !inString;
|
|
126
|
-
continue;
|
|
127
|
-
}
|
|
128
|
-
if (!inString) {
|
|
129
|
-
if (char === '{')
|
|
130
|
-
depth++;
|
|
131
|
-
else if (char === '}')
|
|
132
|
-
depth--;
|
|
133
|
-
if (depth === 0) {
|
|
134
|
-
end = i;
|
|
135
|
-
break;
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
if (end === -1)
|
|
140
|
-
continue;
|
|
141
|
-
const body = text.substring(start, end).trim();
|
|
142
|
-
if (!body)
|
|
143
|
-
continue;
|
|
144
|
-
const spreadMatch = /^\.\.\.([A-Za-z_$][\w$]*)$/.exec(body);
|
|
145
|
-
if (spreadMatch?.[1]) {
|
|
146
|
-
aliasMap.set(objectName, spreadMatch[1]);
|
|
147
|
-
continue;
|
|
148
|
-
}
|
|
149
|
-
const entries = parseObjectEntries(body);
|
|
150
|
-
if (entries.size > 0)
|
|
151
|
-
objectMaps.set(objectName, entries);
|
|
152
|
-
}
|
|
153
|
-
return { templateMap, aliasMap, objectMaps };
|
|
154
|
-
}
|
|
155
|
-
function resolveFlightCodeRef(name, refs, seen = new Set()) {
|
|
156
|
-
if (!name || seen.has(name))
|
|
157
|
-
return undefined;
|
|
158
|
-
seen.add(name);
|
|
159
|
-
const direct = refs.templateMap.get(name);
|
|
160
|
-
if (direct)
|
|
161
|
-
return direct;
|
|
162
|
-
const alias = refs.aliasMap.get(name);
|
|
163
|
-
if (alias)
|
|
164
|
-
return resolveFlightCodeRef(alias, refs, seen);
|
|
165
|
-
const objectMap = refs.objectMaps.get(name);
|
|
166
|
-
if (!objectMap)
|
|
167
|
-
return undefined;
|
|
168
|
-
for (const ref of objectMap.values()) {
|
|
169
|
-
const resolved = resolveFlightCodeRef(ref, refs, seen);
|
|
170
|
-
if (resolved)
|
|
171
|
-
return resolved;
|
|
172
|
-
}
|
|
173
|
-
return undefined;
|
|
174
|
-
}
|
|
175
|
-
function escapeMarkdownTableCell(value) {
|
|
176
|
-
const normalized = decodeHtmlEntities(value).replace(/\s+/g, ' ').trim();
|
|
177
|
-
return (normalized || '-').replace(/\|/g, '\\|');
|
|
178
|
-
}
|
|
179
|
-
function buildMarkdownTable(rows) {
|
|
180
|
-
if (rows.length === 0)
|
|
181
|
-
return '';
|
|
182
|
-
const lines = [
|
|
183
|
-
'| Prop | Type | Description | Default |',
|
|
184
|
-
'| ---- | ---- | ----------- | ------- |',
|
|
185
|
-
];
|
|
186
|
-
for (const row of rows) {
|
|
187
|
-
lines.push(`| ${escapeMarkdownTableCell(row.attribute)} | ${escapeMarkdownTableCell(row.type)} | ${escapeMarkdownTableCell(row.description)} | ${escapeMarkdownTableCell(row.defaultValue)} |`);
|
|
188
|
-
}
|
|
189
|
-
return lines.join('\n');
|
|
190
|
-
}
|
|
191
|
-
function buildCodeBlock(code) {
|
|
192
|
-
const trimmed = code.trim();
|
|
193
|
-
if (!trimmed)
|
|
194
|
-
return '';
|
|
195
|
-
const language = detectLanguageFromCode(trimmed) ?? 'tsx';
|
|
196
|
-
return `\`\`\`${language}\n${trimmed}\n\`\`\``;
|
|
197
|
-
}
|
|
198
|
-
function buildMermaidBlock(chart) {
|
|
199
|
-
const normalized = decodeFlightStringValue(chart).trim();
|
|
200
|
-
if (!normalized)
|
|
201
|
-
return '';
|
|
202
|
-
return `\`\`\`mermaid\n${normalized}\n\`\`\``;
|
|
203
|
-
}
|
|
204
|
-
function normalizeSupplementHeadingText(value) {
|
|
205
|
-
return value
|
|
206
|
-
.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
|
|
207
|
-
.replace(/\s+/g, ' ')
|
|
208
|
-
.trim()
|
|
209
|
-
.toLowerCase();
|
|
210
|
-
}
|
|
211
|
-
function getMarkdownHeadingInfo(line) {
|
|
212
|
-
const match = /^(#{1,6})\s+(.+?)(?:\s+#*)?\s*$/.exec(line.trim());
|
|
213
|
-
if (!match)
|
|
214
|
-
return null;
|
|
215
|
-
return {
|
|
216
|
-
level: match[1]?.length ?? 0,
|
|
217
|
-
title: normalizeSupplementHeadingText(match[2] ?? ''),
|
|
218
|
-
};
|
|
219
|
-
}
|
|
220
|
-
function findMarkdownSection(lines, title) {
|
|
221
|
-
const target = normalizeSupplementHeadingText(title);
|
|
222
|
-
const startIndex = lines.findIndex((line) => {
|
|
223
|
-
const heading = getMarkdownHeadingInfo(line);
|
|
224
|
-
return heading?.title === target;
|
|
225
|
-
});
|
|
226
|
-
if (startIndex === -1)
|
|
227
|
-
return null;
|
|
228
|
-
const startHeading = getMarkdownHeadingInfo(lines[startIndex] ?? '');
|
|
229
|
-
if (!startHeading)
|
|
230
|
-
return null;
|
|
231
|
-
let end = lines.length;
|
|
232
|
-
for (let j = startIndex + 1; j < lines.length; j += 1) {
|
|
233
|
-
const nextHeading = getMarkdownHeadingInfo(lines[j] ?? '');
|
|
234
|
-
if (nextHeading && nextHeading.level <= startHeading.level) {
|
|
235
|
-
end = j;
|
|
236
|
-
break;
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
return { start: startIndex, end };
|
|
240
|
-
}
|
|
241
|
-
function getSectionBody(lines, section) {
|
|
242
|
-
return lines
|
|
243
|
-
.slice(section.start + 1, section.end)
|
|
244
|
-
.join('\n')
|
|
245
|
-
.trim();
|
|
246
|
-
}
|
|
247
|
-
function updateMarkdownSection(lines, title, strategy) {
|
|
248
|
-
const section = findMarkdownSection(lines, title);
|
|
249
|
-
if (!section)
|
|
250
|
-
return false;
|
|
251
|
-
const bodyText = getSectionBody(lines, section);
|
|
252
|
-
const nextBody = strategy(bodyText);
|
|
253
|
-
if (nextBody === null)
|
|
254
|
-
return false;
|
|
255
|
-
const replacement = nextBody.trim().length > 0
|
|
256
|
-
? ['', ...nextBody.trim().split('\n'), '']
|
|
257
|
-
: [''];
|
|
258
|
-
lines.splice(section.start + 1, section.end - section.start - 1, ...replacement);
|
|
259
|
-
return true;
|
|
260
|
-
}
|
|
261
|
-
function upsertMarkdownSection(lines, title, content, options) {
|
|
262
|
-
return updateMarkdownSection(lines, title, (bodyText) => {
|
|
263
|
-
if (options?.replacement)
|
|
264
|
-
return content;
|
|
265
|
-
if (options?.exclusionPattern) {
|
|
266
|
-
if (options.exclusionPattern instanceof RegExp) {
|
|
267
|
-
if (options.exclusionPattern.test(bodyText))
|
|
268
|
-
return null;
|
|
269
|
-
}
|
|
270
|
-
else if (bodyText.includes(options.exclusionPattern)) {
|
|
271
|
-
return null;
|
|
272
|
-
}
|
|
273
|
-
}
|
|
274
|
-
return bodyText ? `${bodyText}\n\n${content.trim()}` : content.trim();
|
|
275
|
-
});
|
|
276
|
-
}
|
|
277
|
-
function parseFlightApiRow(rowMatch) {
|
|
278
|
-
const attribute = rowMatch[1];
|
|
279
|
-
const type = rowMatch[2];
|
|
280
|
-
const description = rowMatch[3];
|
|
281
|
-
const defaultValue = rowMatch[4];
|
|
282
|
-
if (!attribute ||
|
|
283
|
-
!type ||
|
|
284
|
-
description === undefined ||
|
|
285
|
-
defaultValue === undefined) {
|
|
286
|
-
return null;
|
|
287
|
-
}
|
|
288
|
-
return { attribute, type, description, defaultValue };
|
|
289
|
-
}
|
|
290
|
-
function extractFlightApiTables(text) {
|
|
291
|
-
const apiTables = new Map();
|
|
292
|
-
for (const match of text.matchAll(FLIGHT_API_RE)) {
|
|
293
|
-
const title = match[1];
|
|
294
|
-
const rawRows = match[2] ?? '';
|
|
295
|
-
if (!title)
|
|
296
|
-
continue;
|
|
297
|
-
const rows = [];
|
|
298
|
-
for (const rowMatch of rawRows.matchAll(FLIGHT_API_ROW_RE)) {
|
|
299
|
-
const row = parseFlightApiRow(rowMatch);
|
|
300
|
-
if (row)
|
|
301
|
-
rows.push(row);
|
|
302
|
-
}
|
|
303
|
-
const table = buildMarkdownTable(rows);
|
|
304
|
-
if (table)
|
|
305
|
-
apiTables.set(title, table);
|
|
306
|
-
}
|
|
307
|
-
return apiTables;
|
|
308
|
-
}
|
|
309
|
-
function extractFlightMermaidDiagrams(text) {
|
|
310
|
-
const mermaidDiagrams = new Map();
|
|
311
|
-
for (const match of text.matchAll(FLIGHT_MERMAID_SECTION_RE)) {
|
|
312
|
-
const title = match[1] ? decodeFlightStringValue(match[1]).trim() : '';
|
|
313
|
-
const chart = match[2] ? buildMermaidBlock(match[2]) : '';
|
|
314
|
-
if (title && chart)
|
|
315
|
-
mermaidDiagrams.set(title, chart);
|
|
316
|
-
}
|
|
317
|
-
return mermaidDiagrams;
|
|
318
|
-
}
|
|
319
|
-
function extractFlightDemoBlocks(text, refs) {
|
|
320
|
-
const demoCodeBlocks = new Map();
|
|
321
|
-
for (const match of text.matchAll(FLIGHT_DEMO_RE)) {
|
|
322
|
-
const title = match[1];
|
|
323
|
-
const objectName = match[2];
|
|
324
|
-
const key = match[3];
|
|
325
|
-
const ref = objectName
|
|
326
|
-
? refs.objectMaps.get(objectName)?.get(key ?? '')
|
|
327
|
-
: undefined;
|
|
328
|
-
const code = resolveFlightCodeRef(ref, refs);
|
|
329
|
-
const codeBlock = code ? buildCodeBlock(code) : '';
|
|
330
|
-
if (title && codeBlock)
|
|
331
|
-
demoCodeBlocks.set(title, codeBlock);
|
|
332
|
-
}
|
|
333
|
-
return demoCodeBlocks;
|
|
334
|
-
}
|
|
335
|
-
function extractNextFlightSupplement(originalHtml) {
|
|
336
|
-
const payloads = decodeNextFlightPayloads(originalHtml);
|
|
337
|
-
if (payloads.length === 0)
|
|
338
|
-
return null;
|
|
339
|
-
const text = payloads.join('\n');
|
|
340
|
-
const refs = parseFlightObjectRefs(text);
|
|
341
|
-
const installMatch = FLIGHT_INSTALL_RE.exec(text);
|
|
342
|
-
const importMatch = FLIGHT_IMPORT_RE.exec(text);
|
|
343
|
-
return {
|
|
344
|
-
...(installMatch ? { installationCommands: installMatch.slice(1) } : {}),
|
|
345
|
-
...(importMatch ? { importCommands: importMatch.slice(1) } : {}),
|
|
346
|
-
apiTables: extractFlightApiTables(text),
|
|
347
|
-
demoCodeBlocks: extractFlightDemoBlocks(text, refs),
|
|
348
|
-
mermaidDiagrams: extractFlightMermaidDiagrams(text),
|
|
349
|
-
};
|
|
350
|
-
}
|
|
351
|
-
export function supplementMarkdownFromNextFlight(markdown, originalHtml) {
|
|
352
|
-
const payloadData = extractNextFlightSupplement(originalHtml);
|
|
353
|
-
if (!payloadData)
|
|
354
|
-
return markdown;
|
|
355
|
-
const lines = markdown.split('\n');
|
|
356
|
-
if (payloadData.installationCommands?.length) {
|
|
357
|
-
upsertMarkdownSection(lines, 'Installation', buildCodeBlock(payloadData.installationCommands.join('\n')), { exclusionPattern: /(npm|pnpm|yarn|bun|npx)\s+(install|add)/ });
|
|
358
|
-
}
|
|
359
|
-
if (payloadData.importCommands?.length) {
|
|
360
|
-
upsertMarkdownSection(lines, 'Import', buildCodeBlock(payloadData.importCommands.join('\n\n')), { exclusionPattern: /import\s+\{/ });
|
|
361
|
-
}
|
|
362
|
-
for (const [title, table] of payloadData.apiTables) {
|
|
363
|
-
upsertMarkdownSection(lines, title, table, { replacement: true });
|
|
364
|
-
}
|
|
365
|
-
for (const [title, mermaidBlock] of payloadData.mermaidDiagrams) {
|
|
366
|
-
upsertMarkdownSection(lines, title, mermaidBlock, {
|
|
367
|
-
exclusionPattern: '```mermaid',
|
|
368
|
-
});
|
|
369
|
-
}
|
|
370
|
-
for (const [title, codeBlock] of payloadData.demoCodeBlocks) {
|
|
371
|
-
upsertMarkdownSection(lines, title, codeBlock, { exclusionPattern: '```' });
|
|
372
|
-
}
|
|
373
|
-
return lines.join('\n');
|
|
374
|
-
}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import type { MarkdownTransformResult, TransformOptions, TransformWorkerOutgoingMessage } from './types.js';
|
|
2
|
-
interface WorkerMessageHandlerOptions {
|
|
3
|
-
sendMessage: (message: TransformWorkerOutgoingMessage) => void;
|
|
4
|
-
runTransform: (html: string, url: string, options: TransformOptions) => MarkdownTransformResult;
|
|
5
|
-
}
|
|
6
|
-
export declare function createTransformMessageHandler(options: WorkerMessageHandlerOptions): (raw: unknown) => void;
|
|
7
|
-
export {};
|
|
8
|
-
//# sourceMappingURL=shared.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/transform/shared.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,uBAAuB,EACvB,gBAAgB,EAChB,8BAA8B,EAE/B,MAAM,YAAY,CAAC;AAEpB,UAAU,2BAA2B;IACnC,WAAW,EAAE,CAAC,OAAO,EAAE,8BAA8B,KAAK,IAAI,CAAC;IAC/D,YAAY,EAAE,CACZ,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,KACtB,uBAAuB,CAAC;CAC9B;AAkLD,wBAAgB,6BAA6B,CAC3C,OAAO,EAAE,2BAA2B,GACnC,CAAC,GAAG,EAAE,OAAO,KAAK,IAAI,CA2BxB"}
|
package/dist/transform/shared.js
DELETED
|
@@ -1,137 +0,0 @@
|
|
|
1
|
-
import { FetchError, getErrorMessage } from '../lib/utils.js';
|
|
2
|
-
function isTransformMessage(message) {
|
|
3
|
-
if (!message || typeof message !== 'object')
|
|
4
|
-
return false;
|
|
5
|
-
const value = message;
|
|
6
|
-
const { id, url, html, htmlBuffer, encoding, includeMetadataFooter, inputTruncated, } = value;
|
|
7
|
-
return (typeof id === 'string' &&
|
|
8
|
-
typeof url === 'string' &&
|
|
9
|
-
typeof includeMetadataFooter === 'boolean' &&
|
|
10
|
-
(html === undefined || typeof html === 'string') &&
|
|
11
|
-
(htmlBuffer === undefined || htmlBuffer instanceof Uint8Array) &&
|
|
12
|
-
(encoding === undefined || typeof encoding === 'string') &&
|
|
13
|
-
(inputTruncated === undefined || typeof inputTruncated === 'boolean'));
|
|
14
|
-
}
|
|
15
|
-
function decodeHtml(html, htmlBuffer, encoding, decoder) {
|
|
16
|
-
if (!htmlBuffer)
|
|
17
|
-
return html ?? '';
|
|
18
|
-
if (!encoding || encoding === 'utf-8') {
|
|
19
|
-
return decoder.decode(htmlBuffer);
|
|
20
|
-
}
|
|
21
|
-
try {
|
|
22
|
-
const decoded = new TextDecoder(encoding).decode(htmlBuffer);
|
|
23
|
-
return decoded;
|
|
24
|
-
}
|
|
25
|
-
catch {
|
|
26
|
-
return decoder.decode(htmlBuffer);
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
function createErrorMessage(id, url, error) {
|
|
30
|
-
if (error instanceof FetchError) {
|
|
31
|
-
return {
|
|
32
|
-
type: 'error',
|
|
33
|
-
id,
|
|
34
|
-
error: {
|
|
35
|
-
name: error.name,
|
|
36
|
-
message: error.message,
|
|
37
|
-
url: error.url,
|
|
38
|
-
statusCode: error.statusCode,
|
|
39
|
-
details: { ...error.details },
|
|
40
|
-
},
|
|
41
|
-
};
|
|
42
|
-
}
|
|
43
|
-
return {
|
|
44
|
-
type: 'error',
|
|
45
|
-
id,
|
|
46
|
-
error: {
|
|
47
|
-
name: error instanceof Error ? error.name : 'Error',
|
|
48
|
-
message: getErrorMessage(error),
|
|
49
|
-
url,
|
|
50
|
-
},
|
|
51
|
-
};
|
|
52
|
-
}
|
|
53
|
-
function createResultMessage(id, result) {
|
|
54
|
-
return {
|
|
55
|
-
type: 'result',
|
|
56
|
-
id,
|
|
57
|
-
result: {
|
|
58
|
-
markdown: result.markdown,
|
|
59
|
-
...(result.metadata ? { metadata: result.metadata } : {}),
|
|
60
|
-
...(result.title !== undefined ? { title: result.title } : {}),
|
|
61
|
-
truncated: result.truncated,
|
|
62
|
-
},
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
function createValidationErrorMessage(id, url, message) {
|
|
66
|
-
return {
|
|
67
|
-
type: 'error',
|
|
68
|
-
id,
|
|
69
|
-
error: {
|
|
70
|
-
name: 'ValidationError',
|
|
71
|
-
message,
|
|
72
|
-
url,
|
|
73
|
-
},
|
|
74
|
-
};
|
|
75
|
-
}
|
|
76
|
-
function handleCancelMessage(params) {
|
|
77
|
-
const controller = params.controllersById.get(params.id);
|
|
78
|
-
if (controller)
|
|
79
|
-
controller.abort(new Error('Canceled'));
|
|
80
|
-
params.sendMessage({ type: 'cancelled', id: params.id });
|
|
81
|
-
}
|
|
82
|
-
function executeTransformMessage(params) {
|
|
83
|
-
const { message, controllersById, decoder, runTransform, sendMessage } = params;
|
|
84
|
-
const { id, url, html, htmlBuffer, encoding, includeMetadataFooter, inputTruncated, } = message;
|
|
85
|
-
if (!id.trim()) {
|
|
86
|
-
sendMessage(createValidationErrorMessage(id, url || '', 'Missing transform message id'));
|
|
87
|
-
return;
|
|
88
|
-
}
|
|
89
|
-
if (!url.trim()) {
|
|
90
|
-
sendMessage(createValidationErrorMessage(id, url, 'Missing transform URL'));
|
|
91
|
-
return;
|
|
92
|
-
}
|
|
93
|
-
const controller = new AbortController();
|
|
94
|
-
controllersById.set(id, controller);
|
|
95
|
-
try {
|
|
96
|
-
const content = decodeHtml(html, htmlBuffer, encoding, decoder);
|
|
97
|
-
const result = runTransform(content, url, {
|
|
98
|
-
includeMetadataFooter,
|
|
99
|
-
signal: controller.signal,
|
|
100
|
-
...(inputTruncated ? { inputTruncated: true } : {}),
|
|
101
|
-
});
|
|
102
|
-
sendMessage(createResultMessage(id, result));
|
|
103
|
-
}
|
|
104
|
-
catch (error) {
|
|
105
|
-
sendMessage(createErrorMessage(id, url, error));
|
|
106
|
-
}
|
|
107
|
-
finally {
|
|
108
|
-
controllersById.delete(id);
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
export function createTransformMessageHandler(options) {
|
|
112
|
-
const { sendMessage, runTransform } = options;
|
|
113
|
-
const controllersById = new Map();
|
|
114
|
-
const decoder = new TextDecoder('utf-8');
|
|
115
|
-
return (raw) => {
|
|
116
|
-
if (!raw || typeof raw !== 'object')
|
|
117
|
-
return;
|
|
118
|
-
const message = raw;
|
|
119
|
-
const messageType = message['type'];
|
|
120
|
-
const messageId = message['id'];
|
|
121
|
-
if (messageType === 'cancel') {
|
|
122
|
-
if (typeof messageId !== 'string')
|
|
123
|
-
return;
|
|
124
|
-
handleCancelMessage({ id: messageId, controllersById, sendMessage });
|
|
125
|
-
return;
|
|
126
|
-
}
|
|
127
|
-
if (messageType !== 'transform' || !isTransformMessage(message))
|
|
128
|
-
return;
|
|
129
|
-
executeTransformMessage({
|
|
130
|
-
message,
|
|
131
|
-
controllersById,
|
|
132
|
-
decoder,
|
|
133
|
-
runTransform,
|
|
134
|
-
sendMessage,
|
|
135
|
-
});
|
|
136
|
-
};
|
|
137
|
-
}
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
import { cleanupMarkdownArtifacts, finalizeMarkdownSections, processFencedContent } from './markdown-cleanup.js';
|
|
2
|
-
import type { ExtractedArticle, ExtractedMetadata, ExtractionResult, MarkdownTransformResult, MetadataBlock, TransformOptions, TransformStageContext } from './types.js';
|
|
3
|
-
interface StageBudget {
|
|
4
|
-
totalBudgetMs: number;
|
|
5
|
-
elapsedMs: number;
|
|
6
|
-
}
|
|
7
|
-
export declare function startTransformStage(url: string, stage: string, budget?: StageBudget): TransformStageContext | null;
|
|
8
|
-
export declare function endTransformStage(context: TransformStageContext | null, options?: {
|
|
9
|
-
truncated?: boolean;
|
|
10
|
-
}): number;
|
|
11
|
-
export declare function extractContent(html: string, url: string, options?: {
|
|
12
|
-
extractArticle?: boolean;
|
|
13
|
-
signal?: AbortSignal;
|
|
14
|
-
}): ExtractionResult;
|
|
15
|
-
export declare function htmlToMarkdown(html: string, metadata?: MetadataBlock, options?: {
|
|
16
|
-
url?: string;
|
|
17
|
-
signal?: AbortSignal | undefined;
|
|
18
|
-
document?: Document | undefined;
|
|
19
|
-
skipNoiseRemoval?: boolean | undefined;
|
|
20
|
-
}): string;
|
|
21
|
-
export declare function isExtractionSufficient(article: ExtractedArticle | null, originalHtmlOrDocument: string | Document): boolean;
|
|
22
|
-
export declare function determineContentExtractionSource(article: ExtractedArticle | null): article is ExtractedArticle;
|
|
23
|
-
export declare function createContentMetadataBlock(url: string, article: ExtractedArticle | null, extractedMeta: ExtractedMetadata, shouldExtractFromArticle: boolean, includeMetadataFooter: boolean): MetadataBlock | undefined;
|
|
24
|
-
export declare function transformHtmlToMarkdownInProcess(html: string, url: string, options: TransformOptions): MarkdownTransformResult;
|
|
25
|
-
interface TransformPoolStats {
|
|
26
|
-
queueDepth: number;
|
|
27
|
-
activeWorkers: number;
|
|
28
|
-
capacity: number;
|
|
29
|
-
}
|
|
30
|
-
export declare function getTransformPoolStats(): TransformPoolStats | null;
|
|
31
|
-
export declare function shutdownTransformWorkerPool(): Promise<void>;
|
|
32
|
-
type TransformExecutionOptions = TransformOptions & {
|
|
33
|
-
encoding?: string;
|
|
34
|
-
};
|
|
35
|
-
export declare function transformHtmlToMarkdown(html: string, url: string, options: TransformOptions): Promise<MarkdownTransformResult>;
|
|
36
|
-
export declare function transformBufferToMarkdown(htmlBuffer: Uint8Array, url: string, options: TransformExecutionOptions): Promise<MarkdownTransformResult>;
|
|
37
|
-
export { cleanupMarkdownArtifacts, finalizeMarkdownSections, processFencedContent, };
|
|
38
|
-
//# sourceMappingURL=transform.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"AA6CA,OAAO,EACL,wBAAwB,EACxB,wBAAwB,EACxB,oBAAoB,EACrB,MAAM,uBAAuB,CAAC;AAkB/B,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAoCpB,UAAU,WAAW;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AAqMD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,WAAW,GACnB,qBAAqB,GAAG,IAAI,CAE9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,MAAM,CAER;AA8bD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AAuKD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IACR,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IACjC,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAChC,gBAAgB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACxC,GACA,MAAM,CAgCR;AA+DD,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,sBAAsB,EAAE,MAAM,GAAG,QAAQ,GACxC,OAAO,CAQT;AAKD,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,qBAAqB,EAAE,OAAO,GAC7B,aAAa,GAAG,SAAS,CAuB3B;AA6bD,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CAMzB;AAcD,UAAU,kBAAkB;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,IAAI,kBAAkB,GAAG,IAAI,CAEjE;AAED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjE;AAED,KAAK,yBAAyB,GAAG,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAkH1E,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,wBAAsB,yBAAyB,CAC7C,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,OAAO,EACL,wBAAwB,EACxB,wBAAwB,EACxB,oBAAoB,GACrB,CAAC"}
|