@intuned/runtime-dev 1.6.0-dev-52-5 → 1.6.0-dev-52-6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,2214 +1,3 @@
|
|
|
1
|
-
(
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
var MatchSource;
|
|
5
|
-
(function (MatchSource) {
|
|
6
|
-
MatchSource["ATTRIBUTE"] = "attribute";
|
|
7
|
-
MatchSource["TEXT_CONTENT"] = "text_content";
|
|
8
|
-
MatchSource["DIRECT_TEXT_NODE"] = "direct_text_node";
|
|
9
|
-
})(MatchSource || (MatchSource = {}));
|
|
10
|
-
var MatchMode;
|
|
11
|
-
(function (MatchMode) {
|
|
12
|
-
MatchMode["FULL"] = "full";
|
|
13
|
-
MatchMode["PARTIAL"] = "partial";
|
|
14
|
-
MatchMode["FUZZY"] = "fuzzy";
|
|
15
|
-
})(MatchMode || (MatchMode = {}));
|
|
16
|
-
|
|
17
|
-
function* searchExact(needle, haystack, startIndex = 0, endIndex = null) {
|
|
18
|
-
const needleLen = needle.length;
|
|
19
|
-
if (needleLen === 0)
|
|
20
|
-
return;
|
|
21
|
-
if (endIndex === null) {
|
|
22
|
-
endIndex = haystack.length;
|
|
23
|
-
}
|
|
24
|
-
let index;
|
|
25
|
-
while ((index = haystack.indexOf(needle, startIndex)) > -1) {
|
|
26
|
-
if (index + needle.length > endIndex)
|
|
27
|
-
break;
|
|
28
|
-
yield index;
|
|
29
|
-
startIndex = index + 1;
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
function reverse(string) {
|
|
33
|
-
return string.split("").reverse().join("");
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
function makeChar2needleIdx(needle, maxDist) {
|
|
37
|
-
const res = {};
|
|
38
|
-
for (let i = Math.min(needle.length - 1, maxDist); i >= 0; i--) {
|
|
39
|
-
res[needle[i]] = i;
|
|
40
|
-
}
|
|
41
|
-
return res;
|
|
42
|
-
}
|
|
43
|
-
function* fuzzySearch(needle, haystack, maxDist) {
|
|
44
|
-
if (needle.length > haystack.length + maxDist)
|
|
45
|
-
return;
|
|
46
|
-
const ngramLen = Math.floor(needle.length / (maxDist + 1));
|
|
47
|
-
if (maxDist === 0) {
|
|
48
|
-
for (const index of searchExact(needle, haystack)) {
|
|
49
|
-
yield {
|
|
50
|
-
start: index,
|
|
51
|
-
end: index + needle.length,
|
|
52
|
-
dist: 0,
|
|
53
|
-
};
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
else if (ngramLen >= 10) {
|
|
57
|
-
yield* fuzzySearchNgrams(needle, haystack, maxDist);
|
|
58
|
-
}
|
|
59
|
-
else {
|
|
60
|
-
yield* fuzzySearchCandidates(needle, haystack, maxDist);
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
function _expand(needle, haystack, maxDist) {
|
|
64
|
-
maxDist = +maxDist;
|
|
65
|
-
let firstDiff;
|
|
66
|
-
for (firstDiff = 0; firstDiff < Math.min(needle.length, haystack.length); firstDiff++) {
|
|
67
|
-
if (needle.charCodeAt(firstDiff) !== haystack.charCodeAt(firstDiff))
|
|
68
|
-
break;
|
|
69
|
-
}
|
|
70
|
-
if (firstDiff) {
|
|
71
|
-
needle = needle.slice(firstDiff);
|
|
72
|
-
haystack = haystack.slice(firstDiff);
|
|
73
|
-
}
|
|
74
|
-
if (!needle) {
|
|
75
|
-
return [0, firstDiff];
|
|
76
|
-
}
|
|
77
|
-
else if (!haystack) {
|
|
78
|
-
if (needle.length <= maxDist) {
|
|
79
|
-
return [needle.length, firstDiff];
|
|
80
|
-
}
|
|
81
|
-
else {
|
|
82
|
-
return [null, null];
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
if (maxDist === 0)
|
|
86
|
-
return [null, null];
|
|
87
|
-
let scores = new Array(needle.length + 1);
|
|
88
|
-
for (let i = 0; i <= maxDist; i++) {
|
|
89
|
-
scores[i] = i;
|
|
90
|
-
}
|
|
91
|
-
let newScores = new Array(needle.length + 1);
|
|
92
|
-
let minScore = null;
|
|
93
|
-
let minScoreIdx = null;
|
|
94
|
-
let maxGoodScore = maxDist;
|
|
95
|
-
let firstGoodScoreIdx = 0;
|
|
96
|
-
let lastGoodScoreIdx = needle.length - 1;
|
|
97
|
-
for (let haystackIdx = 0; haystackIdx < haystack.length; haystackIdx++) {
|
|
98
|
-
const char = haystack.charCodeAt(haystackIdx);
|
|
99
|
-
const needleIdxStart = Math.max(0, firstGoodScoreIdx - 1);
|
|
100
|
-
const needleIdxLimit = Math.min(haystackIdx + maxDist, needle.length - 1, lastGoodScoreIdx);
|
|
101
|
-
newScores[0] = scores[0] + 1;
|
|
102
|
-
firstGoodScoreIdx = newScores[0] <= maxGoodScore ? 0 : null;
|
|
103
|
-
lastGoodScoreIdx = newScores[0] <= maxGoodScore ? 0 : -1;
|
|
104
|
-
let needleIdx;
|
|
105
|
-
for (needleIdx = needleIdxStart; needleIdx < needleIdxLimit; needleIdx++) {
|
|
106
|
-
const score = (newScores[needleIdx + 1] = Math.min(scores[needleIdx] + +(char !== needle.charCodeAt(needleIdx)), scores[needleIdx + 1] + 1, newScores[needleIdx] + 1));
|
|
107
|
-
if (score <= maxGoodScore) {
|
|
108
|
-
if (firstGoodScoreIdx === null)
|
|
109
|
-
firstGoodScoreIdx = needleIdx + 1;
|
|
110
|
-
lastGoodScoreIdx = Math.max(lastGoodScoreIdx, needleIdx + 1 + (maxGoodScore - score));
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
const lastScore = (newScores[needleIdx + 1] = Math.min(scores[needleIdx] + +(char !== needle.charCodeAt(needleIdx)), newScores[needleIdx] + 1));
|
|
114
|
-
if (lastScore <= maxGoodScore) {
|
|
115
|
-
if (firstGoodScoreIdx === null)
|
|
116
|
-
firstGoodScoreIdx = needleIdx + 1;
|
|
117
|
-
lastGoodScoreIdx = needleIdx + 1;
|
|
118
|
-
}
|
|
119
|
-
if (needleIdx === needle.length - 1 &&
|
|
120
|
-
(minScore === null || lastScore <= minScore)) {
|
|
121
|
-
minScore = lastScore;
|
|
122
|
-
minScoreIdx = haystackIdx;
|
|
123
|
-
if (minScore < maxGoodScore)
|
|
124
|
-
maxGoodScore = minScore;
|
|
125
|
-
}
|
|
126
|
-
[scores, newScores] = [newScores, scores];
|
|
127
|
-
if (firstGoodScoreIdx === null)
|
|
128
|
-
break;
|
|
129
|
-
}
|
|
130
|
-
if (minScore !== null && minScore <= maxDist) {
|
|
131
|
-
return [minScore, minScoreIdx + 1 + firstDiff];
|
|
132
|
-
}
|
|
133
|
-
else {
|
|
134
|
-
return [null, null];
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
function* fuzzySearchNgrams(needle, haystack, maxDist) {
|
|
138
|
-
// use n-gram search
|
|
139
|
-
const ngramLen = Math.floor(needle.length / (maxDist + 1));
|
|
140
|
-
const needleLen = needle.length;
|
|
141
|
-
const haystackLen = haystack.length;
|
|
142
|
-
for (let ngramStartIdx = 0; ngramStartIdx <= needle.length - ngramLen; ngramStartIdx += ngramLen) {
|
|
143
|
-
const ngram = needle.slice(ngramStartIdx, ngramStartIdx + ngramLen);
|
|
144
|
-
const ngramEnd = ngramStartIdx + ngramLen;
|
|
145
|
-
const needleBeforeReversed = reverse(needle.slice(0, ngramStartIdx));
|
|
146
|
-
const needleAfter = needle.slice(ngramEnd);
|
|
147
|
-
const startIdx = Math.max(0, ngramStartIdx - maxDist);
|
|
148
|
-
const endIdx = Math.min(haystackLen, haystackLen - needleLen + ngramEnd + maxDist);
|
|
149
|
-
for (const haystackMatchIdx of searchExact(ngram, haystack, startIdx, endIdx)) {
|
|
150
|
-
// try to expand left
|
|
151
|
-
const [distRight, rightExpandSize] = _expand(needleAfter, haystack.slice(haystackMatchIdx + ngramLen, haystackMatchIdx - ngramStartIdx + needleLen + maxDist), maxDist);
|
|
152
|
-
if (distRight === null)
|
|
153
|
-
continue;
|
|
154
|
-
const [distLeft, leftExpandSize] = _expand(needleBeforeReversed, reverse(haystack.slice(Math.max(0, haystackMatchIdx - ngramStartIdx - (maxDist - distRight)), haystackMatchIdx)), maxDist - distRight);
|
|
155
|
-
if (distLeft === null)
|
|
156
|
-
continue;
|
|
157
|
-
yield {
|
|
158
|
-
start: haystackMatchIdx - leftExpandSize,
|
|
159
|
-
end: haystackMatchIdx + ngramLen + rightExpandSize,
|
|
160
|
-
dist: distLeft + distRight,
|
|
161
|
-
};
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
function* fuzzySearchCandidates(needle, haystack, maxDist) {
|
|
166
|
-
const needleLen = needle.length;
|
|
167
|
-
const haystackLen = haystack.length;
|
|
168
|
-
if (needleLen > haystackLen + maxDist)
|
|
169
|
-
return;
|
|
170
|
-
const char2needleIdx = makeChar2needleIdx(needle, maxDist);
|
|
171
|
-
let prevCandidates = new Map(); // candidates from the last iteration
|
|
172
|
-
let candidates = new Map(); // new candidates from the current iteration
|
|
173
|
-
// iterate over the chars in the haystack, updating the candidates for each
|
|
174
|
-
for (let i = 0; i < haystack.length; i++) {
|
|
175
|
-
const haystackChar = haystack[i];
|
|
176
|
-
prevCandidates = candidates;
|
|
177
|
-
candidates = new Map();
|
|
178
|
-
const needleIdx = char2needleIdx[haystackChar];
|
|
179
|
-
if (needleIdx !== undefined) {
|
|
180
|
-
if (needleIdx + 1 === needleLen) {
|
|
181
|
-
yield {
|
|
182
|
-
start: i,
|
|
183
|
-
end: i + 1,
|
|
184
|
-
dist: needleIdx,
|
|
185
|
-
};
|
|
186
|
-
}
|
|
187
|
-
else {
|
|
188
|
-
candidates.set(`${i},${needleIdx + 1},${needleIdx}`, {
|
|
189
|
-
startIdx: i,
|
|
190
|
-
needleIdx: needleIdx + 1,
|
|
191
|
-
dist: needleIdx,
|
|
192
|
-
});
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
for (const [, candidate] of prevCandidates) {
|
|
196
|
-
// if this sequence char is the candidate's next expected char
|
|
197
|
-
if (needle[candidate.needleIdx] === haystackChar) {
|
|
198
|
-
// if reached the end of the needle, return a match
|
|
199
|
-
if (candidate.needleIdx + 1 === needleLen) {
|
|
200
|
-
yield {
|
|
201
|
-
start: candidate.startIdx,
|
|
202
|
-
end: i + 1,
|
|
203
|
-
dist: candidate.dist,
|
|
204
|
-
};
|
|
205
|
-
}
|
|
206
|
-
else {
|
|
207
|
-
// otherwise, update the candidate's needleIdx and keep it
|
|
208
|
-
candidates.set(`${candidate.startIdx},${candidate.needleIdx + 1},${candidate.dist}`, {
|
|
209
|
-
startIdx: candidate.startIdx,
|
|
210
|
-
needleIdx: candidate.needleIdx + 1,
|
|
211
|
-
dist: candidate.dist,
|
|
212
|
-
});
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
else {
|
|
216
|
-
if (candidate.dist === maxDist)
|
|
217
|
-
continue;
|
|
218
|
-
candidates.set(`${candidate.startIdx},${candidate.needleIdx},${candidate.dist + 1}`, {
|
|
219
|
-
startIdx: candidate.startIdx,
|
|
220
|
-
needleIdx: candidate.needleIdx,
|
|
221
|
-
dist: candidate.dist + 1,
|
|
222
|
-
});
|
|
223
|
-
for (let nSkipped = 1; nSkipped <= maxDist - candidate.dist; nSkipped++) {
|
|
224
|
-
if (candidate.needleIdx + nSkipped === needleLen) {
|
|
225
|
-
yield {
|
|
226
|
-
start: candidate.startIdx,
|
|
227
|
-
end: i + 1,
|
|
228
|
-
dist: candidate.dist + nSkipped,
|
|
229
|
-
};
|
|
230
|
-
break;
|
|
231
|
-
}
|
|
232
|
-
else if (needle[candidate.needleIdx + nSkipped] === haystackChar) {
|
|
233
|
-
if (candidate.needleIdx + nSkipped + 1 === needleLen) {
|
|
234
|
-
yield {
|
|
235
|
-
start: candidate.startIdx,
|
|
236
|
-
end: i + 1,
|
|
237
|
-
dist: candidate.dist + nSkipped,
|
|
238
|
-
};
|
|
239
|
-
}
|
|
240
|
-
else {
|
|
241
|
-
candidates.set(`${candidate.startIdx},${candidate.needleIdx + 1 + nSkipped},${candidate.dist + nSkipped}`, {
|
|
242
|
-
startIdx: candidate.startIdx,
|
|
243
|
-
needleIdx: candidate.needleIdx + 1 + nSkipped,
|
|
244
|
-
dist: candidate.dist + nSkipped,
|
|
245
|
-
});
|
|
246
|
-
}
|
|
247
|
-
break;
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
if (i + 1 < haystackLen && candidate.needleIdx + 1 < needleLen) {
|
|
251
|
-
candidates.set(`${candidate.startIdx},${candidate.needleIdx + 1},${candidate.dist + 1}`, {
|
|
252
|
-
startIdx: candidate.startIdx,
|
|
253
|
-
needleIdx: candidate.needleIdx + 1,
|
|
254
|
-
dist: candidate.dist + 1,
|
|
255
|
-
});
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
for (const [, candidate] of candidates) {
|
|
261
|
-
candidate.dist += needle.length - candidate.needleIdx;
|
|
262
|
-
if (candidate.dist <= maxDist) {
|
|
263
|
-
yield {
|
|
264
|
-
start: candidate.startIdx,
|
|
265
|
-
end: haystack.length,
|
|
266
|
-
dist: candidate.dist,
|
|
267
|
-
};
|
|
268
|
-
}
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
function findClosestMatch(searchTerm, content, maxLDist) {
|
|
273
|
-
const results = [];
|
|
274
|
-
for (const result of fuzzySearch(searchTerm, content, maxLDist)) {
|
|
275
|
-
results.push(result);
|
|
276
|
-
}
|
|
277
|
-
results.sort((a, b) => {
|
|
278
|
-
if (a.dist === b.dist) {
|
|
279
|
-
return b.end - b.start - (a.end - a.start); // Sort by match length if distances are equal
|
|
280
|
-
}
|
|
281
|
-
return a.dist - b.dist; // Sort by distance
|
|
282
|
-
});
|
|
283
|
-
return results[0];
|
|
284
|
-
}
|
|
285
|
-
function normalizeSpacing(text) {
|
|
286
|
-
if (!text) {
|
|
287
|
-
return "";
|
|
288
|
-
}
|
|
289
|
-
// Replace newlines and tabs with spaces
|
|
290
|
-
let normalized = text.replace(/\n/g, " ").replace(/\t/g, " ");
|
|
291
|
-
// Replace multiple spaces with a single space
|
|
292
|
-
normalized = normalized.split(/\s+/).join(" ");
|
|
293
|
-
return normalized.trim();
|
|
294
|
-
}
|
|
295
|
-
function isMatchExact(data, value) {
|
|
296
|
-
if (!data || !value) {
|
|
297
|
-
return [false, null];
|
|
298
|
-
}
|
|
299
|
-
const normalizedData = normalizeSpacing(data);
|
|
300
|
-
const normalizedValue = normalizeSpacing(value);
|
|
301
|
-
return [normalizedData === normalizedValue, normalizedValue];
|
|
302
|
-
}
|
|
303
|
-
function calculateMaxLDist(value) {
|
|
304
|
-
const length = value.length;
|
|
305
|
-
const Pmax = 0.2;
|
|
306
|
-
const Pmin = 0.05;
|
|
307
|
-
const lengthAtPmax = 10;
|
|
308
|
-
let percentage;
|
|
309
|
-
if (length <= lengthAtPmax) {
|
|
310
|
-
percentage = Pmax;
|
|
311
|
-
}
|
|
312
|
-
else {
|
|
313
|
-
const k = -Math.log(Pmin / Pmax) / (600 - lengthAtPmax);
|
|
314
|
-
percentage = Pmax * Math.exp(-k * (length - lengthAtPmax));
|
|
315
|
-
}
|
|
316
|
-
percentage = Math.max(Pmin, percentage);
|
|
317
|
-
return Math.max(1, Math.floor(length * percentage));
|
|
318
|
-
}
|
|
319
|
-
function isFuzzMatch(searchTerm, content) {
|
|
320
|
-
if (!searchTerm || !content) {
|
|
321
|
-
return {
|
|
322
|
-
found: false,
|
|
323
|
-
matchedValue: null,
|
|
324
|
-
distance: null,
|
|
325
|
-
matchedSourceValue: null,
|
|
326
|
-
};
|
|
327
|
-
}
|
|
328
|
-
const maxLDist = calculateMaxLDist(searchTerm);
|
|
329
|
-
const normalizedSearchTerm = normalizeSpacing(searchTerm);
|
|
330
|
-
const normalizedContent = normalizeSpacing(content);
|
|
331
|
-
const match = findClosestMatch(normalizedSearchTerm.toLowerCase(), normalizedContent.toLowerCase(), maxLDist);
|
|
332
|
-
if (!match) {
|
|
333
|
-
return {
|
|
334
|
-
found: false,
|
|
335
|
-
matchedValue: null,
|
|
336
|
-
distance: null,
|
|
337
|
-
matchedSourceValue: null,
|
|
338
|
-
};
|
|
339
|
-
}
|
|
340
|
-
return {
|
|
341
|
-
found: true,
|
|
342
|
-
matchedValue: normalizedContent.slice(match.start, match.end),
|
|
343
|
-
matchedSourceValue: normalizedContent,
|
|
344
|
-
distance: match.dist,
|
|
345
|
-
};
|
|
346
|
-
}
|
|
347
|
-
function hasNonFuzzyOrCloseFuzzyMatch(matches) {
|
|
348
|
-
const hasNonFuzzyMatch = matches.some((match) => match.match_mode !== MatchMode.FUZZY);
|
|
349
|
-
const hasVeryCloseFuzzyMatch = matches.some((match) => match.match_mode === MatchMode.FUZZY &&
|
|
350
|
-
match.fuzzy_distance &&
|
|
351
|
-
match.fuzzy_distance < 5);
|
|
352
|
-
return hasNonFuzzyMatch || hasVeryCloseFuzzyMatch;
|
|
353
|
-
}
|
|
354
|
-
function getElementXPath(element) {
|
|
355
|
-
if (!element || !element.parentNode || element.nodeName === "#document") {
|
|
356
|
-
return null;
|
|
357
|
-
}
|
|
358
|
-
let siblingsCount = 1;
|
|
359
|
-
const parent = element.parentNode;
|
|
360
|
-
const nodeName = element.nodeName.toLowerCase();
|
|
361
|
-
const siblings = Array.from(parent.childNodes).filter((node) => node.nodeType === 1 // Node.ELEMENT_NODE
|
|
362
|
-
);
|
|
363
|
-
for (const sibling of siblings) {
|
|
364
|
-
if (sibling === element) {
|
|
365
|
-
break;
|
|
366
|
-
}
|
|
367
|
-
if (sibling.nodeName.toLowerCase() === nodeName) {
|
|
368
|
-
siblingsCount++;
|
|
369
|
-
}
|
|
370
|
-
}
|
|
371
|
-
const parentXPath = getElementXPath(parent);
|
|
372
|
-
if (element.nodeName === "#text") {
|
|
373
|
-
return parentXPath;
|
|
374
|
-
}
|
|
375
|
-
return parentXPath
|
|
376
|
-
? `${parentXPath}/${nodeName}[${siblingsCount}]`
|
|
377
|
-
: `${nodeName}[${siblingsCount}]`;
|
|
378
|
-
}
|
|
379
|
-
function traverseAndPrune(node, conditionFunc) {
|
|
380
|
-
const children = Array.from(node.children ?? []);
|
|
381
|
-
children.forEach((child) => {
|
|
382
|
-
if (child.children) {
|
|
383
|
-
if (!conditionFunc(child)) {
|
|
384
|
-
traverseAndPrune(child, conditionFunc);
|
|
385
|
-
}
|
|
386
|
-
}
|
|
387
|
-
});
|
|
388
|
-
}
|
|
389
|
-
function isPartOfString(input, dom) {
|
|
390
|
-
if (!input || !dom) {
|
|
391
|
-
return [false, null, null];
|
|
392
|
-
}
|
|
393
|
-
const normalizedInput = normalizeSpacing(input);
|
|
394
|
-
const normalizedDom = normalizeSpacing(dom);
|
|
395
|
-
const matchIndex = normalizedDom
|
|
396
|
-
.toLowerCase()
|
|
397
|
-
.indexOf(normalizedInput.toLowerCase());
|
|
398
|
-
const matchedText = matchIndex !== -1
|
|
399
|
-
? normalizedDom.substring(matchIndex, matchIndex + normalizedInput.length)
|
|
400
|
-
: null;
|
|
401
|
-
return [matchIndex !== -1, matchedText, normalizedDom];
|
|
402
|
-
}
|
|
403
|
-
|
|
404
|
-
function matchStringsWithDomContent(domNode, stringsList) {
|
|
405
|
-
const exactMatchedMap = matchExactStrings(domNode, stringsList);
|
|
406
|
-
const stringsWithNoExactMatch = stringsList.filter((data) => !hasNonFuzzyOrCloseFuzzyMatch(exactMatchedMap[data]));
|
|
407
|
-
if (stringsWithNoExactMatch.length === 0) {
|
|
408
|
-
return exactMatchedMap;
|
|
409
|
-
}
|
|
410
|
-
const fuzzMatchedMap = matchFuzzyStrings(domNode, stringsWithNoExactMatch);
|
|
411
|
-
for (const [data, fuzzyMatches] of Object.entries(fuzzMatchedMap)) {
|
|
412
|
-
if (data in exactMatchedMap) {
|
|
413
|
-
exactMatchedMap[data].push(...fuzzyMatches);
|
|
414
|
-
}
|
|
415
|
-
else {
|
|
416
|
-
exactMatchedMap[data] = fuzzyMatches;
|
|
417
|
-
}
|
|
418
|
-
}
|
|
419
|
-
// attributes to try fuzzy match attributes on
|
|
420
|
-
const stringsWithNoMatch = stringsList.filter((data) => !hasNonFuzzyOrCloseFuzzyMatch(exactMatchedMap[data]));
|
|
421
|
-
const attributesFuzzyMatchedMap = matchFuzzyAttributes(domNode, stringsWithNoMatch);
|
|
422
|
-
for (const [data, attributeFuzzyMatches] of Object.entries(attributesFuzzyMatchedMap)) {
|
|
423
|
-
if (data in exactMatchedMap) {
|
|
424
|
-
exactMatchedMap[data].push(...attributeFuzzyMatches);
|
|
425
|
-
}
|
|
426
|
-
else {
|
|
427
|
-
exactMatchedMap[data] = attributeFuzzyMatches;
|
|
428
|
-
}
|
|
429
|
-
}
|
|
430
|
-
return exactMatchedMap;
|
|
431
|
-
}
|
|
432
|
-
function matchExactStrings(domNode, stringsList) {
|
|
433
|
-
const allNodes = [
|
|
434
|
-
domNode,
|
|
435
|
-
...Array.from(domNode.querySelectorAll("*")),
|
|
436
|
-
].reverse();
|
|
437
|
-
const matchesMap = Object.fromEntries(stringsList.map((data) => [data, []]));
|
|
438
|
-
for (const tag of allNodes) {
|
|
439
|
-
const xpath = getElementXPath(tag);
|
|
440
|
-
for (const stringValue of stringsList) {
|
|
441
|
-
const matchesXPaths = matchesMap[stringValue].map((match) => match.xpath || "");
|
|
442
|
-
const xpathIsChildOfMatch = matchesXPaths.some((matchXPath) => matchXPath !== xpath && matchXPath.startsWith(xpath));
|
|
443
|
-
if (xpathIsChildOfMatch)
|
|
444
|
-
continue;
|
|
445
|
-
const attributeNames = tag.getAttributeNames();
|
|
446
|
-
for (const attr of attributeNames) {
|
|
447
|
-
const attributeValue = tag.getAttribute(attr) || "";
|
|
448
|
-
const [isPartOfStringResult, matchedValue] = isPartOfString(stringValue, attributeValue);
|
|
449
|
-
if (isPartOfStringResult) {
|
|
450
|
-
const [isExact] = isMatchExact(stringValue, attributeValue);
|
|
451
|
-
matchesMap[stringValue].push({
|
|
452
|
-
attribute: attr,
|
|
453
|
-
fuzzy_distance: null,
|
|
454
|
-
match_mode: isExact ? MatchMode.FULL : MatchMode.PARTIAL,
|
|
455
|
-
match_source: MatchSource.ATTRIBUTE,
|
|
456
|
-
matched_value: matchedValue,
|
|
457
|
-
matched_source_value: attributeValue,
|
|
458
|
-
tag: tag.tagName.toLowerCase(),
|
|
459
|
-
xpath,
|
|
460
|
-
});
|
|
461
|
-
}
|
|
462
|
-
}
|
|
463
|
-
if (tag["href"]) {
|
|
464
|
-
const result = matchHref(tag, stringValue);
|
|
465
|
-
if (result) {
|
|
466
|
-
matchesMap[stringValue].push(result);
|
|
467
|
-
}
|
|
468
|
-
}
|
|
469
|
-
// Check for direct text nodes
|
|
470
|
-
for (const childNode of tag.childNodes) {
|
|
471
|
-
// Node.TEXT_NODE
|
|
472
|
-
if (childNode.nodeType === 3) {
|
|
473
|
-
const directTextContent = childNode.textContent?.trim() || "";
|
|
474
|
-
if (directTextContent) {
|
|
475
|
-
const [isPartOfStringResult, matchedValue, source_value] = isPartOfString(stringValue, directTextContent);
|
|
476
|
-
if (isPartOfStringResult) {
|
|
477
|
-
const [isExact] = isMatchExact(stringValue, directTextContent);
|
|
478
|
-
matchesMap[stringValue].push({
|
|
479
|
-
attribute: null,
|
|
480
|
-
fuzzy_distance: null,
|
|
481
|
-
match_mode: isExact ? MatchMode.FULL : MatchMode.PARTIAL,
|
|
482
|
-
match_source: MatchSource.DIRECT_TEXT_NODE,
|
|
483
|
-
matched_value: matchedValue,
|
|
484
|
-
matched_source_value: source_value,
|
|
485
|
-
tag: tag.tagName.toLowerCase(),
|
|
486
|
-
xpath,
|
|
487
|
-
});
|
|
488
|
-
}
|
|
489
|
-
}
|
|
490
|
-
}
|
|
491
|
-
}
|
|
492
|
-
const tagTextContent = tag.textContent || "";
|
|
493
|
-
const [isPartOfStringResult, matchedValue, source_value] = isPartOfString(stringValue, tagTextContent);
|
|
494
|
-
if (isPartOfStringResult) {
|
|
495
|
-
const [isExact] = isMatchExact(stringValue, tagTextContent);
|
|
496
|
-
matchesMap[stringValue].push({
|
|
497
|
-
attribute: null,
|
|
498
|
-
fuzzy_distance: null,
|
|
499
|
-
match_mode: isExact ? MatchMode.FULL : MatchMode.PARTIAL,
|
|
500
|
-
match_source: MatchSource.TEXT_CONTENT,
|
|
501
|
-
matched_value: matchedValue,
|
|
502
|
-
matched_source_value: source_value,
|
|
503
|
-
tag: tag.tagName.toLowerCase(),
|
|
504
|
-
xpath,
|
|
505
|
-
});
|
|
506
|
-
}
|
|
507
|
-
}
|
|
508
|
-
}
|
|
509
|
-
return matchesMap;
|
|
510
|
-
}
|
|
511
|
-
function matchFuzzyStrings(domNode, stringsToMatch) {
|
|
512
|
-
const matchesMap = Object.fromEntries(stringsToMatch.map((data) => [data, []]));
|
|
513
|
-
const conditionFunc = (stringToMatch, node) => {
|
|
514
|
-
let foundMatch = false;
|
|
515
|
-
const currentXPath = getElementXPath(node);
|
|
516
|
-
for (const attr of node.getAttributeNames()) {
|
|
517
|
-
const attributeValue = node.getAttribute(attr) || "";
|
|
518
|
-
const { found: isFuzzMatchFound, matchedValue, distance: dist, matchedSourceValue, } = isFuzzMatch(stringToMatch, attributeValue);
|
|
519
|
-
if (isFuzzMatchFound) {
|
|
520
|
-
matchesMap[stringToMatch].push({
|
|
521
|
-
attribute: attr,
|
|
522
|
-
fuzzy_distance: dist,
|
|
523
|
-
match_mode: MatchMode.FUZZY,
|
|
524
|
-
match_source: MatchSource.ATTRIBUTE,
|
|
525
|
-
matched_value: matchedValue,
|
|
526
|
-
tag: node.tagName.toLowerCase(),
|
|
527
|
-
xpath: currentXPath,
|
|
528
|
-
matched_source_value: matchedSourceValue,
|
|
529
|
-
});
|
|
530
|
-
foundMatch = true;
|
|
531
|
-
}
|
|
532
|
-
}
|
|
533
|
-
const tagTextContent = node.textContent || "";
|
|
534
|
-
if (tagTextContent) {
|
|
535
|
-
const { found: isFuzzMatchFound, matchedValue, distance: dist, matchedSourceValue, } = isFuzzMatch(stringToMatch, tagTextContent);
|
|
536
|
-
if (isFuzzMatchFound) {
|
|
537
|
-
matchesMap[stringToMatch].push({
|
|
538
|
-
attribute: null,
|
|
539
|
-
fuzzy_distance: dist,
|
|
540
|
-
match_mode: MatchMode.FUZZY,
|
|
541
|
-
match_source: MatchSource.TEXT_CONTENT,
|
|
542
|
-
matched_value: matchedValue,
|
|
543
|
-
tag: node.tagName.toLowerCase(),
|
|
544
|
-
xpath: currentXPath,
|
|
545
|
-
matched_source_value: matchedSourceValue,
|
|
546
|
-
});
|
|
547
|
-
foundMatch = true;
|
|
548
|
-
}
|
|
549
|
-
}
|
|
550
|
-
// Check for direct text nodes
|
|
551
|
-
for (const childNode of node.childNodes) {
|
|
552
|
-
// Node.TEXT_NODE
|
|
553
|
-
if (childNode.nodeType === 3) {
|
|
554
|
-
const directTextContent = childNode.textContent?.trim() || "";
|
|
555
|
-
if (directTextContent) {
|
|
556
|
-
const { found: isFuzzMatchFound, matchedValue, distance: dist, matchedSourceValue, } = isFuzzMatch(stringToMatch, directTextContent);
|
|
557
|
-
if (isFuzzMatchFound) {
|
|
558
|
-
matchesMap[stringToMatch].push({
|
|
559
|
-
attribute: null,
|
|
560
|
-
fuzzy_distance: dist,
|
|
561
|
-
match_mode: MatchMode.FUZZY,
|
|
562
|
-
match_source: MatchSource.DIRECT_TEXT_NODE,
|
|
563
|
-
matched_value: matchedValue,
|
|
564
|
-
tag: node.tagName.toLowerCase(),
|
|
565
|
-
xpath: currentXPath,
|
|
566
|
-
matched_source_value: matchedSourceValue,
|
|
567
|
-
});
|
|
568
|
-
foundMatch = true;
|
|
569
|
-
}
|
|
570
|
-
}
|
|
571
|
-
}
|
|
572
|
-
}
|
|
573
|
-
return !foundMatch;
|
|
574
|
-
};
|
|
575
|
-
for (const stringToMatch of stringsToMatch) {
|
|
576
|
-
conditionFunc(stringToMatch, domNode);
|
|
577
|
-
traverseAndPrune(domNode, (node) => conditionFunc(stringToMatch, node));
|
|
578
|
-
}
|
|
579
|
-
for (const [stringToMatch, matches] of Object.entries(matchesMap)) {
|
|
580
|
-
const matchesToRemove = new Set();
|
|
581
|
-
matches.forEach((match, i) => {
|
|
582
|
-
for (const otherMatch of matches.slice(i + 1)) {
|
|
583
|
-
if ((otherMatch.xpath || "").startsWith((match.xpath || "") + "/")) {
|
|
584
|
-
matchesToRemove.add(i);
|
|
585
|
-
break;
|
|
586
|
-
}
|
|
587
|
-
}
|
|
588
|
-
});
|
|
589
|
-
matchesMap[stringToMatch] = matches.filter((_, i) => !matchesToRemove.has(i));
|
|
590
|
-
}
|
|
591
|
-
return matchesMap;
|
|
592
|
-
}
|
|
593
|
-
function matchFuzzyAttributes(domNode, stringsToMatch) {
|
|
594
|
-
const matchesMap = Object.fromEntries(stringsToMatch.map((data) => [data, []]));
|
|
595
|
-
const allAttributes = getAllAttributes(domNode);
|
|
596
|
-
for (const stringToMatch of stringsToMatch) {
|
|
597
|
-
const stringToSearchIn = allAttributes
|
|
598
|
-
.filter((attr) => attr.value.length > 10)
|
|
599
|
-
.filter((attr) => {
|
|
600
|
-
const lengthDiff = Math.abs(attr.value.length - stringToMatch.length);
|
|
601
|
-
return lengthDiff <= 0.2 * stringToMatch.length;
|
|
602
|
-
})
|
|
603
|
-
.map((attr) => attr.value)
|
|
604
|
-
.join("\n");
|
|
605
|
-
const { found: isFuzzMatchFound, matchedValue, distance: dist, } = isFuzzMatch(stringToMatch, stringToSearchIn);
|
|
606
|
-
if (isFuzzMatchFound) {
|
|
607
|
-
const matchLine = allAttributes.find((attr) => matchedValue && attr.value.includes(matchedValue));
|
|
608
|
-
if (!matchLine)
|
|
609
|
-
continue;
|
|
610
|
-
matchesMap[stringToMatch].push({
|
|
611
|
-
attribute: matchLine.attr,
|
|
612
|
-
fuzzy_distance: dist,
|
|
613
|
-
match_mode: MatchMode.FUZZY,
|
|
614
|
-
match_source: MatchSource.ATTRIBUTE,
|
|
615
|
-
matched_value: matchedValue,
|
|
616
|
-
xpath: matchLine.node,
|
|
617
|
-
matched_source_value: matchLine.value,
|
|
618
|
-
tag: matchLine.tag,
|
|
619
|
-
});
|
|
620
|
-
}
|
|
621
|
-
}
|
|
622
|
-
return matchesMap;
|
|
623
|
-
}
|
|
624
|
-
function getAllAttributes(node) {
|
|
625
|
-
const allNodes = [node, ...Array.from(node.querySelectorAll("*"))].reverse();
|
|
626
|
-
return allNodes.flatMap((node) => node
|
|
627
|
-
.getAttributeNames()
|
|
628
|
-
.map((attr) => ({
|
|
629
|
-
node: getElementXPath(node),
|
|
630
|
-
attr,
|
|
631
|
-
value: node.getAttribute(attr) || "",
|
|
632
|
-
tag: node.tagName.toLowerCase(),
|
|
633
|
-
}))
|
|
634
|
-
.filter((i) => i.value.length > 10));
|
|
635
|
-
}
|
|
636
|
-
function matchHref(node, stringToMatch) {
|
|
637
|
-
if (!node["href"] || typeof node["href"] !== "string") {
|
|
638
|
-
return;
|
|
639
|
-
}
|
|
640
|
-
const attributeValue = node["href"] || "";
|
|
641
|
-
let [isPartOfStringResult, matchedValue] = isPartOfString(stringToMatch, attributeValue);
|
|
642
|
-
if (isPartOfStringResult) {
|
|
643
|
-
const [isExact] = isMatchExact(stringToMatch, attributeValue);
|
|
644
|
-
return {
|
|
645
|
-
attribute: "href",
|
|
646
|
-
fuzzy_distance: null,
|
|
647
|
-
match_mode: isExact ? MatchMode.FULL : MatchMode.PARTIAL,
|
|
648
|
-
match_source: MatchSource.ATTRIBUTE,
|
|
649
|
-
matched_value: matchedValue,
|
|
650
|
-
matched_source_value: attributeValue,
|
|
651
|
-
tag: node.tagName.toLowerCase(),
|
|
652
|
-
xpath: getElementXPath(node),
|
|
653
|
-
};
|
|
654
|
-
}
|
|
655
|
-
let decodedStringToMatch;
|
|
656
|
-
try {
|
|
657
|
-
decodedStringToMatch = decodeURI(stringToMatch);
|
|
658
|
-
}
|
|
659
|
-
catch (e) {
|
|
660
|
-
console.log("failed to decode stringToMatch", stringToMatch);
|
|
661
|
-
return;
|
|
662
|
-
}
|
|
663
|
-
[isPartOfStringResult, matchedValue] = isPartOfString(decodedStringToMatch, attributeValue);
|
|
664
|
-
if (isPartOfStringResult) {
|
|
665
|
-
const [isExact] = isMatchExact(stringToMatch, attributeValue);
|
|
666
|
-
return {
|
|
667
|
-
attribute: "href",
|
|
668
|
-
fuzzy_distance: null,
|
|
669
|
-
match_mode: isExact ? MatchMode.FULL : MatchMode.PARTIAL,
|
|
670
|
-
match_source: MatchSource.ATTRIBUTE,
|
|
671
|
-
matched_value: matchedValue,
|
|
672
|
-
matched_source_value: attributeValue,
|
|
673
|
-
tag: node.tagName.toLowerCase(),
|
|
674
|
-
xpath: getElementXPath(node),
|
|
675
|
-
};
|
|
676
|
-
}
|
|
677
|
-
}
|
|
678
|
-
|
|
679
|
-
function convertElementToMarkdown(element) {
|
|
680
|
-
const mdCharsMatcher = /([\\[\]()])/g;
|
|
681
|
-
function escapeMd(text) {
|
|
682
|
-
// Escapes markdown-sensitive characters within other markdown constructs.
|
|
683
|
-
return text.replace(mdCharsMatcher, "\\$1");
|
|
684
|
-
}
|
|
685
|
-
function listNumberingStart(attrs) {
|
|
686
|
-
const start = attrs.getNamedItem("start")?.value;
|
|
687
|
-
if (start) {
|
|
688
|
-
return parseInt(start, 10) - 1;
|
|
689
|
-
}
|
|
690
|
-
else {
|
|
691
|
-
return 0;
|
|
692
|
-
}
|
|
693
|
-
}
|
|
694
|
-
// Define the characters that require escaping
|
|
695
|
-
const slashChars = "\\`*_{}[]()#+-.!";
|
|
696
|
-
// Escape any special regex characters in slashChars
|
|
697
|
-
const escapedSlashChars = slashChars.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\$&");
|
|
698
|
-
// Create the regular expression
|
|
699
|
-
const mdBackslashMatcher = new RegExp(`\\\\(?=[${escapedSlashChars}])`, "g");
|
|
700
|
-
const mdDotMatcher = new RegExp(`^(\\s*\\d+)(\\.)(?=\\s)`, "gm");
|
|
701
|
-
const mdPlusMatcher = new RegExp(`^(\\s*)(\\+)(?=\\s)`, "gm");
|
|
702
|
-
const mdDashMatcher = new RegExp(`^(\\s*)(-)(?=\\s|-)`, "gm");
|
|
703
|
-
function escapeMdSection(text) {
|
|
704
|
-
text = text.replace(mdBackslashMatcher, "\\\\");
|
|
705
|
-
text = text.replace(mdDotMatcher, "$1\\$2");
|
|
706
|
-
text = text.replace(mdPlusMatcher, "$1\\$2");
|
|
707
|
-
text = text.replace(mdDashMatcher, "$1\\$2");
|
|
708
|
-
return text;
|
|
709
|
-
}
|
|
710
|
-
function isFirstTbody(element) {
|
|
711
|
-
const previousSibling = element.previousSibling;
|
|
712
|
-
return (element.nodeName === "TBODY" &&
|
|
713
|
-
(!previousSibling ||
|
|
714
|
-
(previousSibling.nodeName === "THEAD" &&
|
|
715
|
-
/^\s*$/i.test(previousSibling.textContent ?? ""))));
|
|
716
|
-
}
|
|
717
|
-
function isHeadingRow(tr) {
|
|
718
|
-
const parentNode = tr.parentNode;
|
|
719
|
-
return (parentNode.nodeName === "THEAD" ||
|
|
720
|
-
(parentNode.firstChild === tr &&
|
|
721
|
-
(parentNode.nodeName === "TABLE" ||
|
|
722
|
-
isFirstTbody(parentNode)) &&
|
|
723
|
-
Array.from(tr.childNodes).every(function (n) {
|
|
724
|
-
return n.nodeName === "TH";
|
|
725
|
-
})));
|
|
726
|
-
}
|
|
727
|
-
class Html2Text {
|
|
728
|
-
p_p = 0;
|
|
729
|
-
abbrData; // last inner HTML (for abbr being defined)
|
|
730
|
-
pre = false;
|
|
731
|
-
code = false;
|
|
732
|
-
startPre = false;
|
|
733
|
-
blockquote = 0;
|
|
734
|
-
list = [];
|
|
735
|
-
start = true;
|
|
736
|
-
breakToggle = "";
|
|
737
|
-
space;
|
|
738
|
-
lastWasNewLine = false;
|
|
739
|
-
a = null;
|
|
740
|
-
outCount = 0;
|
|
741
|
-
baseurl;
|
|
742
|
-
abbrList = {};
|
|
743
|
-
outText = "";
|
|
744
|
-
outTextList = [];
|
|
745
|
-
abbr_title;
|
|
746
|
-
skipInternalLinks = true;
|
|
747
|
-
aStack = [];
|
|
748
|
-
maybeAutomaticLink;
|
|
749
|
-
lastWasList = false;
|
|
750
|
-
absoluteUrlMatcher = new RegExp("^[a-zA-Z+]+://");
|
|
751
|
-
emphasis_mark = "_";
|
|
752
|
-
strong_mark = "**";
|
|
753
|
-
break() {
|
|
754
|
-
if (this.p_p === 0) {
|
|
755
|
-
this.p_p = 1;
|
|
756
|
-
}
|
|
757
|
-
}
|
|
758
|
-
softBreak() {
|
|
759
|
-
this.break();
|
|
760
|
-
this.breakToggle = " ";
|
|
761
|
-
}
|
|
762
|
-
processOutput(data, pureData = 0, force = 0) {
|
|
763
|
-
if (this.abbrData !== undefined) {
|
|
764
|
-
this.abbrData += data;
|
|
765
|
-
}
|
|
766
|
-
if (pureData && !this.pre) {
|
|
767
|
-
data = data.replace(/\s+/g, " ");
|
|
768
|
-
if (data && data[0] === " ") {
|
|
769
|
-
this.space = 1;
|
|
770
|
-
data = data.substring(1);
|
|
771
|
-
}
|
|
772
|
-
}
|
|
773
|
-
if (!data && force !== "end")
|
|
774
|
-
return;
|
|
775
|
-
if (this.startPre) {
|
|
776
|
-
if (!data.startsWith("\n")) {
|
|
777
|
-
data = "\n" + data;
|
|
778
|
-
}
|
|
779
|
-
}
|
|
780
|
-
let newLineIndent = ">".repeat(this.blockquote ?? 0);
|
|
781
|
-
if (!(force === "end" && data && data[0] === ">") && this.blockquote) {
|
|
782
|
-
newLineIndent += " ";
|
|
783
|
-
}
|
|
784
|
-
if (this.pre) {
|
|
785
|
-
if (this.list.length === 0) {
|
|
786
|
-
newLineIndent += " ";
|
|
787
|
-
}
|
|
788
|
-
else {
|
|
789
|
-
for (let i = 0; i < this.list.length + 1; i++) {
|
|
790
|
-
newLineIndent += " ";
|
|
791
|
-
}
|
|
792
|
-
}
|
|
793
|
-
data = data.replace(/\n/g, `\n${newLineIndent}`);
|
|
794
|
-
}
|
|
795
|
-
if (this.startPre) {
|
|
796
|
-
this.startPre = false;
|
|
797
|
-
if (this.list.length > 0) {
|
|
798
|
-
data = data.trimStart();
|
|
799
|
-
}
|
|
800
|
-
}
|
|
801
|
-
if (this.start) {
|
|
802
|
-
this.space = 0;
|
|
803
|
-
this.p_p = 0;
|
|
804
|
-
this.start = false;
|
|
805
|
-
}
|
|
806
|
-
if (force === "end") {
|
|
807
|
-
this.p_p = 0;
|
|
808
|
-
this.out("\n");
|
|
809
|
-
this.space = 0;
|
|
810
|
-
}
|
|
811
|
-
if (this.p_p) {
|
|
812
|
-
this.out((this.breakToggle + "\n" + newLineIndent).repeat(this.p_p));
|
|
813
|
-
this.space = 0;
|
|
814
|
-
this.breakToggle = "";
|
|
815
|
-
}
|
|
816
|
-
if (this.space) {
|
|
817
|
-
if (!this.lastWasNewLine) {
|
|
818
|
-
this.out(" ");
|
|
819
|
-
}
|
|
820
|
-
this.space = 0;
|
|
821
|
-
}
|
|
822
|
-
if (this.a && force === "end") {
|
|
823
|
-
if (force === "end") {
|
|
824
|
-
this.out("\n");
|
|
825
|
-
}
|
|
826
|
-
const newA = this.a.filter((link) => {
|
|
827
|
-
if (this.outCount > link.outcount) {
|
|
828
|
-
this.out(" [" +
|
|
829
|
-
link.count +
|
|
830
|
-
"]: " +
|
|
831
|
-
new URL(link.href, this.baseurl).toString());
|
|
832
|
-
if (link.title) {
|
|
833
|
-
this.out(" (" + link.title + ")");
|
|
834
|
-
}
|
|
835
|
-
this.out("\n");
|
|
836
|
-
return false;
|
|
837
|
-
}
|
|
838
|
-
return true;
|
|
839
|
-
});
|
|
840
|
-
if (this.a.length !== newA.length) {
|
|
841
|
-
this.out("\n");
|
|
842
|
-
}
|
|
843
|
-
this.a = newA;
|
|
844
|
-
}
|
|
845
|
-
if (this.abbrList && force === "end") {
|
|
846
|
-
for (const [abbr, definition] of Object.entries(this.abbrList)) {
|
|
847
|
-
this.out("\n *[" + abbr + "]: " + definition + "\n");
|
|
848
|
-
}
|
|
849
|
-
}
|
|
850
|
-
this.p_p = 0;
|
|
851
|
-
this.out(data);
|
|
852
|
-
this.outCount++;
|
|
853
|
-
}
|
|
854
|
-
out(string) {
|
|
855
|
-
this.outTextList.push(string);
|
|
856
|
-
if (string) {
|
|
857
|
-
this.lastWasNewLine = string.charAt(string.length - 1) === "\n";
|
|
858
|
-
}
|
|
859
|
-
}
|
|
860
|
-
getResult() {
|
|
861
|
-
this.processOutput("", 0, "end");
|
|
862
|
-
this.outText = this.outTextList.join("");
|
|
863
|
-
this.outText = this.outText.replace(" _place_holder;", " ");
|
|
864
|
-
return this.outText;
|
|
865
|
-
}
|
|
866
|
-
getHeadingLevel(tag) {
|
|
867
|
-
if (tag[0] === "h" && tag.length === 2) {
|
|
868
|
-
try {
|
|
869
|
-
const n = parseInt(tag[1]);
|
|
870
|
-
if (!isNaN(n) && n >= 1 && n <= 9) {
|
|
871
|
-
return n;
|
|
872
|
-
}
|
|
873
|
-
}
|
|
874
|
-
catch (error) {
|
|
875
|
-
return 0;
|
|
876
|
-
}
|
|
877
|
-
}
|
|
878
|
-
return 0;
|
|
879
|
-
}
|
|
880
|
-
padding() {
|
|
881
|
-
this.p_p = 2;
|
|
882
|
-
}
|
|
883
|
-
handleData(node) {
|
|
884
|
-
if (this.maybeAutomaticLink) {
|
|
885
|
-
const href = this.maybeAutomaticLink;
|
|
886
|
-
if (href?.value === node.nodeValue &&
|
|
887
|
-
this.absoluteUrlMatcher.test(href.value)) {
|
|
888
|
-
this.processOutput(`<${node.nodeValue}>`);
|
|
889
|
-
return;
|
|
890
|
-
}
|
|
891
|
-
else {
|
|
892
|
-
this.processOutput("[");
|
|
893
|
-
this.maybeAutomaticLink = null;
|
|
894
|
-
}
|
|
895
|
-
}
|
|
896
|
-
if (!this.code && !this.pre && node.nodeValue) {
|
|
897
|
-
const data = escapeMdSection(node.nodeValue);
|
|
898
|
-
this.processOutput(data, 1);
|
|
899
|
-
return;
|
|
900
|
-
}
|
|
901
|
-
this.processOutput(node.textContent || "", 1);
|
|
902
|
-
}
|
|
903
|
-
handleTag(node) {
|
|
904
|
-
const tag = node.nodeName.toLowerCase();
|
|
905
|
-
if (["head", "style", "script"].includes(tag)) {
|
|
906
|
-
return;
|
|
907
|
-
}
|
|
908
|
-
if (this.getHeadingLevel(tag)) {
|
|
909
|
-
this.padding();
|
|
910
|
-
this.processOutput("#".repeat(this.getHeadingLevel(tag)) + " ");
|
|
911
|
-
}
|
|
912
|
-
if (tag == "br")
|
|
913
|
-
this.processOutput(" \n");
|
|
914
|
-
if (tag == "hr") {
|
|
915
|
-
this.padding();
|
|
916
|
-
this.processOutput("---");
|
|
917
|
-
this.padding();
|
|
918
|
-
}
|
|
919
|
-
if (tag == "blockquote") {
|
|
920
|
-
this.padding();
|
|
921
|
-
this.processOutput("> ", 0, 1);
|
|
922
|
-
}
|
|
923
|
-
}
|
|
924
|
-
handleTagPrefix(node) {
|
|
925
|
-
const nodeName = node.nodeName.toLowerCase();
|
|
926
|
-
let attrs = node.nodeType === node.ELEMENT_NODE
|
|
927
|
-
? node.attributes
|
|
928
|
-
: null;
|
|
929
|
-
if (["table"].includes(nodeName)) {
|
|
930
|
-
this.padding();
|
|
931
|
-
}
|
|
932
|
-
if (nodeName == "td" || nodeName == "th") {
|
|
933
|
-
const index = Array.from(node.parentNode?.children ?? []).indexOf(node);
|
|
934
|
-
let prefix = " ";
|
|
935
|
-
if (index === 0)
|
|
936
|
-
prefix = "| ";
|
|
937
|
-
this.processOutput(prefix);
|
|
938
|
-
// this.break();
|
|
939
|
-
}
|
|
940
|
-
if (["div", "p"].includes(nodeName)) {
|
|
941
|
-
this.padding();
|
|
942
|
-
}
|
|
943
|
-
if (nodeName === "blockquote") {
|
|
944
|
-
this.blockquote += 1;
|
|
945
|
-
}
|
|
946
|
-
if (nodeName === "pre") {
|
|
947
|
-
this.pre = true;
|
|
948
|
-
this.startPre = true;
|
|
949
|
-
this.padding();
|
|
950
|
-
}
|
|
951
|
-
if (["code", "tt"].includes(nodeName)) {
|
|
952
|
-
this.processOutput("`");
|
|
953
|
-
}
|
|
954
|
-
if (["em", "i", "u"].includes(nodeName)) {
|
|
955
|
-
this.processOutput(this.emphasis_mark);
|
|
956
|
-
}
|
|
957
|
-
if (["strong", "b"].includes(nodeName)) {
|
|
958
|
-
this.processOutput(this.strong_mark);
|
|
959
|
-
}
|
|
960
|
-
if (["del", "strike", "s"].includes(nodeName)) {
|
|
961
|
-
this.processOutput("<" + nodeName + ">");
|
|
962
|
-
}
|
|
963
|
-
if (nodeName === "abbr") {
|
|
964
|
-
this.abbr_title = null;
|
|
965
|
-
this.abbrData = "";
|
|
966
|
-
const title = attrs && attrs.getNamedItem("title");
|
|
967
|
-
if (attrs && title) {
|
|
968
|
-
this.abbr_title = title.value;
|
|
969
|
-
}
|
|
970
|
-
}
|
|
971
|
-
if (nodeName === "dl") {
|
|
972
|
-
this.padding();
|
|
973
|
-
}
|
|
974
|
-
if (nodeName === "dd") {
|
|
975
|
-
this.processOutput(" ");
|
|
976
|
-
}
|
|
977
|
-
if (nodeName == "a") {
|
|
978
|
-
const href = attrs ? attrs.getNamedItem("href") : null;
|
|
979
|
-
if (href && !(this.skipInternalLinks && href.value.startsWith("#"))) {
|
|
980
|
-
this.aStack.push(attrs);
|
|
981
|
-
this.maybeAutomaticLink = href;
|
|
982
|
-
}
|
|
983
|
-
else {
|
|
984
|
-
this.aStack.push(null);
|
|
985
|
-
}
|
|
986
|
-
}
|
|
987
|
-
if (nodeName === "img") {
|
|
988
|
-
const src = attrs ? attrs.getNamedItem("src") : null;
|
|
989
|
-
if (src) {
|
|
990
|
-
node.setAttribute("href", src.value);
|
|
991
|
-
attrs = node.attributes;
|
|
992
|
-
const alt = attrs.getNamedItem("alt")?.value;
|
|
993
|
-
this.processOutput("![" + escapeMd(alt ?? "") + "]");
|
|
994
|
-
this.processOutput("(" + escapeMd(attrs.getNamedItem("href")?.value ?? "") + ")");
|
|
995
|
-
}
|
|
996
|
-
}
|
|
997
|
-
if (["ul", "ol"].includes(nodeName)) {
|
|
998
|
-
const listStyle = nodeName;
|
|
999
|
-
const numberingStart = listNumberingStart(node.attributes);
|
|
1000
|
-
this.list.push({ name: listStyle, num: numberingStart });
|
|
1001
|
-
this.lastWasList = true;
|
|
1002
|
-
}
|
|
1003
|
-
else {
|
|
1004
|
-
this.lastWasList = false;
|
|
1005
|
-
}
|
|
1006
|
-
if (nodeName === "li") {
|
|
1007
|
-
let li;
|
|
1008
|
-
this.break();
|
|
1009
|
-
if (this.list.length > 0) {
|
|
1010
|
-
li = this.list[this.list.length - 1];
|
|
1011
|
-
}
|
|
1012
|
-
else {
|
|
1013
|
-
li = { name: "ul", num: 0 };
|
|
1014
|
-
}
|
|
1015
|
-
const nestCount = this.list.length;
|
|
1016
|
-
this.processOutput(" ".repeat(nestCount));
|
|
1017
|
-
if (li["name"] == "ul")
|
|
1018
|
-
this.processOutput("*" + " ");
|
|
1019
|
-
else if (li["name"] == "ol") {
|
|
1020
|
-
li["num"] += 1;
|
|
1021
|
-
this.processOutput(li["num"] + ". ");
|
|
1022
|
-
}
|
|
1023
|
-
this.start = true;
|
|
1024
|
-
}
|
|
1025
|
-
}
|
|
1026
|
-
handleTagSuffix(node) {
|
|
1027
|
-
const nodeName = node.nodeName.toLowerCase();
|
|
1028
|
-
if (nodeName === "blockquote") {
|
|
1029
|
-
this.blockquote -= 1;
|
|
1030
|
-
}
|
|
1031
|
-
if (nodeName == "td" || nodeName == "th") {
|
|
1032
|
-
this.processOutput(" |");
|
|
1033
|
-
}
|
|
1034
|
-
if (nodeName == "tr") {
|
|
1035
|
-
const cell = (content, node) => {
|
|
1036
|
-
const index = Array.from(node.parentNode.childNodes).indexOf(node);
|
|
1037
|
-
let prefix = " ";
|
|
1038
|
-
if (index === 0)
|
|
1039
|
-
prefix = "| ";
|
|
1040
|
-
return prefix + content + " |";
|
|
1041
|
-
};
|
|
1042
|
-
let borderCells = "";
|
|
1043
|
-
const alignMap = { left: ":--", right: "--:", center: ":-:" };
|
|
1044
|
-
if (isHeadingRow(node)) {
|
|
1045
|
-
for (let i = 0; i < node.children.length; i++) {
|
|
1046
|
-
let border = "---";
|
|
1047
|
-
const align = (node.children[i].getAttribute("align") || "").toLowerCase();
|
|
1048
|
-
if (align)
|
|
1049
|
-
border = alignMap[align] || border;
|
|
1050
|
-
borderCells += cell(border, node.childNodes[i]);
|
|
1051
|
-
}
|
|
1052
|
-
}
|
|
1053
|
-
this.processOutput(borderCells ? "\n" + borderCells + "\n" : "\n");
|
|
1054
|
-
}
|
|
1055
|
-
if (nodeName === "pre") {
|
|
1056
|
-
this.pre = false;
|
|
1057
|
-
this.padding();
|
|
1058
|
-
}
|
|
1059
|
-
if (["code", "tt"].includes(nodeName)) {
|
|
1060
|
-
this.processOutput("`");
|
|
1061
|
-
}
|
|
1062
|
-
if (["em", "i", "u"].includes(nodeName)) {
|
|
1063
|
-
this.processOutput(this.emphasis_mark);
|
|
1064
|
-
}
|
|
1065
|
-
if (["strong", "b"].includes(nodeName)) {
|
|
1066
|
-
this.processOutput(this.strong_mark);
|
|
1067
|
-
}
|
|
1068
|
-
if (["div", "p"].includes(nodeName)) {
|
|
1069
|
-
this.padding();
|
|
1070
|
-
}
|
|
1071
|
-
if (["del", "strike", "s"].includes(nodeName)) {
|
|
1072
|
-
this.processOutput("</" + nodeName + ">");
|
|
1073
|
-
}
|
|
1074
|
-
if (nodeName === "abbr") {
|
|
1075
|
-
if (this.abbr_title && this.abbrData) {
|
|
1076
|
-
this.abbrList[this.abbrData] = this.abbr_title;
|
|
1077
|
-
this.abbr_title = null;
|
|
1078
|
-
}
|
|
1079
|
-
this.abbrData = "";
|
|
1080
|
-
}
|
|
1081
|
-
if (nodeName === "dt") {
|
|
1082
|
-
this.break();
|
|
1083
|
-
}
|
|
1084
|
-
if (nodeName === "dd") {
|
|
1085
|
-
this.break();
|
|
1086
|
-
}
|
|
1087
|
-
if (nodeName === "a") {
|
|
1088
|
-
if (this.aStack.length > 0) {
|
|
1089
|
-
const a = this.aStack.pop();
|
|
1090
|
-
if (this.maybeAutomaticLink) {
|
|
1091
|
-
this.maybeAutomaticLink = null;
|
|
1092
|
-
}
|
|
1093
|
-
else if (a) {
|
|
1094
|
-
this.processOutput(`](${escapeMd(a.getNamedItem("href")?.value || "")})`);
|
|
1095
|
-
}
|
|
1096
|
-
}
|
|
1097
|
-
}
|
|
1098
|
-
if (["ul", "ol"].includes(nodeName)) {
|
|
1099
|
-
if (this.list.length > 0)
|
|
1100
|
-
this.list.pop();
|
|
1101
|
-
this.lastWasList = true;
|
|
1102
|
-
}
|
|
1103
|
-
else {
|
|
1104
|
-
this.lastWasList = false;
|
|
1105
|
-
}
|
|
1106
|
-
if (nodeName === "li") {
|
|
1107
|
-
this.break();
|
|
1108
|
-
}
|
|
1109
|
-
}
|
|
1110
|
-
previousIndex(attrs) {
|
|
1111
|
-
// Returns the index of a certain set of attributes (of a link) in the
|
|
1112
|
-
// this.a list.
|
|
1113
|
-
// If the set of attributes is not found, returns null.
|
|
1114
|
-
const href = attrs.getNamedItem("href");
|
|
1115
|
-
if (!attrs.getNamedItem("href"))
|
|
1116
|
-
return null;
|
|
1117
|
-
let itemIndex = -1;
|
|
1118
|
-
for (const a of this.a ?? []) {
|
|
1119
|
-
itemIndex += 1;
|
|
1120
|
-
let match = false;
|
|
1121
|
-
if (a.getNamedItem("href") === href) {
|
|
1122
|
-
if (a.getNamedItem("title") || attrs.getNamedItem("title")) {
|
|
1123
|
-
if (a.getNamedItem("title") &&
|
|
1124
|
-
attrs.getNamedItem("title") &&
|
|
1125
|
-
a.getNamedItem("title") === attrs.getNamedItem("title")) {
|
|
1126
|
-
match = true;
|
|
1127
|
-
}
|
|
1128
|
-
}
|
|
1129
|
-
else {
|
|
1130
|
-
match = true;
|
|
1131
|
-
}
|
|
1132
|
-
}
|
|
1133
|
-
if (match)
|
|
1134
|
-
return itemIndex;
|
|
1135
|
-
}
|
|
1136
|
-
return null;
|
|
1137
|
-
}
|
|
1138
|
-
handle(htmlElement) {
|
|
1139
|
-
// jsdom failed to parse hilton page due to invalid stylesheet
|
|
1140
|
-
// Nodes to be removed
|
|
1141
|
-
const filteredNodes = ["style", "script", "noscript"];
|
|
1142
|
-
for (const node of filteredNodes) {
|
|
1143
|
-
const nodeSelectors = htmlElement.querySelectorAll(node);
|
|
1144
|
-
nodeSelectors.forEach((nodeSelector) => {
|
|
1145
|
-
if (nodeSelector && nodeSelector.parentNode) {
|
|
1146
|
-
nodeSelector.parentNode.removeChild(nodeSelector);
|
|
1147
|
-
}
|
|
1148
|
-
});
|
|
1149
|
-
}
|
|
1150
|
-
// Get the cleaned-up HTML content
|
|
1151
|
-
const htmlContent = htmlElement.outerHTML;
|
|
1152
|
-
const parser = new DOMParser();
|
|
1153
|
-
const doc = parser.parseFromString(htmlContent, "text/html");
|
|
1154
|
-
const traverseDOM = (node) => {
|
|
1155
|
-
const tag = node.nodeName.toLowerCase();
|
|
1156
|
-
if (node.nodeType === node.TEXT_NODE) {
|
|
1157
|
-
const element = node;
|
|
1158
|
-
this.handleData(element);
|
|
1159
|
-
return;
|
|
1160
|
-
}
|
|
1161
|
-
if (node.nodeType === node.ELEMENT_NODE) {
|
|
1162
|
-
const element = node;
|
|
1163
|
-
this.handleTag(element);
|
|
1164
|
-
}
|
|
1165
|
-
if (!["head", "style", "script"].includes(tag)) {
|
|
1166
|
-
this.handleTagPrefix(node);
|
|
1167
|
-
node.childNodes.forEach((child) => traverseDOM(child));
|
|
1168
|
-
this.handleTagSuffix(node);
|
|
1169
|
-
}
|
|
1170
|
-
};
|
|
1171
|
-
traverseDOM(doc.documentElement);
|
|
1172
|
-
return this.getResult();
|
|
1173
|
-
}
|
|
1174
|
-
}
|
|
1175
|
-
const converter = new Html2Text();
|
|
1176
|
-
const result = converter.handle(element);
|
|
1177
|
-
return result;
|
|
1178
|
-
}
|
|
1179
|
-
|
|
1180
|
-
var node = {};
|
|
1181
|
-
|
|
1182
|
-
var htmlToMarkdownAST$1 = {};
|
|
1183
|
-
|
|
1184
|
-
var ElementNode = {};
|
|
1185
|
-
|
|
1186
|
-
Object.defineProperty(ElementNode, "__esModule", { value: true });
|
|
1187
|
-
ElementNode._Node = void 0;
|
|
1188
|
-
// this is by value copy of the global Node
|
|
1189
|
-
ElementNode._Node = {
|
|
1190
|
-
/** node is an element. */
|
|
1191
|
-
ELEMENT_NODE: 1,
|
|
1192
|
-
ATTRIBUTE_NODE: 2,
|
|
1193
|
-
/** node is a Text node. */
|
|
1194
|
-
TEXT_NODE: 3,
|
|
1195
|
-
/** node is a CDATASection node. */
|
|
1196
|
-
CDATA_SECTION_NODE: 4,
|
|
1197
|
-
ENTITY_REFERENCE_NODE: 5,
|
|
1198
|
-
ENTITY_NODE: 6,
|
|
1199
|
-
/** node is a ProcessingInstruction node. */
|
|
1200
|
-
PROCESSING_INSTRUCTION_NODE: 7,
|
|
1201
|
-
/** node is a Comment node. */
|
|
1202
|
-
COMMENT_NODE: 8,
|
|
1203
|
-
/** node is a document. */
|
|
1204
|
-
DOCUMENT_NODE: 9,
|
|
1205
|
-
/** node is a doctype. */
|
|
1206
|
-
DOCUMENT_TYPE_NODE: 10,
|
|
1207
|
-
/** node is a DocumentFragment node. */
|
|
1208
|
-
DOCUMENT_FRAGMENT_NODE: 11,
|
|
1209
|
-
NOTATION_NODE: 12,
|
|
1210
|
-
/** Set when node and other are not in the same tree. */
|
|
1211
|
-
DOCUMENT_POSITION_DISCONNECTED: 0x01,
|
|
1212
|
-
/** Set when other is preceding node. */
|
|
1213
|
-
DOCUMENT_POSITION_PRECEDING: 0x02,
|
|
1214
|
-
/** Set when other is following node. */
|
|
1215
|
-
DOCUMENT_POSITION_FOLLOWING: 0x04,
|
|
1216
|
-
/** Set when other is an ancestor of node. */
|
|
1217
|
-
DOCUMENT_POSITION_CONTAINS: 0x08,
|
|
1218
|
-
/** Set when other is a descendant of node. */
|
|
1219
|
-
DOCUMENT_POSITION_CONTAINED_BY: 0x10,
|
|
1220
|
-
DOCUMENT_POSITION_IMPLEMENTATION_SPECIFIC: 0x20,
|
|
1221
|
-
};
|
|
1222
|
-
|
|
1223
|
-
Object.defineProperty(htmlToMarkdownAST$1, "__esModule", { value: true });
|
|
1224
|
-
htmlToMarkdownAST$1.htmlToMarkdownAST = htmlToMarkdownAST;
|
|
1225
|
-
const ElementNode_1$1 = ElementNode;
|
|
1226
|
-
function htmlToMarkdownAST(element, options, indentLevel = 0) {
|
|
1227
|
-
let result = [];
|
|
1228
|
-
const debugLog = (message) => {
|
|
1229
|
-
if (options?.debug) {
|
|
1230
|
-
console.log(message);
|
|
1231
|
-
}
|
|
1232
|
-
};
|
|
1233
|
-
element.childNodes.forEach((childElement) => {
|
|
1234
|
-
const overriddenElementProcessing = options?.overrideElementProcessing?.(childElement, options, indentLevel);
|
|
1235
|
-
if (overriddenElementProcessing) {
|
|
1236
|
-
debugLog(`Element Processing Overridden: '${childElement.nodeType}'`);
|
|
1237
|
-
result.push(...overriddenElementProcessing);
|
|
1238
|
-
}
|
|
1239
|
-
else if (childElement.nodeType === ElementNode_1$1._Node.TEXT_NODE) {
|
|
1240
|
-
const textContent = escapeMarkdownCharacters(childElement.textContent?.trim() ?? '');
|
|
1241
|
-
if (textContent && !!childElement.textContent) {
|
|
1242
|
-
debugLog(`Text Node: '${textContent}'`);
|
|
1243
|
-
// preserve whitespaces when text childElement is not empty
|
|
1244
|
-
result.push({ type: 'text', content: childElement.textContent?.trim() });
|
|
1245
|
-
}
|
|
1246
|
-
}
|
|
1247
|
-
else if (childElement.nodeType === ElementNode_1$1._Node.ELEMENT_NODE) {
|
|
1248
|
-
const elem = childElement;
|
|
1249
|
-
if (/^h[1-6]$/i.test(elem.tagName)) {
|
|
1250
|
-
const level = parseInt(elem.tagName.substring(1));
|
|
1251
|
-
const content = escapeMarkdownCharacters(elem.textContent || '').trim();
|
|
1252
|
-
if (content) {
|
|
1253
|
-
debugLog(`Heading ${level}: '${elem.textContent}'`);
|
|
1254
|
-
result.push({ type: 'heading', level, content });
|
|
1255
|
-
}
|
|
1256
|
-
}
|
|
1257
|
-
else if (elem.tagName.toLowerCase() === 'p') {
|
|
1258
|
-
debugLog("Paragraph");
|
|
1259
|
-
result.push(...htmlToMarkdownAST(elem, options));
|
|
1260
|
-
// Add a new line after the paragraph
|
|
1261
|
-
result.push({ type: 'text', content: '\n\n' });
|
|
1262
|
-
}
|
|
1263
|
-
else if (elem.tagName.toLowerCase() === 'a') {
|
|
1264
|
-
debugLog(`Link: '${elem.href}' with text '${elem.textContent}'`);
|
|
1265
|
-
// Check if the href is a data URL for an image
|
|
1266
|
-
if (typeof elem.href === 'string' && elem.href.startsWith("data:image")) {
|
|
1267
|
-
// If it's a data URL for an image, skip this link
|
|
1268
|
-
result.push({
|
|
1269
|
-
type: 'link',
|
|
1270
|
-
href: '-',
|
|
1271
|
-
content: htmlToMarkdownAST(elem, options)
|
|
1272
|
-
});
|
|
1273
|
-
}
|
|
1274
|
-
else {
|
|
1275
|
-
// Process the link as usual
|
|
1276
|
-
let href = elem.href;
|
|
1277
|
-
if (typeof href === 'string') {
|
|
1278
|
-
href = options?.websiteDomain && href.startsWith(options.websiteDomain) ?
|
|
1279
|
-
href.substring(options.websiteDomain.length) : href;
|
|
1280
|
-
}
|
|
1281
|
-
else {
|
|
1282
|
-
href = '#'; // Use a default value when href is not a string
|
|
1283
|
-
}
|
|
1284
|
-
// if all children are text,
|
|
1285
|
-
if (Array.from(elem.childNodes).every(_ => _.nodeType === ElementNode_1$1._Node.TEXT_NODE)) {
|
|
1286
|
-
result.push({
|
|
1287
|
-
type: 'link',
|
|
1288
|
-
href: href,
|
|
1289
|
-
content: [{ type: 'text', content: elem.textContent?.trim() ?? '' }]
|
|
1290
|
-
});
|
|
1291
|
-
}
|
|
1292
|
-
else {
|
|
1293
|
-
result.push({
|
|
1294
|
-
type: 'link',
|
|
1295
|
-
href: href,
|
|
1296
|
-
content: htmlToMarkdownAST(elem, options)
|
|
1297
|
-
});
|
|
1298
|
-
}
|
|
1299
|
-
}
|
|
1300
|
-
}
|
|
1301
|
-
else if (elem.tagName.toLowerCase() === 'img') {
|
|
1302
|
-
debugLog(`Image: src='${elem.src}', alt='${elem.alt}'`);
|
|
1303
|
-
if (elem.src?.startsWith("data:image")) {
|
|
1304
|
-
result.push({
|
|
1305
|
-
type: 'image',
|
|
1306
|
-
src: '-',
|
|
1307
|
-
alt: escapeMarkdownCharacters(elem.alt)
|
|
1308
|
-
});
|
|
1309
|
-
}
|
|
1310
|
-
else {
|
|
1311
|
-
const src = options?.websiteDomain && elem.src?.startsWith(options.websiteDomain) ?
|
|
1312
|
-
elem.src?.substring(options.websiteDomain.length) :
|
|
1313
|
-
elem.src;
|
|
1314
|
-
result.push({ type: 'image', src, alt: escapeMarkdownCharacters(elem.alt) });
|
|
1315
|
-
}
|
|
1316
|
-
}
|
|
1317
|
-
else if (elem.tagName.toLowerCase() === 'video') {
|
|
1318
|
-
debugLog(`Video: src='${elem.src}', poster='${elem.poster}', controls='${elem.controls}'`);
|
|
1319
|
-
result.push({
|
|
1320
|
-
type: 'video',
|
|
1321
|
-
src: elem.src,
|
|
1322
|
-
poster: escapeMarkdownCharacters(elem.poster),
|
|
1323
|
-
controls: elem.controls
|
|
1324
|
-
});
|
|
1325
|
-
}
|
|
1326
|
-
else if (elem.tagName.toLowerCase() === 'ul' || elem.tagName.toLowerCase() === 'ol') {
|
|
1327
|
-
debugLog(`${elem.tagName.toLowerCase() === 'ul' ? 'Unordered' : 'Ordered'} List`);
|
|
1328
|
-
result.push({
|
|
1329
|
-
type: 'list',
|
|
1330
|
-
ordered: elem.tagName.toLowerCase() === 'ol',
|
|
1331
|
-
items: Array.from(elem.children).map(li => ({
|
|
1332
|
-
type: 'listItem',
|
|
1333
|
-
content: htmlToMarkdownAST(li, options, indentLevel + 1)
|
|
1334
|
-
}))
|
|
1335
|
-
});
|
|
1336
|
-
}
|
|
1337
|
-
else if (elem.tagName.toLowerCase() === 'br') {
|
|
1338
|
-
debugLog("Line Break");
|
|
1339
|
-
result.push({ type: 'text', content: '\n' });
|
|
1340
|
-
}
|
|
1341
|
-
else if (elem.tagName.toLowerCase() === 'table') {
|
|
1342
|
-
debugLog("Table");
|
|
1343
|
-
let colIds = [];
|
|
1344
|
-
if (options?.enableTableColumnTracking) {
|
|
1345
|
-
// Generate unique column IDs
|
|
1346
|
-
const headerCells = Array.from(elem.querySelectorAll('th, td'));
|
|
1347
|
-
headerCells.forEach((_, index) => {
|
|
1348
|
-
colIds.push(`col-${index}`);
|
|
1349
|
-
});
|
|
1350
|
-
}
|
|
1351
|
-
const tableRows = Array.from(elem.querySelectorAll('tr'));
|
|
1352
|
-
const markdownTableRows = tableRows.map(row => {
|
|
1353
|
-
let columnIndex = 0;
|
|
1354
|
-
const cells = Array.from(row.querySelectorAll('th, td')).map((cell) => {
|
|
1355
|
-
const colspan = parseInt(cell.getAttribute('colspan') || '1', 10);
|
|
1356
|
-
const rowspan = parseInt(cell.getAttribute('rowspan') || '1', 10);
|
|
1357
|
-
const cellNode = {
|
|
1358
|
-
type: 'tableCell',
|
|
1359
|
-
content: cell.nodeType === ElementNode_1$1._Node.TEXT_NODE
|
|
1360
|
-
? escapeMarkdownCharacters(cell.textContent?.trim() ?? '')
|
|
1361
|
-
: htmlToMarkdownAST(cell, options, indentLevel + 1),
|
|
1362
|
-
colId: colIds[columnIndex],
|
|
1363
|
-
colspan: colspan > 1 ? colspan : undefined,
|
|
1364
|
-
rowspan: rowspan > 1 ? rowspan : undefined
|
|
1365
|
-
};
|
|
1366
|
-
columnIndex += colspan;
|
|
1367
|
-
return cellNode;
|
|
1368
|
-
});
|
|
1369
|
-
return { type: 'tableRow', cells };
|
|
1370
|
-
});
|
|
1371
|
-
if (markdownTableRows.length > 0) {
|
|
1372
|
-
// Check if the first row contains header cells
|
|
1373
|
-
const hasHeaders = tableRows[0].querySelector('th') !== null;
|
|
1374
|
-
if (hasHeaders) {
|
|
1375
|
-
// Create a header separator row
|
|
1376
|
-
const headerSeparatorCells = Array.from(tableRows[0].querySelectorAll('th, td'))
|
|
1377
|
-
.map(() => ({
|
|
1378
|
-
type: 'tableCell',
|
|
1379
|
-
content: '---',
|
|
1380
|
-
colId: undefined,
|
|
1381
|
-
colspan: undefined,
|
|
1382
|
-
rowspan: undefined,
|
|
1383
|
-
}));
|
|
1384
|
-
const headerSeparatorRow = {
|
|
1385
|
-
type: 'tableRow',
|
|
1386
|
-
cells: headerSeparatorCells,
|
|
1387
|
-
};
|
|
1388
|
-
markdownTableRows.splice(1, 0, headerSeparatorRow);
|
|
1389
|
-
}
|
|
1390
|
-
}
|
|
1391
|
-
result.push({ type: 'table', rows: markdownTableRows, colIds });
|
|
1392
|
-
}
|
|
1393
|
-
else if (elem.tagName.toLowerCase() === 'head' && !!options?.includeMetaData) {
|
|
1394
|
-
const node = {
|
|
1395
|
-
type: 'meta',
|
|
1396
|
-
content: {
|
|
1397
|
-
standard: {},
|
|
1398
|
-
openGraph: {},
|
|
1399
|
-
twitter: {},
|
|
1400
|
-
}
|
|
1401
|
-
};
|
|
1402
|
-
elem.querySelectorAll('title')
|
|
1403
|
-
.forEach(titleElem => {
|
|
1404
|
-
node.content.standard['title'] = escapeMarkdownCharacters(titleElem.text);
|
|
1405
|
-
});
|
|
1406
|
-
// Extract meta tags
|
|
1407
|
-
const metaTags = elem.querySelectorAll('meta');
|
|
1408
|
-
const nonSemanticTagNames = [
|
|
1409
|
-
"viewport",
|
|
1410
|
-
"referrer",
|
|
1411
|
-
"Content-Security-Policy"
|
|
1412
|
-
];
|
|
1413
|
-
metaTags.forEach(metaTag => {
|
|
1414
|
-
const name = metaTag.getAttribute('name');
|
|
1415
|
-
const property = metaTag.getAttribute('property');
|
|
1416
|
-
const content = metaTag.getAttribute('content');
|
|
1417
|
-
if (property && property.startsWith('og:') && content) {
|
|
1418
|
-
if (options.includeMetaData === 'extended') {
|
|
1419
|
-
node.content.openGraph[property.substring(3)] = content;
|
|
1420
|
-
}
|
|
1421
|
-
}
|
|
1422
|
-
else if (name && name.startsWith('twitter:') && content) {
|
|
1423
|
-
if (options.includeMetaData === 'extended') {
|
|
1424
|
-
node.content.twitter[name.substring(8)] = content;
|
|
1425
|
-
}
|
|
1426
|
-
}
|
|
1427
|
-
else if (name && !nonSemanticTagNames.includes(name) && content) {
|
|
1428
|
-
node.content.standard[name] = content;
|
|
1429
|
-
}
|
|
1430
|
-
});
|
|
1431
|
-
// Extract JSON-LD data
|
|
1432
|
-
if (options.includeMetaData === 'extended') {
|
|
1433
|
-
const jsonLdData = [];
|
|
1434
|
-
const jsonLDScripts = elem.querySelectorAll('script[type="application/ld+json"]');
|
|
1435
|
-
jsonLDScripts.forEach(script => {
|
|
1436
|
-
try {
|
|
1437
|
-
const jsonContent = script.textContent;
|
|
1438
|
-
if (jsonContent) {
|
|
1439
|
-
const parsedData = JSON.parse(jsonContent);
|
|
1440
|
-
jsonLdData.push(parsedData);
|
|
1441
|
-
}
|
|
1442
|
-
}
|
|
1443
|
-
catch (error) {
|
|
1444
|
-
console.error('Failed to parse JSON-LD', error);
|
|
1445
|
-
}
|
|
1446
|
-
});
|
|
1447
|
-
node.content.jsonLd = jsonLdData;
|
|
1448
|
-
}
|
|
1449
|
-
result.push(node);
|
|
1450
|
-
}
|
|
1451
|
-
else {
|
|
1452
|
-
const content = escapeMarkdownCharacters(elem.textContent || '');
|
|
1453
|
-
switch (elem.tagName.toLowerCase()) {
|
|
1454
|
-
case 'noscript':
|
|
1455
|
-
case 'script':
|
|
1456
|
-
case 'style':
|
|
1457
|
-
case 'html':
|
|
1458
|
-
// blackhole..
|
|
1459
|
-
break;
|
|
1460
|
-
case 'strong':
|
|
1461
|
-
case 'b':
|
|
1462
|
-
if (content) {
|
|
1463
|
-
debugLog(`Bold: '${content}'`);
|
|
1464
|
-
result.push({
|
|
1465
|
-
type: 'bold',
|
|
1466
|
-
content: htmlToMarkdownAST(elem, options, indentLevel + 1)
|
|
1467
|
-
});
|
|
1468
|
-
}
|
|
1469
|
-
break;
|
|
1470
|
-
case 'em':
|
|
1471
|
-
case 'i':
|
|
1472
|
-
if (content) {
|
|
1473
|
-
debugLog(`Italic: '${content}'`);
|
|
1474
|
-
result.push({
|
|
1475
|
-
type: 'italic',
|
|
1476
|
-
content: htmlToMarkdownAST(elem, options, indentLevel + 1)
|
|
1477
|
-
});
|
|
1478
|
-
}
|
|
1479
|
-
break;
|
|
1480
|
-
case 's':
|
|
1481
|
-
case 'strike':
|
|
1482
|
-
if (content) {
|
|
1483
|
-
debugLog(`Strikethrough: '${content}'`);
|
|
1484
|
-
result.push({
|
|
1485
|
-
type: 'strikethrough',
|
|
1486
|
-
content: htmlToMarkdownAST(elem, options, indentLevel + 1)
|
|
1487
|
-
});
|
|
1488
|
-
}
|
|
1489
|
-
break;
|
|
1490
|
-
case 'code':
|
|
1491
|
-
if (content) {
|
|
1492
|
-
// Handling inline code differently
|
|
1493
|
-
const isCodeBlock = elem.parentNode && elem.parentNode.nodeName.toLowerCase() === 'pre';
|
|
1494
|
-
debugLog(`${isCodeBlock ? 'Code Block' : 'Inline Code'}: '${content}'`);
|
|
1495
|
-
const languageClass = elem.className?.split(" ").find(cls => cls.startsWith("language-"));
|
|
1496
|
-
const language = languageClass ? languageClass.replace("language-", "") : "";
|
|
1497
|
-
result.push({
|
|
1498
|
-
type: 'code',
|
|
1499
|
-
content: elem.textContent?.trim() ?? '',
|
|
1500
|
-
language,
|
|
1501
|
-
inline: !isCodeBlock
|
|
1502
|
-
});
|
|
1503
|
-
}
|
|
1504
|
-
break;
|
|
1505
|
-
case 'blockquote':
|
|
1506
|
-
debugLog(`Blockquote`);
|
|
1507
|
-
result.push({
|
|
1508
|
-
type: 'blockquote',
|
|
1509
|
-
content: htmlToMarkdownAST(elem, options)
|
|
1510
|
-
});
|
|
1511
|
-
break;
|
|
1512
|
-
case 'article':
|
|
1513
|
-
case 'aside':
|
|
1514
|
-
case 'details':
|
|
1515
|
-
case 'figcaption':
|
|
1516
|
-
case 'figure':
|
|
1517
|
-
case 'footer':
|
|
1518
|
-
case 'header':
|
|
1519
|
-
case 'main':
|
|
1520
|
-
case 'mark':
|
|
1521
|
-
case 'nav':
|
|
1522
|
-
case 'section':
|
|
1523
|
-
case 'summary':
|
|
1524
|
-
case 'time':
|
|
1525
|
-
debugLog(`Semantic HTML Element: '${elem.tagName}'`);
|
|
1526
|
-
result.push({
|
|
1527
|
-
type: 'semanticHtml',
|
|
1528
|
-
htmlType: elem.tagName.toLowerCase(),
|
|
1529
|
-
content: htmlToMarkdownAST(elem, options)
|
|
1530
|
-
});
|
|
1531
|
-
break;
|
|
1532
|
-
default:
|
|
1533
|
-
const unhandledElementProcessing = options?.processUnhandledElement?.(elem, options, indentLevel);
|
|
1534
|
-
if (unhandledElementProcessing) {
|
|
1535
|
-
debugLog(`Processing Unhandled Element: '${elem.tagName}'`);
|
|
1536
|
-
result.push(...unhandledElementProcessing);
|
|
1537
|
-
}
|
|
1538
|
-
else {
|
|
1539
|
-
debugLog(`Generic HTMLElement: '${elem.tagName}'`);
|
|
1540
|
-
result.push(...htmlToMarkdownAST(elem, options, indentLevel + 1));
|
|
1541
|
-
}
|
|
1542
|
-
break;
|
|
1543
|
-
}
|
|
1544
|
-
}
|
|
1545
|
-
}
|
|
1546
|
-
});
|
|
1547
|
-
return result;
|
|
1548
|
-
}
|
|
1549
|
-
function escapeMarkdownCharacters(text, isInlineCode = false) {
|
|
1550
|
-
if (isInlineCode || !text?.trim()) {
|
|
1551
|
-
// In inline code, we don't escape any characters
|
|
1552
|
-
return text;
|
|
1553
|
-
}
|
|
1554
|
-
// First, replace special HTML characters with their entity equivalents
|
|
1555
|
-
let escapedText = text.replace(/&/g, '&') // Replace & first
|
|
1556
|
-
.replace(/</g, '<')
|
|
1557
|
-
.replace(/>/g, '>');
|
|
1558
|
-
// Then escape characters that have special meaning in Markdown
|
|
1559
|
-
escapedText = escapedText.replace(/([\\`*_{}[\]#+!|])/g, '\\$1');
|
|
1560
|
-
return escapedText;
|
|
1561
|
-
}
|
|
1562
|
-
|
|
1563
|
-
var markdownASTToString = {};
|
|
1564
|
-
|
|
1565
|
-
var hasRequiredMarkdownASTToString;
|
|
1566
|
-
|
|
1567
|
-
function requireMarkdownASTToString () {
|
|
1568
|
-
if (hasRequiredMarkdownASTToString) return markdownASTToString;
|
|
1569
|
-
hasRequiredMarkdownASTToString = 1;
|
|
1570
|
-
Object.defineProperty(markdownASTToString, "__esModule", { value: true });
|
|
1571
|
-
markdownASTToString.markdownASTToString = markdownASTToString$1;
|
|
1572
|
-
const index_1 = requireNode();
|
|
1573
|
-
function markdownASTToString$1(nodes, options, indentLevel = 0) {
|
|
1574
|
-
let markdownString = '';
|
|
1575
|
-
markdownString += markdownMetaASTToString(nodes, options, indentLevel);
|
|
1576
|
-
markdownString += markdownContentASTToString(nodes, options, indentLevel);
|
|
1577
|
-
return markdownString;
|
|
1578
|
-
}
|
|
1579
|
-
function markdownMetaASTToString(nodes, options, indentLevel = 0) {
|
|
1580
|
-
let markdownString = '';
|
|
1581
|
-
if (options?.includeMetaData) {
|
|
1582
|
-
// include meta-data
|
|
1583
|
-
markdownString += '---\n';
|
|
1584
|
-
const node = (0, index_1.findInMarkdownAST)(nodes, _ => _.type === 'meta');
|
|
1585
|
-
if (node?.type === 'meta') {
|
|
1586
|
-
if (node.content.standard) {
|
|
1587
|
-
Object.keys(node.content.standard).forEach(key => {
|
|
1588
|
-
markdownString += `${key}: "${node.content.standard[key]}"\n`;
|
|
1589
|
-
});
|
|
1590
|
-
}
|
|
1591
|
-
if (options.includeMetaData === 'extended') {
|
|
1592
|
-
if (node.content.openGraph) {
|
|
1593
|
-
if (Object.keys(node.content.openGraph).length > 0) {
|
|
1594
|
-
markdownString += 'openGraph:\n';
|
|
1595
|
-
for (const [key, value] of Object.entries(node.content.openGraph)) {
|
|
1596
|
-
markdownString += ` ${key}: "${value}"\n`;
|
|
1597
|
-
}
|
|
1598
|
-
}
|
|
1599
|
-
}
|
|
1600
|
-
if (node.content.twitter) {
|
|
1601
|
-
if (Object.keys(node.content.twitter).length > 0) {
|
|
1602
|
-
markdownString += 'twitter:\n';
|
|
1603
|
-
for (const [key, value] of Object.entries(node.content.twitter)) {
|
|
1604
|
-
markdownString += ` ${key}: "${value}"\n`;
|
|
1605
|
-
}
|
|
1606
|
-
}
|
|
1607
|
-
}
|
|
1608
|
-
if (node.content.jsonLd && node.content.jsonLd.length > 0) {
|
|
1609
|
-
markdownString += 'schema:\n';
|
|
1610
|
-
node.content.jsonLd.forEach(item => {
|
|
1611
|
-
const { '@context': jldContext, '@type': jldType, ...semanticData } = item;
|
|
1612
|
-
markdownString += ` ${jldType ?? '(unknown type)'}:\n`;
|
|
1613
|
-
Object.keys(semanticData).forEach(key => {
|
|
1614
|
-
markdownString += ` ${key}: ${JSON.stringify(semanticData[key])}\n`;
|
|
1615
|
-
});
|
|
1616
|
-
});
|
|
1617
|
-
}
|
|
1618
|
-
}
|
|
1619
|
-
}
|
|
1620
|
-
markdownString += '---\n\n';
|
|
1621
|
-
}
|
|
1622
|
-
return markdownString;
|
|
1623
|
-
}
|
|
1624
|
-
function markdownContentASTToString(nodes, options, indentLevel = 0) {
|
|
1625
|
-
let markdownString = '';
|
|
1626
|
-
nodes.forEach((node) => {
|
|
1627
|
-
const indent = ' '.repeat(indentLevel * 2); // Adjust the multiplier for different indent sizes
|
|
1628
|
-
const nodeRenderingOverride = options?.overrideNodeRenderer?.(node, options, indentLevel);
|
|
1629
|
-
if (nodeRenderingOverride) {
|
|
1630
|
-
markdownString += nodeRenderingOverride;
|
|
1631
|
-
}
|
|
1632
|
-
else {
|
|
1633
|
-
switch (node.type) {
|
|
1634
|
-
case 'text':
|
|
1635
|
-
case 'bold':
|
|
1636
|
-
case 'italic':
|
|
1637
|
-
case 'strikethrough':
|
|
1638
|
-
case 'link':
|
|
1639
|
-
let content = node.content; // might be a nodes array but we take care of that below
|
|
1640
|
-
if (Array.isArray(node.content)) {
|
|
1641
|
-
content = markdownContentASTToString(node.content, options, indentLevel);
|
|
1642
|
-
}
|
|
1643
|
-
const isMarkdownStringNotEmpty = markdownString.length > 0;
|
|
1644
|
-
const isFirstCharOfContentWhitespace = /\s/.test(content.slice(0, 1));
|
|
1645
|
-
const isLastCharOfMarkdownWhitespace = /\s/.test(markdownString.slice(-1));
|
|
1646
|
-
const isContentPunctuation = content.length === 1 && /^[.,!?;:]/.test(content);
|
|
1647
|
-
if (isMarkdownStringNotEmpty && !isContentPunctuation && !isFirstCharOfContentWhitespace && !isLastCharOfMarkdownWhitespace) {
|
|
1648
|
-
markdownString += ' ';
|
|
1649
|
-
}
|
|
1650
|
-
if (node.type === 'text') {
|
|
1651
|
-
markdownString += `${indent}${content}`;
|
|
1652
|
-
}
|
|
1653
|
-
else {
|
|
1654
|
-
if (node.type === 'bold') {
|
|
1655
|
-
markdownString += `**${content}**`;
|
|
1656
|
-
}
|
|
1657
|
-
else if (node.type === 'italic') {
|
|
1658
|
-
markdownString += `*${content}*`;
|
|
1659
|
-
}
|
|
1660
|
-
else if (node.type === 'strikethrough') {
|
|
1661
|
-
markdownString += `~~${content}~~`;
|
|
1662
|
-
}
|
|
1663
|
-
else if (node.type === 'link') {
|
|
1664
|
-
// check if the link contains only text
|
|
1665
|
-
if (node.content.length === 1 && node.content[0].type === 'text') {
|
|
1666
|
-
// use native markdown syntax for text-only links
|
|
1667
|
-
markdownString += `[${content}](${encodeURI(node.href)})`;
|
|
1668
|
-
}
|
|
1669
|
-
else {
|
|
1670
|
-
// Use HTML <a> tag for links with rich content
|
|
1671
|
-
markdownString += `<a href="${node.href}">${content}</a>`;
|
|
1672
|
-
}
|
|
1673
|
-
}
|
|
1674
|
-
}
|
|
1675
|
-
break;
|
|
1676
|
-
case 'heading':
|
|
1677
|
-
const isEndsWithNewLine = markdownString.slice(-1) === '\n';
|
|
1678
|
-
if (!isEndsWithNewLine) {
|
|
1679
|
-
markdownString += '\n';
|
|
1680
|
-
}
|
|
1681
|
-
markdownString += `${'#'.repeat(node.level)} ${node.content}\n\n`;
|
|
1682
|
-
break;
|
|
1683
|
-
case 'image':
|
|
1684
|
-
if (!node.alt?.trim() || !!node.src?.trim()) {
|
|
1685
|
-
markdownString += ``;
|
|
1686
|
-
}
|
|
1687
|
-
break;
|
|
1688
|
-
case 'list':
|
|
1689
|
-
node.items.forEach((item, i) => {
|
|
1690
|
-
const listItemPrefix = node.ordered ? `${i + 1}.` : '-';
|
|
1691
|
-
const contents = markdownContentASTToString(item.content, options, indentLevel + 1).trim();
|
|
1692
|
-
if (markdownString.slice(-1) !== '\n') {
|
|
1693
|
-
markdownString += '\n';
|
|
1694
|
-
}
|
|
1695
|
-
if (contents) {
|
|
1696
|
-
markdownString += `${indent}${listItemPrefix} ${contents}\n`;
|
|
1697
|
-
}
|
|
1698
|
-
});
|
|
1699
|
-
markdownString += '\n';
|
|
1700
|
-
break;
|
|
1701
|
-
case 'video':
|
|
1702
|
-
markdownString += `\n\n`;
|
|
1703
|
-
if (node.poster) {
|
|
1704
|
-
markdownString += `\n`;
|
|
1705
|
-
}
|
|
1706
|
-
if (node.controls) {
|
|
1707
|
-
markdownString += `Controls: ${node.controls}\n`;
|
|
1708
|
-
}
|
|
1709
|
-
markdownString += '\n';
|
|
1710
|
-
break;
|
|
1711
|
-
case 'table':
|
|
1712
|
-
const maxColumns = Math.max(...node.rows.map(row => row.cells.reduce((sum, cell) => sum + (cell.colspan || 1), 0)));
|
|
1713
|
-
node.rows.forEach((row) => {
|
|
1714
|
-
let currentColumn = 0;
|
|
1715
|
-
row.cells.forEach((cell) => {
|
|
1716
|
-
let cellContent = typeof cell.content === 'string'
|
|
1717
|
-
? cell.content
|
|
1718
|
-
: markdownContentASTToString(cell.content, options, indentLevel + 1).trim();
|
|
1719
|
-
if (cell.colId) {
|
|
1720
|
-
cellContent += ` <!-- ${cell.colId} -->`;
|
|
1721
|
-
}
|
|
1722
|
-
if (cell.colspan && cell.colspan > 1) {
|
|
1723
|
-
cellContent += ` <!-- colspan: ${cell.colspan} -->`;
|
|
1724
|
-
}
|
|
1725
|
-
if (cell.rowspan && cell.rowspan > 1) {
|
|
1726
|
-
cellContent += ` <!-- rowspan: ${cell.rowspan} -->`;
|
|
1727
|
-
}
|
|
1728
|
-
markdownString += `| ${cellContent} `;
|
|
1729
|
-
currentColumn += cell.colspan || 1;
|
|
1730
|
-
// Add empty cells for colspan
|
|
1731
|
-
for (let i = 1; i < (cell.colspan || 1); i++) {
|
|
1732
|
-
markdownString += '| ';
|
|
1733
|
-
}
|
|
1734
|
-
});
|
|
1735
|
-
// Fill remaining columns with empty cells
|
|
1736
|
-
while (currentColumn < maxColumns) {
|
|
1737
|
-
markdownString += '| ';
|
|
1738
|
-
currentColumn++;
|
|
1739
|
-
}
|
|
1740
|
-
markdownString += '|\n';
|
|
1741
|
-
});
|
|
1742
|
-
markdownString += '\n';
|
|
1743
|
-
break;
|
|
1744
|
-
case 'code':
|
|
1745
|
-
if (node.inline) {
|
|
1746
|
-
const isLsatWhitespace = /\s/.test(markdownString.slice(-1));
|
|
1747
|
-
if (!isLsatWhitespace) {
|
|
1748
|
-
markdownString += ' ';
|
|
1749
|
-
}
|
|
1750
|
-
markdownString += `\`${node.content}\``;
|
|
1751
|
-
}
|
|
1752
|
-
else {
|
|
1753
|
-
// For code blocks, we do not escape characters and preserve formatting
|
|
1754
|
-
markdownString += '\n```' + (node.language ?? '') + '\n';
|
|
1755
|
-
markdownString += `${node.content}\n`;
|
|
1756
|
-
markdownString += '```\n\n';
|
|
1757
|
-
}
|
|
1758
|
-
break;
|
|
1759
|
-
case 'blockquote':
|
|
1760
|
-
markdownString += `> ${markdownContentASTToString(node.content, options).trim()}\n\n`;
|
|
1761
|
-
break;
|
|
1762
|
-
case "meta":
|
|
1763
|
-
// already handled
|
|
1764
|
-
break;
|
|
1765
|
-
case 'semanticHtml':
|
|
1766
|
-
switch (node.htmlType) {
|
|
1767
|
-
case "article":
|
|
1768
|
-
markdownString += '\n\n' + markdownContentASTToString(node.content, options);
|
|
1769
|
-
break;
|
|
1770
|
-
case "summary":
|
|
1771
|
-
case "time":
|
|
1772
|
-
case "aside":
|
|
1773
|
-
case "nav":
|
|
1774
|
-
case "figcaption":
|
|
1775
|
-
case "main":
|
|
1776
|
-
case "mark":
|
|
1777
|
-
case "header":
|
|
1778
|
-
case "footer":
|
|
1779
|
-
case "details":
|
|
1780
|
-
case "figure":
|
|
1781
|
-
markdownString += `\n\n<-${node.htmlType}->\n` + markdownContentASTToString(node.content, options) + `\n\n</-${node.htmlType}->\n`;
|
|
1782
|
-
break;
|
|
1783
|
-
case "section":
|
|
1784
|
-
markdownString += '---\n\n';
|
|
1785
|
-
markdownString += markdownContentASTToString(node.content, options);
|
|
1786
|
-
markdownString += '\n\n';
|
|
1787
|
-
markdownString += '---\n\n';
|
|
1788
|
-
break;
|
|
1789
|
-
}
|
|
1790
|
-
break;
|
|
1791
|
-
case "custom":
|
|
1792
|
-
const customNodeRendering = options?.renderCustomNode?.(node, options, indentLevel);
|
|
1793
|
-
if (customNodeRendering) {
|
|
1794
|
-
markdownString += customNodeRendering;
|
|
1795
|
-
}
|
|
1796
|
-
break;
|
|
1797
|
-
}
|
|
1798
|
-
}
|
|
1799
|
-
});
|
|
1800
|
-
return markdownString;
|
|
1801
|
-
}
|
|
1802
|
-
return markdownASTToString;
|
|
1803
|
-
}
|
|
1804
|
-
|
|
1805
|
-
var domUtils = {};
|
|
1806
|
-
|
|
1807
|
-
Object.defineProperty(domUtils, "__esModule", { value: true });
|
|
1808
|
-
domUtils.findMainContent = findMainContent;
|
|
1809
|
-
domUtils.wrapMainContent = wrapMainContent;
|
|
1810
|
-
domUtils.isElementVisible = isElementVisible;
|
|
1811
|
-
domUtils.getVisibleText = getVisibleText;
|
|
1812
|
-
const ElementNode_1 = ElementNode;
|
|
1813
|
-
const debugMessage = (message) => {
|
|
1814
|
-
};
|
|
1815
|
-
/**
|
|
1816
|
-
* Attempts to find the main content of a web page.
|
|
1817
|
-
* @param document The Document object to search.
|
|
1818
|
-
* @returns The Element containing the main content, or the body if no main content is found.
|
|
1819
|
-
*/
|
|
1820
|
-
function findMainContent(document) {
|
|
1821
|
-
const mainElement = document.querySelector('main');
|
|
1822
|
-
if (mainElement) {
|
|
1823
|
-
return mainElement;
|
|
1824
|
-
}
|
|
1825
|
-
if (!document.body) {
|
|
1826
|
-
return document.documentElement;
|
|
1827
|
-
}
|
|
1828
|
-
return detectMainContent(document.body);
|
|
1829
|
-
}
|
|
1830
|
-
function wrapMainContent(mainContentElement, document) {
|
|
1831
|
-
if (mainContentElement.tagName.toLowerCase() !== 'main') {
|
|
1832
|
-
const mainElement = document.createElement('main');
|
|
1833
|
-
mainContentElement.before(mainElement);
|
|
1834
|
-
mainElement.appendChild(mainContentElement);
|
|
1835
|
-
mainElement.id = 'detected-main-content';
|
|
1836
|
-
}
|
|
1837
|
-
}
|
|
1838
|
-
function detectMainContent(rootElement) {
|
|
1839
|
-
const candidates = [];
|
|
1840
|
-
const minScore = 20;
|
|
1841
|
-
collectCandidates(rootElement, candidates, minScore);
|
|
1842
|
-
if (candidates.length === 0) {
|
|
1843
|
-
return rootElement;
|
|
1844
|
-
}
|
|
1845
|
-
candidates.sort((a, b) => calculateScore(b) - calculateScore(a));
|
|
1846
|
-
let bestIndependentCandidate = candidates[0];
|
|
1847
|
-
for (let i = 1; i < candidates.length; i++) {
|
|
1848
|
-
if (!candidates.some((otherCandidate, j) => j !== i && otherCandidate.contains(candidates[i]))) {
|
|
1849
|
-
if (calculateScore(candidates[i]) > calculateScore(bestIndependentCandidate)) {
|
|
1850
|
-
bestIndependentCandidate = candidates[i];
|
|
1851
|
-
debugMessage(`New best independent candidate found: ${elementToString(bestIndependentCandidate)}`);
|
|
1852
|
-
}
|
|
1853
|
-
}
|
|
1854
|
-
}
|
|
1855
|
-
debugMessage(`Final main content candidate: ${elementToString(bestIndependentCandidate)}`);
|
|
1856
|
-
return bestIndependentCandidate;
|
|
1857
|
-
}
|
|
1858
|
-
function elementToString(element) {
|
|
1859
|
-
if (!element) {
|
|
1860
|
-
return 'No element';
|
|
1861
|
-
}
|
|
1862
|
-
return `${element.tagName}#${element.id || 'no-id'}.${Array.from(element.classList).join('.')}`;
|
|
1863
|
-
}
|
|
1864
|
-
function collectCandidates(element, candidates, minScore) {
|
|
1865
|
-
const score = calculateScore(element);
|
|
1866
|
-
if (score >= minScore) {
|
|
1867
|
-
candidates.push(element);
|
|
1868
|
-
debugMessage(`Candidate found: ${elementToString(element)}, score: ${score}`);
|
|
1869
|
-
}
|
|
1870
|
-
Array.from(element.children).forEach(child => {
|
|
1871
|
-
collectCandidates(child, candidates, minScore);
|
|
1872
|
-
});
|
|
1873
|
-
}
|
|
1874
|
-
function calculateScore(element) {
|
|
1875
|
-
let score = 0;
|
|
1876
|
-
let scoreLog = [];
|
|
1877
|
-
// High impact attributes
|
|
1878
|
-
const highImpactAttributes = ['article', 'content', 'main-container', 'main', 'main-content'];
|
|
1879
|
-
highImpactAttributes.forEach(attr => {
|
|
1880
|
-
if (element.classList.contains(attr) || element.id.includes(attr)) {
|
|
1881
|
-
score += 10;
|
|
1882
|
-
scoreLog.push(`High impact attribute found: ${attr}, score increased by 10`);
|
|
1883
|
-
}
|
|
1884
|
-
});
|
|
1885
|
-
// High impact tags
|
|
1886
|
-
const highImpactTags = ['article', 'main', 'section'];
|
|
1887
|
-
if (highImpactTags.includes(element.tagName.toLowerCase())) {
|
|
1888
|
-
score += 5;
|
|
1889
|
-
scoreLog.push(`High impact tag found: ${element.tagName}, score increased by 5`);
|
|
1890
|
-
}
|
|
1891
|
-
// Paragraph count
|
|
1892
|
-
const paragraphCount = element.getElementsByTagName('p').length;
|
|
1893
|
-
const paragraphScore = Math.min(paragraphCount, 5);
|
|
1894
|
-
if (paragraphScore > 0) {
|
|
1895
|
-
score += paragraphScore;
|
|
1896
|
-
scoreLog.push(`Paragraph count: ${paragraphCount}, score increased by ${paragraphScore}`);
|
|
1897
|
-
}
|
|
1898
|
-
// Text content length
|
|
1899
|
-
const textContentLength = element.textContent?.trim().length || 0;
|
|
1900
|
-
if (textContentLength > 200) {
|
|
1901
|
-
const textScore = Math.min(Math.floor(textContentLength / 200), 5);
|
|
1902
|
-
score += textScore;
|
|
1903
|
-
scoreLog.push(`Text content length: ${textContentLength}, score increased by ${textScore}`);
|
|
1904
|
-
}
|
|
1905
|
-
// Link density
|
|
1906
|
-
const linkDensity = calculateLinkDensity(element);
|
|
1907
|
-
if (linkDensity < 0.3) {
|
|
1908
|
-
score += 5;
|
|
1909
|
-
scoreLog.push(`Link density: ${linkDensity.toFixed(2)}, score increased by 5`);
|
|
1910
|
-
}
|
|
1911
|
-
// Data attributes
|
|
1912
|
-
if (element.hasAttribute('data-main') || element.hasAttribute('data-content')) {
|
|
1913
|
-
score += 10;
|
|
1914
|
-
scoreLog.push('Data attribute for main content found, score increased by 10');
|
|
1915
|
-
}
|
|
1916
|
-
// Role attribute
|
|
1917
|
-
if (element.getAttribute('role')?.includes('main')) {
|
|
1918
|
-
score += 10;
|
|
1919
|
-
scoreLog.push('Role attribute indicating main content found, score increased by 10');
|
|
1920
|
-
}
|
|
1921
|
-
if (scoreLog.length > 0) {
|
|
1922
|
-
debugMessage(`Scoring for ${elementToString(element)}:`);
|
|
1923
|
-
}
|
|
1924
|
-
return score;
|
|
1925
|
-
}
|
|
1926
|
-
function calculateLinkDensity(element) {
|
|
1927
|
-
const linkLength = Array.from(element.getElementsByTagName('a'))
|
|
1928
|
-
.reduce((sum, link) => sum + (link.textContent?.length || 0), 0);
|
|
1929
|
-
const textLength = element.textContent?.length || 1; // Avoid division by zero
|
|
1930
|
-
return linkLength / textLength;
|
|
1931
|
-
}
|
|
1932
|
-
function isElementVisible(element) {
|
|
1933
|
-
if (!(element instanceof HTMLElement)) {
|
|
1934
|
-
return true; // Non-HTMLElements are considered visible
|
|
1935
|
-
}
|
|
1936
|
-
const style = window.getComputedStyle(element);
|
|
1937
|
-
return style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0';
|
|
1938
|
-
}
|
|
1939
|
-
function getVisibleText(element) {
|
|
1940
|
-
if (!isElementVisible(element)) {
|
|
1941
|
-
return '';
|
|
1942
|
-
}
|
|
1943
|
-
let text = '';
|
|
1944
|
-
for (const child of Array.from(element.childNodes)) {
|
|
1945
|
-
if (child.nodeType === ElementNode_1._Node.TEXT_NODE) {
|
|
1946
|
-
text += child.textContent;
|
|
1947
|
-
}
|
|
1948
|
-
else if (child.nodeType === ElementNode_1._Node.ELEMENT_NODE) {
|
|
1949
|
-
text += getVisibleText(child);
|
|
1950
|
-
}
|
|
1951
|
-
}
|
|
1952
|
-
return text.trim();
|
|
1953
|
-
}
|
|
1954
|
-
|
|
1955
|
-
var urlUtils = {};
|
|
1956
|
-
|
|
1957
|
-
Object.defineProperty(urlUtils, "__esModule", { value: true });
|
|
1958
|
-
urlUtils.refifyUrls = refifyUrls;
|
|
1959
|
-
const mediaSuffixes = ["jpeg", "jpg", "png", "gif", "bmp", "tiff", "tif", "svg",
|
|
1960
|
-
"webp", "ico", "avi", "mov", "mp4", "mkv", "flv", "wmv", "webm", "mpeg",
|
|
1961
|
-
"mpg", "mp3", "wav", "aac", "ogg", "flac", "m4a", "pdf", "doc", "docx",
|
|
1962
|
-
"ppt", "pptx", "xls", "xlsx", "txt", "css", "js", "xml", "json",
|
|
1963
|
-
"html", "htm"
|
|
1964
|
-
];
|
|
1965
|
-
const addRefPrefix = (prefix, prefixesToRefs) => {
|
|
1966
|
-
if (!prefixesToRefs[prefix]) {
|
|
1967
|
-
prefixesToRefs[prefix] = 'ref' + Object.values(prefixesToRefs).length;
|
|
1968
|
-
}
|
|
1969
|
-
return prefixesToRefs[prefix];
|
|
1970
|
-
};
|
|
1971
|
-
const processUrl = (url, prefixesToRefs) => {
|
|
1972
|
-
if (!url.startsWith('http')) {
|
|
1973
|
-
return url;
|
|
1974
|
-
}
|
|
1975
|
-
else {
|
|
1976
|
-
const mediaSuffix = url.split('.').slice(-1)[0];
|
|
1977
|
-
if (mediaSuffix && mediaSuffixes.includes(mediaSuffix)) {
|
|
1978
|
-
const parts = url.split('/'); // Split URL keeping the slash before text
|
|
1979
|
-
const prefix = parts.slice(0, -1).join('/'); // Get the prefix by removing last part
|
|
1980
|
-
const refPrefix = addRefPrefix(prefix, prefixesToRefs);
|
|
1981
|
-
return `${refPrefix}://${parts.slice(-1).join('')}`;
|
|
1982
|
-
}
|
|
1983
|
-
else {
|
|
1984
|
-
if (url.split('/').length > 4) {
|
|
1985
|
-
return addRefPrefix(url, prefixesToRefs);
|
|
1986
|
-
}
|
|
1987
|
-
else {
|
|
1988
|
-
return url;
|
|
1989
|
-
}
|
|
1990
|
-
}
|
|
1991
|
-
}
|
|
1992
|
-
};
|
|
1993
|
-
function refifyUrls(markdownElement, prefixesToRefs = {}) {
|
|
1994
|
-
if (Array.isArray(markdownElement)) {
|
|
1995
|
-
markdownElement.forEach(element => refifyUrls(element, prefixesToRefs));
|
|
1996
|
-
}
|
|
1997
|
-
else {
|
|
1998
|
-
switch (markdownElement.type) {
|
|
1999
|
-
case 'link':
|
|
2000
|
-
markdownElement.href = processUrl(markdownElement.href, prefixesToRefs);
|
|
2001
|
-
refifyUrls(markdownElement.content, prefixesToRefs);
|
|
2002
|
-
break;
|
|
2003
|
-
case 'image':
|
|
2004
|
-
case 'video':
|
|
2005
|
-
markdownElement.src = processUrl(markdownElement.src, prefixesToRefs);
|
|
2006
|
-
break;
|
|
2007
|
-
case 'list':
|
|
2008
|
-
markdownElement.items.forEach(item => item.content.forEach(_ => refifyUrls(_, prefixesToRefs)));
|
|
2009
|
-
break;
|
|
2010
|
-
case 'table':
|
|
2011
|
-
markdownElement.rows.forEach(row => row.cells.forEach(cell => typeof cell.content === 'string' ? null : refifyUrls(cell.content, prefixesToRefs)));
|
|
2012
|
-
break;
|
|
2013
|
-
case 'blockquote':
|
|
2014
|
-
case 'semanticHtml':
|
|
2015
|
-
refifyUrls(markdownElement.content, prefixesToRefs);
|
|
2016
|
-
break;
|
|
2017
|
-
}
|
|
2018
|
-
}
|
|
2019
|
-
return prefixesToRefs;
|
|
2020
|
-
}
|
|
2021
|
-
|
|
2022
|
-
var astUtils = {};
|
|
2023
|
-
|
|
2024
|
-
(function (exports) {
|
|
2025
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
2026
|
-
exports.isNot = exports.getMainContent = void 0;
|
|
2027
|
-
exports.findInAST = findInAST;
|
|
2028
|
-
exports.findAllInAST = findAllInAST;
|
|
2029
|
-
const getMainContent = (markdownStr) => {
|
|
2030
|
-
if (markdownStr.includes('<-main->')) {
|
|
2031
|
-
const regex = /(?<=<-main->)[\s\S]*?(?=<\/-main->)/;
|
|
2032
|
-
const match = markdownStr.match(regex);
|
|
2033
|
-
return match?.[0] ?? '';
|
|
2034
|
-
}
|
|
2035
|
-
else {
|
|
2036
|
-
const removeSectionsRegex = /(<-nav->[\s\S]*?<\/-nav->)|(<-footer->[\s\S]*?<\/-footer->)|(<-header->[\s\S]*?<\/-header->)|(<-aside->[\s\S]*?<\/-aside->)/g;
|
|
2037
|
-
return markdownStr.replace(removeSectionsRegex, '');
|
|
2038
|
-
}
|
|
2039
|
-
};
|
|
2040
|
-
exports.getMainContent = getMainContent;
|
|
2041
|
-
const isNot = (tPred) => (t) => !tPred(t);
|
|
2042
|
-
exports.isNot = isNot;
|
|
2043
|
-
const isString = (x) => typeof x === "string";
|
|
2044
|
-
function findInAST(markdownElement, checker) {
|
|
2045
|
-
const loopCheck = (z) => {
|
|
2046
|
-
for (const element of z) {
|
|
2047
|
-
const found = findInAST(element, checker);
|
|
2048
|
-
if (found) {
|
|
2049
|
-
return found;
|
|
2050
|
-
}
|
|
2051
|
-
}
|
|
2052
|
-
return undefined;
|
|
2053
|
-
};
|
|
2054
|
-
if (Array.isArray(markdownElement)) {
|
|
2055
|
-
return loopCheck(markdownElement);
|
|
2056
|
-
}
|
|
2057
|
-
else {
|
|
2058
|
-
if (checker(markdownElement)) {
|
|
2059
|
-
return markdownElement;
|
|
2060
|
-
}
|
|
2061
|
-
switch (markdownElement.type) {
|
|
2062
|
-
case 'link':
|
|
2063
|
-
return loopCheck(markdownElement.content);
|
|
2064
|
-
case 'list':
|
|
2065
|
-
return loopCheck(markdownElement.items
|
|
2066
|
-
.map(_ => _.content)
|
|
2067
|
-
.flat());
|
|
2068
|
-
case 'table':
|
|
2069
|
-
return loopCheck(markdownElement.rows
|
|
2070
|
-
.map(row => row.cells.map(_ => _.content)
|
|
2071
|
-
.filter((0, exports.isNot)(isString)))
|
|
2072
|
-
.flat());
|
|
2073
|
-
case 'blockquote':
|
|
2074
|
-
case 'semanticHtml':
|
|
2075
|
-
return loopCheck(markdownElement.content);
|
|
2076
|
-
}
|
|
2077
|
-
return undefined;
|
|
2078
|
-
}
|
|
2079
|
-
}
|
|
2080
|
-
function findAllInAST(markdownElement, checker) {
|
|
2081
|
-
const loopCheck = (z) => {
|
|
2082
|
-
let out = [];
|
|
2083
|
-
for (const element of z) {
|
|
2084
|
-
const found = findAllInAST(element, checker);
|
|
2085
|
-
out = [...out, ...found];
|
|
2086
|
-
}
|
|
2087
|
-
return out;
|
|
2088
|
-
};
|
|
2089
|
-
if (Array.isArray(markdownElement)) {
|
|
2090
|
-
return loopCheck(markdownElement);
|
|
2091
|
-
}
|
|
2092
|
-
else {
|
|
2093
|
-
if (checker(markdownElement)) {
|
|
2094
|
-
return [markdownElement];
|
|
2095
|
-
}
|
|
2096
|
-
switch (markdownElement.type) {
|
|
2097
|
-
case 'link':
|
|
2098
|
-
return loopCheck(markdownElement.content);
|
|
2099
|
-
case 'list':
|
|
2100
|
-
return loopCheck(markdownElement.items
|
|
2101
|
-
.map(_ => _.content)
|
|
2102
|
-
.flat());
|
|
2103
|
-
case 'table':
|
|
2104
|
-
return loopCheck(markdownElement.rows
|
|
2105
|
-
.map(row => row.cells.map(_ => _.content)
|
|
2106
|
-
.filter((0, exports.isNot)(isString)))
|
|
2107
|
-
.flat());
|
|
2108
|
-
case 'blockquote':
|
|
2109
|
-
case 'semanticHtml':
|
|
2110
|
-
return loopCheck(markdownElement.content);
|
|
2111
|
-
}
|
|
2112
|
-
return [];
|
|
2113
|
-
}
|
|
2114
|
-
}
|
|
2115
|
-
} (astUtils));
|
|
2116
|
-
|
|
2117
|
-
var hasRequiredNode;
|
|
2118
|
-
|
|
2119
|
-
function requireNode () {
|
|
2120
|
-
if (hasRequiredNode) return node;
|
|
2121
|
-
hasRequiredNode = 1;
|
|
2122
|
-
(function (exports) {
|
|
2123
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
2124
|
-
exports.wrapMainContent = exports.refifyUrls = exports.findMainContent = exports.markdownASTToString = exports.htmlToMarkdownAST = void 0;
|
|
2125
|
-
exports.convertHtmlToMarkdown = convertHtmlToMarkdown;
|
|
2126
|
-
exports.convertElementToMarkdown = convertElementToMarkdown;
|
|
2127
|
-
exports.findInMarkdownAST = findInMarkdownAST;
|
|
2128
|
-
exports.findAllInMarkdownAST = findAllInMarkdownAST;
|
|
2129
|
-
const htmlToMarkdownAST_1 = htmlToMarkdownAST$1;
|
|
2130
|
-
Object.defineProperty(exports, "htmlToMarkdownAST", { enumerable: true, get: function () { return htmlToMarkdownAST_1.htmlToMarkdownAST; } });
|
|
2131
|
-
const markdownASTToString_1 = requireMarkdownASTToString();
|
|
2132
|
-
Object.defineProperty(exports, "markdownASTToString", { enumerable: true, get: function () { return markdownASTToString_1.markdownASTToString; } });
|
|
2133
|
-
const domUtils_1 = domUtils;
|
|
2134
|
-
Object.defineProperty(exports, "findMainContent", { enumerable: true, get: function () { return domUtils_1.findMainContent; } });
|
|
2135
|
-
Object.defineProperty(exports, "wrapMainContent", { enumerable: true, get: function () { return domUtils_1.wrapMainContent; } });
|
|
2136
|
-
const urlUtils_1 = urlUtils;
|
|
2137
|
-
Object.defineProperty(exports, "refifyUrls", { enumerable: true, get: function () { return urlUtils_1.refifyUrls; } });
|
|
2138
|
-
const astUtils_1 = astUtils;
|
|
2139
|
-
/**
|
|
2140
|
-
* Converts an HTML string to Markdown.
|
|
2141
|
-
* @param html The HTML string to convert.
|
|
2142
|
-
* @param options Conversion options.
|
|
2143
|
-
* @returns The converted Markdown string.
|
|
2144
|
-
*/
|
|
2145
|
-
function convertHtmlToMarkdown(html, options) {
|
|
2146
|
-
const parser = options?.overrideDOMParser ?? (typeof DOMParser !== 'undefined' ? new DOMParser() : null);
|
|
2147
|
-
if (!parser) {
|
|
2148
|
-
throw new Error('DOMParser is not available. Please provide an overrideDOMParser in options.');
|
|
2149
|
-
}
|
|
2150
|
-
const doc = parser.parseFromString(html, 'text/html');
|
|
2151
|
-
let element;
|
|
2152
|
-
if (options?.extractMainContent) {
|
|
2153
|
-
element = (0, domUtils_1.findMainContent)(doc);
|
|
2154
|
-
if (options.includeMetaData && !!doc.querySelector('head')?.innerHTML && !element.querySelector('head')) {
|
|
2155
|
-
// content container was found and extracted, re-attaching the head for meta-data extraction
|
|
2156
|
-
element = parser.parseFromString(`<html>${doc.head.outerHTML}${element.outerHTML}`, 'text/html').documentElement;
|
|
2157
|
-
}
|
|
2158
|
-
}
|
|
2159
|
-
else {
|
|
2160
|
-
// If there's a body, use it; otherwise, use the document element
|
|
2161
|
-
if (options?.includeMetaData && !!doc.querySelector('head')?.innerHTML) {
|
|
2162
|
-
element = doc.documentElement;
|
|
2163
|
-
}
|
|
2164
|
-
else {
|
|
2165
|
-
element = doc.body || doc.documentElement;
|
|
2166
|
-
}
|
|
2167
|
-
}
|
|
2168
|
-
return convertElementToMarkdown(element, options);
|
|
2169
|
-
}
|
|
2170
|
-
/**
|
|
2171
|
-
* Converts an HTML Element to Markdown.
|
|
2172
|
-
* @param element The HTML Element to convert.
|
|
2173
|
-
* @param options Conversion options.
|
|
2174
|
-
* @returns The converted Markdown string.
|
|
2175
|
-
*/
|
|
2176
|
-
function convertElementToMarkdown(element, options) {
|
|
2177
|
-
let ast = (0, htmlToMarkdownAST_1.htmlToMarkdownAST)(element, options);
|
|
2178
|
-
if (options?.refifyUrls) {
|
|
2179
|
-
options.urlMap = (0, urlUtils_1.refifyUrls)(ast);
|
|
2180
|
-
}
|
|
2181
|
-
return (0, markdownASTToString_1.markdownASTToString)(ast, options);
|
|
2182
|
-
}
|
|
2183
|
-
/**
|
|
2184
|
-
* Finds a node in the Markdown AST that matches the given predicate.
|
|
2185
|
-
* @param ast The Markdown AST to search.
|
|
2186
|
-
* @param predicate A function that returns true for the desired node.
|
|
2187
|
-
* @returns The first matching node, or undefined if not found.
|
|
2188
|
-
*/
|
|
2189
|
-
function findInMarkdownAST(ast, predicate) {
|
|
2190
|
-
return (0, astUtils_1.findInAST)(ast, predicate);
|
|
2191
|
-
}
|
|
2192
|
-
/**
|
|
2193
|
-
* Finds all nodes in the Markdown AST that match the given predicate.
|
|
2194
|
-
* @param ast The Markdown AST to search.
|
|
2195
|
-
* @param predicate A function that returns true for the desired nodes.
|
|
2196
|
-
* @returns An array of all matching nodes.
|
|
2197
|
-
*/
|
|
2198
|
-
function findAllInMarkdownAST(ast, predicate) {
|
|
2199
|
-
return (0, astUtils_1.findAllInAST)(ast, predicate);
|
|
2200
|
-
}
|
|
2201
|
-
} (node));
|
|
2202
|
-
return node;
|
|
2203
|
-
}
|
|
2204
|
-
|
|
2205
|
-
var nodeExports = requireNode();
|
|
2206
|
-
|
|
2207
|
-
//@ts-ignore
|
|
2208
|
-
window.__INTUNED__ = {
|
|
2209
|
-
matchStringsWithDomContent,
|
|
2210
|
-
convertElementToMarkdown,
|
|
2211
|
-
convertHtmlStringToSemanticMarkdown: nodeExports.convertHtmlToMarkdown,
|
|
2212
|
-
};
|
|
2213
|
-
|
|
1
|
+
(() => {
|
|
2
|
+
console.log("Hello");
|
|
2214
3
|
})();
|