@intuned/browser-dev 2.2.3-test-build.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/.babelrc +21 -0
  2. package/.eslintignore +10 -0
  3. package/.eslintrc.js +39 -0
  4. package/LICENSE +43 -0
  5. package/dist/ai/export.d.js +5 -0
  6. package/dist/ai/export.d.ts +641 -0
  7. package/dist/ai/extractStructuredData.js +320 -0
  8. package/dist/ai/extractStructuredDataUsingAi.js +139 -0
  9. package/dist/ai/extractionHelpers/screenshotHelpers.js +56 -0
  10. package/dist/ai/extractionHelpers/validateSchema.js +148 -0
  11. package/dist/ai/index.d.ts +641 -0
  12. package/dist/ai/index.js +19 -0
  13. package/dist/ai/isPageLoaded.js +77 -0
  14. package/dist/ai/prompt.js +39 -0
  15. package/dist/ai/tests/testCheckAllTypesAreStrings.spec.js +137 -0
  16. package/dist/ai/tests/testExtractFromContent.spec.js +372 -0
  17. package/dist/ai/tests/testExtractStructuredData.spec.js +646 -0
  18. package/dist/ai/tests/testIsPageLoaded.spec.js +277 -0
  19. package/dist/ai/tools/index.js +48 -0
  20. package/dist/ai/types/errors.js +67 -0
  21. package/dist/ai/types/models.js +45 -0
  22. package/dist/ai/types/types.js +48 -0
  23. package/dist/ai/validators.js +167 -0
  24. package/dist/common/Logger/index.js +60 -0
  25. package/dist/common/Logger/types.js +5 -0
  26. package/dist/common/SdkError.js +50 -0
  27. package/dist/common/aiModelsValidations.js +32 -0
  28. package/dist/common/browser_scripts.js +2596 -0
  29. package/dist/common/ensureBrowserScripts.js +18 -0
  30. package/dist/common/extendedTest.js +148 -0
  31. package/dist/common/extractionHelpers.js +19 -0
  32. package/dist/common/formatZodError.js +18 -0
  33. package/dist/common/fuzzySearch/fuzzySearch.test.js +250 -0
  34. package/dist/common/fuzzySearch/levenshtein-search.js +298 -0
  35. package/dist/common/fuzzySearch/utils.js +23 -0
  36. package/dist/common/getModelProvider.js +18 -0
  37. package/dist/common/getSimplifiedHtml.js +122 -0
  38. package/dist/common/hashObject.js +32 -0
  39. package/dist/common/html2markdown/convertElementToMarkdown.js +469 -0
  40. package/dist/common/html2markdown/index.js +19 -0
  41. package/dist/common/jwtTokenManager.js +18 -0
  42. package/dist/common/loadRuntime.js +16 -0
  43. package/dist/common/locatorHelpers.js +41 -0
  44. package/dist/common/matching/collectStrings.js +32 -0
  45. package/dist/common/matching/levenshtein.js +40 -0
  46. package/dist/common/matching/matching.js +317 -0
  47. package/dist/common/matching/types.js +1 -0
  48. package/dist/common/noEmpty.js +9 -0
  49. package/dist/common/saveSnapshotWithExamples.js +60 -0
  50. package/dist/common/tests/testEnsureBrowserScript.spec.js +31 -0
  51. package/dist/common/xpathMapping.js +107 -0
  52. package/dist/helpers/clickUntilExhausted.js +85 -0
  53. package/dist/helpers/downloadFile.js +125 -0
  54. package/dist/helpers/export.d.js +5 -0
  55. package/dist/helpers/export.d.ts +1220 -0
  56. package/dist/helpers/extractMarkdown.js +35 -0
  57. package/dist/helpers/filterEmptyValues.js +54 -0
  58. package/dist/helpers/gotoUrl.js +98 -0
  59. package/dist/helpers/index.d.ts +1220 -0
  60. package/dist/helpers/index.js +128 -0
  61. package/dist/helpers/processDate.js +25 -0
  62. package/dist/helpers/resolveUrl.js +64 -0
  63. package/dist/helpers/sanitizeHtml.js +74 -0
  64. package/dist/helpers/saveFileToS3.js +50 -0
  65. package/dist/helpers/scrollToLoadContent.js +57 -0
  66. package/dist/helpers/tests/extendedTest.js +130 -0
  67. package/dist/helpers/tests/testClickUntilExhausted.spec.js +387 -0
  68. package/dist/helpers/tests/testDownloadFile.spec.js +204 -0
  69. package/dist/helpers/tests/testExtractMarkdown.spec.js +290 -0
  70. package/dist/helpers/tests/testFilterEmptyValues.spec.js +151 -0
  71. package/dist/helpers/tests/testGoToUrl.spec.js +37 -0
  72. package/dist/helpers/tests/testProcessDate.spec.js +13 -0
  73. package/dist/helpers/tests/testResolveUrl.spec.js +341 -0
  74. package/dist/helpers/tests/testSanitizeHtml.spec.js +330 -0
  75. package/dist/helpers/tests/testScrollToLoadContent.spec.js +163 -0
  76. package/dist/helpers/tests/testValidateDataUsingSchema.spec.js +342 -0
  77. package/dist/helpers/tests/testWithDomSettledWait.spec.js +164 -0
  78. package/dist/helpers/tests/testWithNetworkIdleWait.spec.js +114 -0
  79. package/dist/helpers/types/Attachment.js +115 -0
  80. package/dist/helpers/types/CustomTypeRegistry.js +48 -0
  81. package/dist/helpers/types/RunEnvironment.js +18 -0
  82. package/dist/helpers/types/ValidationError.js +17 -0
  83. package/dist/helpers/types/index.js +51 -0
  84. package/dist/helpers/uploadFileToS3.js +154 -0
  85. package/dist/helpers/utils/getS3Client.js +22 -0
  86. package/dist/helpers/utils/index.js +73 -0
  87. package/dist/helpers/utils/isDownload.js +10 -0
  88. package/dist/helpers/utils/isGenerateCodeMode.js +9 -0
  89. package/dist/helpers/utils/isLocator.js +9 -0
  90. package/dist/helpers/utils/jwtTokenManager.js +18 -0
  91. package/dist/helpers/validateDataUsingSchema.js +103 -0
  92. package/dist/helpers/waitForDomSettled.js +90 -0
  93. package/dist/helpers/withNetworkSettledWait.js +91 -0
  94. package/dist/index.d.js +16 -0
  95. package/dist/index.d.ts +10 -0
  96. package/dist/index.js +16 -0
  97. package/dist/intunedServices/ApiGateway/aiApiGateway.js +99 -0
  98. package/dist/intunedServices/ApiGateway/factory.js +13 -0
  99. package/dist/intunedServices/ApiGateway/providers/Anthropic.js +26 -0
  100. package/dist/intunedServices/ApiGateway/providers/Gemini.js +29 -0
  101. package/dist/intunedServices/ApiGateway/providers/OpenAI.js +29 -0
  102. package/dist/intunedServices/ApiGateway/tests/testApiGateway.spec.js +224 -0
  103. package/dist/intunedServices/ApiGateway/types.js +11 -0
  104. package/dist/intunedServices/cache/cache.js +61 -0
  105. package/dist/intunedServices/cache/index.js +12 -0
  106. package/dist/intunedServices/cache/tests/testCache.spec.js +117 -0
  107. package/dist/optimized-extractors/common/buildExamplesPrompt.js +12 -0
  108. package/dist/optimized-extractors/common/buildImagesFromPage.js +55 -0
  109. package/dist/optimized-extractors/common/extractStructuredDataUsingClaude.js +135 -0
  110. package/dist/optimized-extractors/common/extractStructuredDataUsingGoogle.js +37 -0
  111. package/dist/optimized-extractors/common/extractStructuredDataUsingOpenAi.js +132 -0
  112. package/dist/optimized-extractors/common/extractStrucutredDataUsingAiInstance.js +122 -0
  113. package/dist/optimized-extractors/common/findTableHeaders.js +162 -0
  114. package/dist/optimized-extractors/common/index.js +55 -0
  115. package/dist/optimized-extractors/common/isTableHeaderOrFooter.js +84 -0
  116. package/dist/optimized-extractors/common/matching/matching.js +212 -0
  117. package/dist/optimized-extractors/common/matching/matching.test.js +655 -0
  118. package/dist/optimized-extractors/common/matching/types.js +18 -0
  119. package/dist/optimized-extractors/common/matching/utils.js +184 -0
  120. package/dist/optimized-extractors/common/utils.js +58 -0
  121. package/dist/optimized-extractors/export.d.js +5 -0
  122. package/dist/optimized-extractors/export.d.ts +397 -0
  123. package/dist/optimized-extractors/extractArray.js +120 -0
  124. package/dist/optimized-extractors/extractObject.js +104 -0
  125. package/dist/optimized-extractors/index.d.ts +397 -0
  126. package/dist/optimized-extractors/index.js +31 -0
  127. package/dist/optimized-extractors/listExtractionHelpers/__tests__/dynamicListExtractor.spec.js +312 -0
  128. package/dist/optimized-extractors/listExtractionHelpers/__tests__/findSetOfXpathsToCreateAnArrayExtractor.test.js +22 -0
  129. package/dist/optimized-extractors/listExtractionHelpers/__tests__/getContainerElement.test.js +21 -0
  130. package/dist/optimized-extractors/listExtractionHelpers/__tests__/partOfSameArrayXpath.test.js +42 -0
  131. package/dist/optimized-extractors/listExtractionHelpers/__tests__/verifyThatAllXpathsArePartOfSameArray.test.js +9 -0
  132. package/dist/optimized-extractors/listExtractionHelpers/dynamicListExtractor.js +152 -0
  133. package/dist/optimized-extractors/listExtractionHelpers/errors.js +46 -0
  134. package/dist/optimized-extractors/listExtractionHelpers/getListMatches.js +14 -0
  135. package/dist/optimized-extractors/listExtractionHelpers/runAiExtraction.js +240 -0
  136. package/dist/optimized-extractors/listExtractionHelpers/typesAndSchema.js +5 -0
  137. package/dist/optimized-extractors/listExtractionHelpers/utils/extractPropertiesUsingGPTFromArray.js +277 -0
  138. package/dist/optimized-extractors/listExtractionHelpers/utils/extractStructuredListUsingAi.js +44 -0
  139. package/dist/optimized-extractors/listExtractionHelpers/utils/getListContainerXpath.js +94 -0
  140. package/dist/optimized-extractors/listExtractionHelpers/utils/getRelativeContainerXpathSelector.js +20 -0
  141. package/dist/optimized-extractors/listExtractionHelpers/utils/getSimplifiedHtmlPerListItem.js +21 -0
  142. package/dist/optimized-extractors/listExtractionHelpers/utils/tablesUtils.js +48 -0
  143. package/dist/optimized-extractors/listExtractionHelpers/utils/validateOptions.js +52 -0
  144. package/dist/optimized-extractors/models/anthropicModel.js +23 -0
  145. package/dist/optimized-extractors/models/openaiModel.js +23 -0
  146. package/dist/optimized-extractors/objectExtractionHelpers/AIExtractors.js +73 -0
  147. package/dist/optimized-extractors/objectExtractionHelpers/__tests__/checksumUtils.test.js +103 -0
  148. package/dist/optimized-extractors/objectExtractionHelpers/__tests__/testObjectExtractorFromLocator.spec.js +107 -0
  149. package/dist/optimized-extractors/objectExtractionHelpers/__tests__/testObjectExtractorFromPage.spec.js +107 -0
  150. package/dist/optimized-extractors/objectExtractionHelpers/calculateObjectExampleHash.js +28 -0
  151. package/dist/optimized-extractors/objectExtractionHelpers/captureSnapshot.js +26 -0
  152. package/dist/optimized-extractors/objectExtractionHelpers/checksumUtils.js +32 -0
  153. package/dist/optimized-extractors/objectExtractionHelpers/constants.js +7 -0
  154. package/dist/optimized-extractors/objectExtractionHelpers/dynamicObjectExtractor.js +106 -0
  155. package/dist/optimized-extractors/objectExtractionHelpers/errors.js +42 -0
  156. package/dist/optimized-extractors/objectExtractionHelpers/findDomMatches.js +54 -0
  157. package/dist/optimized-extractors/objectExtractionHelpers/getSimplifiedHtml.js +122 -0
  158. package/dist/optimized-extractors/objectExtractionHelpers/typesAndSchemas.js +5 -0
  159. package/dist/optimized-extractors/objectExtractionHelpers/validateDynamicObjectExtractorOptions.js +52 -0
  160. package/dist/optimized-extractors/types/aiModelsValidation.js +45 -0
  161. package/dist/optimized-extractors/types/errors.js +42 -0
  162. package/dist/optimized-extractors/types/jsonSchema.d.js +5 -0
  163. package/dist/optimized-extractors/types/jsonSchema.d.ts +50 -0
  164. package/dist/optimized-extractors/types/types.js +5 -0
  165. package/dist/optimized-extractors/validators.js +152 -0
  166. package/dist/vite-env.d.js +1 -0
  167. package/dist/vite-env.d.ts +9 -0
  168. package/docs.md +14 -0
  169. package/generated-docs/ai/functions/extractStructuredData.mdx +255 -0
  170. package/generated-docs/ai/functions/isPageLoaded.mdx +88 -0
  171. package/generated-docs/ai/interfaces/ArraySchema.mdx +36 -0
  172. package/generated-docs/ai/interfaces/BasicSchema.mdx +14 -0
  173. package/generated-docs/ai/interfaces/BooleanSchema.mdx +28 -0
  174. package/generated-docs/ai/interfaces/ImageBufferContentItem.mdx +16 -0
  175. package/generated-docs/ai/interfaces/ImageUrlContentItem.mdx +16 -0
  176. package/generated-docs/ai/interfaces/NumberSchema.mdx +35 -0
  177. package/generated-docs/ai/interfaces/ObjectSchema.mdx +39 -0
  178. package/generated-docs/ai/interfaces/StringSchema.mdx +35 -0
  179. package/generated-docs/ai/interfaces/TextContentItem.mdx +14 -0
  180. package/generated-docs/ai/type-aliases/ContentItem.mdx +12 -0
  181. package/generated-docs/ai/type-aliases/JsonSchema.mdx +47 -0
  182. package/generated-docs/ai/type-aliases/SUPPORTED_MODELS.mdx +85 -0
  183. package/generated-docs/helpers/functions/downloadFile.mdx +99 -0
  184. package/generated-docs/helpers/functions/extractMarkdown.mdx +56 -0
  185. package/generated-docs/helpers/functions/filterEmptyValues.mdx +51 -0
  186. package/generated-docs/helpers/functions/goToUrl.mdx +124 -0
  187. package/generated-docs/helpers/functions/processDate.mdx +55 -0
  188. package/generated-docs/helpers/functions/resolveUrl.mdx +165 -0
  189. package/generated-docs/helpers/functions/sanitizeHtml.mdx +113 -0
  190. package/generated-docs/helpers/functions/saveFileToS3.mdx +127 -0
  191. package/generated-docs/helpers/functions/scrollToLoadContent.mdx +89 -0
  192. package/generated-docs/helpers/functions/uploadFileToS3.mdx +121 -0
  193. package/generated-docs/helpers/functions/validateDataUsingSchema.mdx +90 -0
  194. package/generated-docs/helpers/functions/waitForDomSettled.mdx +91 -0
  195. package/generated-docs/helpers/functions/withNetworkSettledWait.mdx +76 -0
  196. package/generated-docs/helpers/interfaces/Attachment.mdx +56 -0
  197. package/generated-docs/helpers/interfaces/S3Configs.mdx +52 -0
  198. package/generated-docs/helpers/interfaces/SanitizeHtmlOptions.mdx +22 -0
  199. package/generated-docs/helpers/type-aliases/AttachmentType.mdx +10 -0
  200. package/generated-docs/helpers/type-aliases/FileType.mdx +61 -0
  201. package/generated-docs/helpers/type-aliases/Trigger.mdx +62 -0
  202. package/how-to-run-tests.md +10 -0
  203. package/intuned-runtime-setup.md +13 -0
  204. package/package.json +119 -0
  205. package/tsconfig.eslint.json +5 -0
  206. package/tsconfig.json +26 -0
@@ -0,0 +1,2596 @@
1
+ (function () {
2
+ "use strict";
3
+
4
+ var MatchSource;
5
+ (function (MatchSource) {
6
+ MatchSource["ATTRIBUTE"] = "attribute";
7
+ MatchSource["TEXT_CONTENT"] = "text_content";
8
+ MatchSource["DIRECT_TEXT_NODE"] = "direct_text_node";
9
+ })(MatchSource || (MatchSource = {}));
10
+ var MatchMode;
11
+ (function (MatchMode) {
12
+ MatchMode["FULL"] = "full";
13
+ MatchMode["PARTIAL"] = "partial";
14
+ MatchMode["FUZZY"] = "fuzzy";
15
+ })(MatchMode || (MatchMode = {}));
16
+
17
+ function* searchExact(needle, haystack, startIndex = 0, endIndex = null) {
18
+ const needleLen = needle.length;
19
+ if (needleLen === 0) return;
20
+ if (endIndex === null) {
21
+ endIndex = haystack.length;
22
+ }
23
+ let index;
24
+ while ((index = haystack.indexOf(needle, startIndex)) > -1) {
25
+ if (index + needle.length > endIndex) break;
26
+ yield index;
27
+ startIndex = index + 1;
28
+ }
29
+ }
30
+ function reverse(string) {
31
+ return string.split("").reverse().join("");
32
+ }
33
+
34
+ function makeChar2needleIdx(needle, maxDist) {
35
+ const res = {};
36
+ for (let i = Math.min(needle.length - 1, maxDist); i >= 0; i--) {
37
+ res[needle[i]] = i;
38
+ }
39
+ return res;
40
+ }
41
+ function* fuzzySearch(needle, haystack, maxDist) {
42
+ if (needle.length > haystack.length + maxDist) return;
43
+ const ngramLen = Math.floor(needle.length / (maxDist + 1));
44
+ if (maxDist === 0) {
45
+ for (const index of searchExact(needle, haystack)) {
46
+ yield {
47
+ start: index,
48
+ end: index + needle.length,
49
+ dist: 0,
50
+ };
51
+ }
52
+ } else if (ngramLen >= 10) {
53
+ yield* fuzzySearchNgrams(needle, haystack, maxDist);
54
+ } else {
55
+ yield* fuzzySearchCandidates(needle, haystack, maxDist);
56
+ }
57
+ }
58
+ function _expand(needle, haystack, maxDist) {
59
+ maxDist = +maxDist;
60
+ let firstDiff;
61
+ for (
62
+ firstDiff = 0;
63
+ firstDiff < Math.min(needle.length, haystack.length);
64
+ firstDiff++
65
+ ) {
66
+ if (needle.charCodeAt(firstDiff) !== haystack.charCodeAt(firstDiff))
67
+ break;
68
+ }
69
+ if (firstDiff) {
70
+ needle = needle.slice(firstDiff);
71
+ haystack = haystack.slice(firstDiff);
72
+ }
73
+ if (!needle) {
74
+ return [0, firstDiff];
75
+ } else if (!haystack) {
76
+ if (needle.length <= maxDist) {
77
+ return [needle.length, firstDiff];
78
+ } else {
79
+ return [null, null];
80
+ }
81
+ }
82
+ if (maxDist === 0) return [null, null];
83
+ let scores = new Array(needle.length + 1);
84
+ for (let i = 0; i <= maxDist; i++) {
85
+ scores[i] = i;
86
+ }
87
+ let newScores = new Array(needle.length + 1);
88
+ let minScore = null;
89
+ let minScoreIdx = null;
90
+ let maxGoodScore = maxDist;
91
+ let firstGoodScoreIdx = 0;
92
+ let lastGoodScoreIdx = needle.length - 1;
93
+ for (let haystackIdx = 0; haystackIdx < haystack.length; haystackIdx++) {
94
+ const char = haystack.charCodeAt(haystackIdx);
95
+ const needleIdxStart = Math.max(0, firstGoodScoreIdx - 1);
96
+ const needleIdxLimit = Math.min(
97
+ haystackIdx + maxDist,
98
+ needle.length - 1,
99
+ lastGoodScoreIdx
100
+ );
101
+ newScores[0] = scores[0] + 1;
102
+ firstGoodScoreIdx = newScores[0] <= maxGoodScore ? 0 : null;
103
+ lastGoodScoreIdx = newScores[0] <= maxGoodScore ? 0 : -1;
104
+ let needleIdx;
105
+ for (
106
+ needleIdx = needleIdxStart;
107
+ needleIdx < needleIdxLimit;
108
+ needleIdx++
109
+ ) {
110
+ const score = (newScores[needleIdx + 1] = Math.min(
111
+ scores[needleIdx] + +(char !== needle.charCodeAt(needleIdx)),
112
+ scores[needleIdx + 1] + 1,
113
+ newScores[needleIdx] + 1
114
+ ));
115
+ if (score <= maxGoodScore) {
116
+ if (firstGoodScoreIdx === null) firstGoodScoreIdx = needleIdx + 1;
117
+ lastGoodScoreIdx = Math.max(
118
+ lastGoodScoreIdx,
119
+ needleIdx + 1 + (maxGoodScore - score)
120
+ );
121
+ }
122
+ }
123
+ const lastScore = (newScores[needleIdx + 1] = Math.min(
124
+ scores[needleIdx] + +(char !== needle.charCodeAt(needleIdx)),
125
+ newScores[needleIdx] + 1
126
+ ));
127
+ if (lastScore <= maxGoodScore) {
128
+ if (firstGoodScoreIdx === null) firstGoodScoreIdx = needleIdx + 1;
129
+ lastGoodScoreIdx = needleIdx + 1;
130
+ }
131
+ if (
132
+ needleIdx === needle.length - 1 &&
133
+ (minScore === null || lastScore <= minScore)
134
+ ) {
135
+ minScore = lastScore;
136
+ minScoreIdx = haystackIdx;
137
+ if (minScore < maxGoodScore) maxGoodScore = minScore;
138
+ }
139
+ [scores, newScores] = [newScores, scores];
140
+ if (firstGoodScoreIdx === null) break;
141
+ }
142
+ if (minScore !== null && minScore <= maxDist) {
143
+ return [minScore, minScoreIdx + 1 + firstDiff];
144
+ } else {
145
+ return [null, null];
146
+ }
147
+ }
148
+ function* fuzzySearchNgrams(needle, haystack, maxDist) {
149
+ // use n-gram search
150
+ const ngramLen = Math.floor(needle.length / (maxDist + 1));
151
+ const needleLen = needle.length;
152
+ const haystackLen = haystack.length;
153
+ for (
154
+ let ngramStartIdx = 0;
155
+ ngramStartIdx <= needle.length - ngramLen;
156
+ ngramStartIdx += ngramLen
157
+ ) {
158
+ const ngram = needle.slice(ngramStartIdx, ngramStartIdx + ngramLen);
159
+ const ngramEnd = ngramStartIdx + ngramLen;
160
+ const needleBeforeReversed = reverse(needle.slice(0, ngramStartIdx));
161
+ const needleAfter = needle.slice(ngramEnd);
162
+ const startIdx = Math.max(0, ngramStartIdx - maxDist);
163
+ const endIdx = Math.min(
164
+ haystackLen,
165
+ haystackLen - needleLen + ngramEnd + maxDist
166
+ );
167
+ for (const haystackMatchIdx of searchExact(
168
+ ngram,
169
+ haystack,
170
+ startIdx,
171
+ endIdx
172
+ )) {
173
+ // try to expand left
174
+ const [distRight, rightExpandSize] = _expand(
175
+ needleAfter,
176
+ haystack.slice(
177
+ haystackMatchIdx + ngramLen,
178
+ haystackMatchIdx - ngramStartIdx + needleLen + maxDist
179
+ ),
180
+ maxDist
181
+ );
182
+ if (distRight === null) continue;
183
+ const [distLeft, leftExpandSize] = _expand(
184
+ needleBeforeReversed,
185
+ reverse(
186
+ haystack.slice(
187
+ Math.max(
188
+ 0,
189
+ haystackMatchIdx - ngramStartIdx - (maxDist - distRight)
190
+ ),
191
+ haystackMatchIdx
192
+ )
193
+ ),
194
+ maxDist - distRight
195
+ );
196
+ if (distLeft === null) continue;
197
+ yield {
198
+ start: haystackMatchIdx - leftExpandSize,
199
+ end: haystackMatchIdx + ngramLen + rightExpandSize,
200
+ dist: distLeft + distRight,
201
+ };
202
+ }
203
+ }
204
+ }
205
+ function* fuzzySearchCandidates(needle, haystack, maxDist) {
206
+ const needleLen = needle.length;
207
+ const haystackLen = haystack.length;
208
+ if (needleLen > haystackLen + maxDist) return;
209
+ const char2needleIdx = makeChar2needleIdx(needle, maxDist);
210
+ let prevCandidates = new Map(); // candidates from the last iteration
211
+ let candidates = new Map(); // new candidates from the current iteration
212
+ // iterate over the chars in the haystack, updating the candidates for each
213
+ for (let i = 0; i < haystack.length; i++) {
214
+ const haystackChar = haystack[i];
215
+ prevCandidates = candidates;
216
+ candidates = new Map();
217
+ const needleIdx = char2needleIdx[haystackChar];
218
+ if (needleIdx !== undefined) {
219
+ if (needleIdx + 1 === needleLen) {
220
+ yield {
221
+ start: i,
222
+ end: i + 1,
223
+ dist: needleIdx,
224
+ };
225
+ } else {
226
+ candidates.set(`${i},${needleIdx + 1},${needleIdx}`, {
227
+ startIdx: i,
228
+ needleIdx: needleIdx + 1,
229
+ dist: needleIdx,
230
+ });
231
+ }
232
+ }
233
+ for (const [, candidate] of prevCandidates) {
234
+ // if this sequence char is the candidate's next expected char
235
+ if (needle[candidate.needleIdx] === haystackChar) {
236
+ // if reached the end of the needle, return a match
237
+ if (candidate.needleIdx + 1 === needleLen) {
238
+ yield {
239
+ start: candidate.startIdx,
240
+ end: i + 1,
241
+ dist: candidate.dist,
242
+ };
243
+ } else {
244
+ // otherwise, update the candidate's needleIdx and keep it
245
+ candidates.set(
246
+ `${candidate.startIdx},${candidate.needleIdx + 1},${
247
+ candidate.dist
248
+ }`,
249
+ {
250
+ startIdx: candidate.startIdx,
251
+ needleIdx: candidate.needleIdx + 1,
252
+ dist: candidate.dist,
253
+ }
254
+ );
255
+ }
256
+ } else {
257
+ if (candidate.dist === maxDist) continue;
258
+ candidates.set(
259
+ `${candidate.startIdx},${candidate.needleIdx},${
260
+ candidate.dist + 1
261
+ }`,
262
+ {
263
+ startIdx: candidate.startIdx,
264
+ needleIdx: candidate.needleIdx,
265
+ dist: candidate.dist + 1,
266
+ }
267
+ );
268
+ for (
269
+ let nSkipped = 1;
270
+ nSkipped <= maxDist - candidate.dist;
271
+ nSkipped++
272
+ ) {
273
+ if (candidate.needleIdx + nSkipped === needleLen) {
274
+ yield {
275
+ start: candidate.startIdx,
276
+ end: i + 1,
277
+ dist: candidate.dist + nSkipped,
278
+ };
279
+ break;
280
+ } else if (
281
+ needle[candidate.needleIdx + nSkipped] === haystackChar
282
+ ) {
283
+ if (candidate.needleIdx + nSkipped + 1 === needleLen) {
284
+ yield {
285
+ start: candidate.startIdx,
286
+ end: i + 1,
287
+ dist: candidate.dist + nSkipped,
288
+ };
289
+ } else {
290
+ candidates.set(
291
+ `${candidate.startIdx},${
292
+ candidate.needleIdx + 1 + nSkipped
293
+ },${candidate.dist + nSkipped}`,
294
+ {
295
+ startIdx: candidate.startIdx,
296
+ needleIdx: candidate.needleIdx + 1 + nSkipped,
297
+ dist: candidate.dist + nSkipped,
298
+ }
299
+ );
300
+ }
301
+ break;
302
+ }
303
+ }
304
+ if (i + 1 < haystackLen && candidate.needleIdx + 1 < needleLen) {
305
+ candidates.set(
306
+ `${candidate.startIdx},${candidate.needleIdx + 1},${
307
+ candidate.dist + 1
308
+ }`,
309
+ {
310
+ startIdx: candidate.startIdx,
311
+ needleIdx: candidate.needleIdx + 1,
312
+ dist: candidate.dist + 1,
313
+ }
314
+ );
315
+ }
316
+ }
317
+ }
318
+ }
319
+ for (const [, candidate] of candidates) {
320
+ candidate.dist += needle.length - candidate.needleIdx;
321
+ if (candidate.dist <= maxDist) {
322
+ yield {
323
+ start: candidate.startIdx,
324
+ end: haystack.length,
325
+ dist: candidate.dist,
326
+ };
327
+ }
328
+ }
329
+ }
330
+
331
+ function findClosestMatch(searchTerm, content, maxLDist) {
332
+ const results = [];
333
+ for (const result of fuzzySearch(searchTerm, content, maxLDist)) {
334
+ results.push(result);
335
+ }
336
+ results.sort((a, b) => {
337
+ if (a.dist === b.dist) {
338
+ return b.end - b.start - (a.end - a.start); // Sort by match length if distances are equal
339
+ }
340
+ return a.dist - b.dist; // Sort by distance
341
+ });
342
+ return results[0];
343
+ }
344
+ function normalizeSpacing(text) {
345
+ if (!text) {
346
+ return "";
347
+ }
348
+ // Replace newlines and tabs with spaces
349
+ let normalized = text.replace(/\n/g, " ").replace(/\t/g, " ");
350
+ // Replace multiple spaces with a single space
351
+ normalized = normalized.split(/\s+/).join(" ");
352
+ return normalized.trim();
353
+ }
354
+ function isMatchExact(data, value) {
355
+ if (!data || !value) {
356
+ return [false, null];
357
+ }
358
+ const normalizedData = normalizeSpacing(data);
359
+ const normalizedValue = normalizeSpacing(value);
360
+ return [normalizedData === normalizedValue, normalizedValue];
361
+ }
362
+ function calculateMaxLDist(value) {
363
+ const length = value.length;
364
+ const Pmax = 0.2;
365
+ const Pmin = 0.05;
366
+ const lengthAtPmax = 10;
367
+ let percentage;
368
+ if (length <= lengthAtPmax) {
369
+ percentage = Pmax;
370
+ } else {
371
+ const k = -Math.log(Pmin / Pmax) / (600 - lengthAtPmax);
372
+ percentage = Pmax * Math.exp(-k * (length - lengthAtPmax));
373
+ }
374
+ percentage = Math.max(Pmin, percentage);
375
+ return Math.max(1, Math.floor(length * percentage));
376
+ }
377
+ function isFuzzMatch(searchTerm, content) {
378
+ if (!searchTerm || !content) {
379
+ return {
380
+ found: false,
381
+ matchedValue: null,
382
+ distance: null,
383
+ matchedSourceValue: null,
384
+ };
385
+ }
386
+ const maxLDist = calculateMaxLDist(searchTerm);
387
+ const normalizedSearchTerm = normalizeSpacing(searchTerm);
388
+ const normalizedContent = normalizeSpacing(content);
389
+ const match = findClosestMatch(
390
+ normalizedSearchTerm.toLowerCase(),
391
+ normalizedContent.toLowerCase(),
392
+ maxLDist
393
+ );
394
+ if (!match) {
395
+ return {
396
+ found: false,
397
+ matchedValue: null,
398
+ distance: null,
399
+ matchedSourceValue: null,
400
+ };
401
+ }
402
+ return {
403
+ found: true,
404
+ matchedValue: normalizedContent.slice(match.start, match.end),
405
+ matchedSourceValue: normalizedContent,
406
+ distance: match.dist,
407
+ };
408
+ }
409
+ function hasNonFuzzyOrCloseFuzzyMatch(matches) {
410
+ const hasNonFuzzyMatch = matches.some(
411
+ (match) => match.match_mode !== MatchMode.FUZZY
412
+ );
413
+ const hasVeryCloseFuzzyMatch = matches.some(
414
+ (match) =>
415
+ match.match_mode === MatchMode.FUZZY &&
416
+ match.fuzzy_distance &&
417
+ match.fuzzy_distance < 5
418
+ );
419
+ return hasNonFuzzyMatch || hasVeryCloseFuzzyMatch;
420
+ }
421
+ function getElementXPath(element) {
422
+ if (!element || !element.parentNode || element.nodeName === "#document") {
423
+ return null;
424
+ }
425
+ let siblingsCount = 1;
426
+ const parent = element.parentNode;
427
+ const nodeName = element.nodeName.toLowerCase();
428
+ const siblings = Array.from(parent.childNodes).filter(
429
+ (node) => node.nodeType === 1 // Node.ELEMENT_NODE
430
+ );
431
+ for (const sibling of siblings) {
432
+ if (sibling === element) {
433
+ break;
434
+ }
435
+ if (sibling.nodeName.toLowerCase() === nodeName) {
436
+ siblingsCount++;
437
+ }
438
+ }
439
+ const parentXPath = getElementXPath(parent);
440
+ if (element.nodeName === "#text") {
441
+ return parentXPath;
442
+ }
443
+
444
+ let nodeXPath;
445
+ if (
446
+ element.namespaceURI &&
447
+ element.namespaceURI !== "http://www.w3.org/1999/xhtml" // HTML namespace, this will make xpath locator succeed without [name()='']
448
+ ) {
449
+ // Element is in a namespace (SVG, MathML, or custom namespace)
450
+ nodeXPath = `*[name()='${nodeName}']`;
451
+ } else {
452
+ // Standard HTML element
453
+ nodeXPath = `${nodeName}[${siblingsCount}]`;
454
+ }
455
+
456
+ return parentXPath ? `${parentXPath}/${nodeXPath}` : nodeXPath;
457
+ }
458
+ function traverseAndPrune(node, conditionFunc) {
459
+ const children = Array.from(node.children ?? []);
460
+ children.forEach((child) => {
461
+ if (child.children) {
462
+ if (!conditionFunc(child)) {
463
+ traverseAndPrune(child, conditionFunc);
464
+ }
465
+ }
466
+ });
467
+ }
468
+ function isPartOfString(input, dom) {
469
+ if (!input || !dom) {
470
+ return [false, null, null];
471
+ }
472
+ const normalizedInput = normalizeSpacing(input);
473
+ const normalizedDom = normalizeSpacing(dom);
474
+ const matchIndex = normalizedDom
475
+ .toLowerCase()
476
+ .indexOf(normalizedInput.toLowerCase());
477
+ const matchedText =
478
+ matchIndex !== -1
479
+ ? normalizedDom.substring(
480
+ matchIndex,
481
+ matchIndex + normalizedInput.length
482
+ )
483
+ : null;
484
+ return [matchIndex !== -1, matchedText, normalizedDom];
485
+ }
486
+
487
+ function matchStringsWithDomContent(domNode, stringsList) {
488
+ const exactMatchedMap = matchExactStrings(domNode, stringsList);
489
+ const stringsWithNoExactMatch = stringsList.filter(
490
+ (data) => !hasNonFuzzyOrCloseFuzzyMatch(exactMatchedMap[data])
491
+ );
492
+ if (stringsWithNoExactMatch.length === 0) {
493
+ return exactMatchedMap;
494
+ }
495
+ const fuzzMatchedMap = matchFuzzyStrings(domNode, stringsWithNoExactMatch);
496
+ for (const [data, fuzzyMatches] of Object.entries(fuzzMatchedMap)) {
497
+ if (data in exactMatchedMap) {
498
+ exactMatchedMap[data].push(...fuzzyMatches);
499
+ } else {
500
+ exactMatchedMap[data] = fuzzyMatches;
501
+ }
502
+ }
503
+ // attributes to try fuzzy match attributes on
504
+ const stringsWithNoMatch = stringsList.filter(
505
+ (data) => !hasNonFuzzyOrCloseFuzzyMatch(exactMatchedMap[data])
506
+ );
507
+ const attributesFuzzyMatchedMap = matchFuzzyAttributes(
508
+ domNode,
509
+ stringsWithNoMatch
510
+ );
511
+ for (const [data, attributeFuzzyMatches] of Object.entries(
512
+ attributesFuzzyMatchedMap
513
+ )) {
514
+ if (data in exactMatchedMap) {
515
+ exactMatchedMap[data].push(...attributeFuzzyMatches);
516
+ } else {
517
+ exactMatchedMap[data] = attributeFuzzyMatches;
518
+ }
519
+ }
520
+ return exactMatchedMap;
521
+ }
522
+ function matchExactStrings(domNode, stringsList) {
523
+ const allNodes = [
524
+ domNode,
525
+ ...Array.from(domNode.querySelectorAll("*")),
526
+ ].reverse();
527
+ const matchesMap = Object.fromEntries(
528
+ stringsList.map((data) => [data, []])
529
+ );
530
+ for (const tag of allNodes) {
531
+ const xpath = getElementXPath(tag);
532
+ for (const stringValue of stringsList) {
533
+ const matchesXPaths = matchesMap[stringValue].map(
534
+ (match) => match.xpath || ""
535
+ );
536
+ const xpathIsChildOfMatch = matchesXPaths.some(
537
+ (matchXPath) => matchXPath !== xpath && matchXPath.startsWith(xpath)
538
+ );
539
+ if (xpathIsChildOfMatch) continue;
540
+ const attributeNames = tag.getAttributeNames();
541
+ for (const attr of attributeNames) {
542
+ const attributeValue = tag.getAttribute(attr) || "";
543
+ const [isPartOfStringResult, matchedValue] = isPartOfString(
544
+ stringValue,
545
+ attributeValue
546
+ );
547
+ if (isPartOfStringResult) {
548
+ const [isExact] = isMatchExact(stringValue, attributeValue);
549
+ matchesMap[stringValue].push({
550
+ attribute: attr,
551
+ fuzzy_distance: null,
552
+ match_mode: isExact ? MatchMode.FULL : MatchMode.PARTIAL,
553
+ match_source: MatchSource.ATTRIBUTE,
554
+ matched_value: matchedValue,
555
+ matched_source_value: attributeValue,
556
+ tag: tag.tagName.toLowerCase(),
557
+ xpath,
558
+ });
559
+ }
560
+ }
561
+ if (tag["href"]) {
562
+ const result = matchHref(tag, stringValue);
563
+ if (result) {
564
+ matchesMap[stringValue].push(result);
565
+ }
566
+ }
567
+ // Check for direct text nodes
568
+ for (const childNode of tag.childNodes) {
569
+ // Node.TEXT_NODE
570
+ if (childNode.nodeType === 3) {
571
+ const directTextContent = childNode.textContent?.trim() || "";
572
+ if (directTextContent) {
573
+ const [isPartOfStringResult, matchedValue, source_value] =
574
+ isPartOfString(stringValue, directTextContent);
575
+ if (isPartOfStringResult) {
576
+ const [isExact] = isMatchExact(stringValue, directTextContent);
577
+ matchesMap[stringValue].push({
578
+ attribute: null,
579
+ fuzzy_distance: null,
580
+ match_mode: isExact ? MatchMode.FULL : MatchMode.PARTIAL,
581
+ match_source: MatchSource.DIRECT_TEXT_NODE,
582
+ matched_value: matchedValue,
583
+ matched_source_value: source_value,
584
+ tag: tag.tagName.toLowerCase(),
585
+ xpath,
586
+ });
587
+ }
588
+ }
589
+ }
590
+ }
591
+ const tagTextContent = tag.textContent || "";
592
+ const [isPartOfStringResult, matchedValue, source_value] =
593
+ isPartOfString(stringValue, tagTextContent);
594
+ if (isPartOfStringResult) {
595
+ const [isExact] = isMatchExact(stringValue, tagTextContent);
596
+ matchesMap[stringValue].push({
597
+ attribute: null,
598
+ fuzzy_distance: null,
599
+ match_mode: isExact ? MatchMode.FULL : MatchMode.PARTIAL,
600
+ match_source: MatchSource.TEXT_CONTENT,
601
+ matched_value: matchedValue,
602
+ matched_source_value: source_value,
603
+ tag: tag.tagName.toLowerCase(),
604
+ xpath,
605
+ });
606
+ }
607
+ }
608
+ }
609
+ return matchesMap;
610
+ }
611
+ function matchFuzzyStrings(domNode, stringsToMatch) {
612
+ const matchesMap = Object.fromEntries(
613
+ stringsToMatch.map((data) => [data, []])
614
+ );
615
+ const conditionFunc = (stringToMatch, node) => {
616
+ let foundMatch = false;
617
+ const currentXPath = getElementXPath(node);
618
+ for (const attr of node.getAttributeNames()) {
619
+ const attributeValue = node.getAttribute(attr) || "";
620
+ const {
621
+ found: isFuzzMatchFound,
622
+ matchedValue,
623
+ distance: dist,
624
+ matchedSourceValue,
625
+ } = isFuzzMatch(stringToMatch, attributeValue);
626
+ if (isFuzzMatchFound) {
627
+ matchesMap[stringToMatch].push({
628
+ attribute: attr,
629
+ fuzzy_distance: dist,
630
+ match_mode: MatchMode.FUZZY,
631
+ match_source: MatchSource.ATTRIBUTE,
632
+ matched_value: matchedValue,
633
+ tag: node.tagName.toLowerCase(),
634
+ xpath: currentXPath,
635
+ matched_source_value: matchedSourceValue,
636
+ });
637
+ foundMatch = true;
638
+ }
639
+ }
640
+ const tagTextContent = node.textContent || "";
641
+ if (tagTextContent) {
642
+ const {
643
+ found: isFuzzMatchFound,
644
+ matchedValue,
645
+ distance: dist,
646
+ matchedSourceValue,
647
+ } = isFuzzMatch(stringToMatch, tagTextContent);
648
+ if (isFuzzMatchFound) {
649
+ matchesMap[stringToMatch].push({
650
+ attribute: null,
651
+ fuzzy_distance: dist,
652
+ match_mode: MatchMode.FUZZY,
653
+ match_source: MatchSource.TEXT_CONTENT,
654
+ matched_value: matchedValue,
655
+ tag: node.tagName.toLowerCase(),
656
+ xpath: currentXPath,
657
+ matched_source_value: matchedSourceValue,
658
+ });
659
+ foundMatch = true;
660
+ }
661
+ }
662
+ // Check for direct text nodes
663
+ for (const childNode of node.childNodes) {
664
+ // Node.TEXT_NODE
665
+ if (childNode.nodeType === 3) {
666
+ const directTextContent = childNode.textContent?.trim() || "";
667
+ if (directTextContent) {
668
+ const {
669
+ found: isFuzzMatchFound,
670
+ matchedValue,
671
+ distance: dist,
672
+ matchedSourceValue,
673
+ } = isFuzzMatch(stringToMatch, directTextContent);
674
+ if (isFuzzMatchFound) {
675
+ matchesMap[stringToMatch].push({
676
+ attribute: null,
677
+ fuzzy_distance: dist,
678
+ match_mode: MatchMode.FUZZY,
679
+ match_source: MatchSource.DIRECT_TEXT_NODE,
680
+ matched_value: matchedValue,
681
+ tag: node.tagName.toLowerCase(),
682
+ xpath: currentXPath,
683
+ matched_source_value: matchedSourceValue,
684
+ });
685
+ foundMatch = true;
686
+ }
687
+ }
688
+ }
689
+ }
690
+ return !foundMatch;
691
+ };
692
+ for (const stringToMatch of stringsToMatch) {
693
+ conditionFunc(stringToMatch, domNode);
694
+ traverseAndPrune(domNode, (node) => conditionFunc(stringToMatch, node));
695
+ }
696
+ for (const [stringToMatch, matches] of Object.entries(matchesMap)) {
697
+ const matchesToRemove = new Set();
698
+ matches.forEach((match, i) => {
699
+ for (const otherMatch of matches.slice(i + 1)) {
700
+ if ((otherMatch.xpath || "").startsWith((match.xpath || "") + "/")) {
701
+ matchesToRemove.add(i);
702
+ break;
703
+ }
704
+ }
705
+ });
706
+ matchesMap[stringToMatch] = matches.filter(
707
+ (_, i) => !matchesToRemove.has(i)
708
+ );
709
+ }
710
+ return matchesMap;
711
+ }
712
+ function matchFuzzyAttributes(domNode, stringsToMatch) {
713
+ const matchesMap = Object.fromEntries(
714
+ stringsToMatch.map((data) => [data, []])
715
+ );
716
+ const allAttributes = getAllAttributes(domNode);
717
+ for (const stringToMatch of stringsToMatch) {
718
+ const stringToSearchIn = allAttributes
719
+ .filter((attr) => attr.value.length > 10)
720
+ .filter((attr) => {
721
+ const lengthDiff = Math.abs(attr.value.length - stringToMatch.length);
722
+ return lengthDiff <= 0.2 * stringToMatch.length;
723
+ })
724
+ .map((attr) => attr.value)
725
+ .join("\n");
726
+ const {
727
+ found: isFuzzMatchFound,
728
+ matchedValue,
729
+ distance: dist,
730
+ } = isFuzzMatch(stringToMatch, stringToSearchIn);
731
+ if (isFuzzMatchFound) {
732
+ const matchLine = allAttributes.find(
733
+ (attr) => matchedValue && attr.value.includes(matchedValue)
734
+ );
735
+ if (!matchLine) continue;
736
+ matchesMap[stringToMatch].push({
737
+ attribute: matchLine.attr,
738
+ fuzzy_distance: dist,
739
+ match_mode: MatchMode.FUZZY,
740
+ match_source: MatchSource.ATTRIBUTE,
741
+ matched_value: matchedValue,
742
+ xpath: matchLine.node,
743
+ matched_source_value: matchLine.value,
744
+ tag: matchLine.tag,
745
+ });
746
+ }
747
+ }
748
+ return matchesMap;
749
+ }
750
+ function getAllAttributes(node) {
751
+ const allNodes = [
752
+ node,
753
+ ...Array.from(node.querySelectorAll("*")),
754
+ ].reverse();
755
+ return allNodes.flatMap((node) =>
756
+ node
757
+ .getAttributeNames()
758
+ .map((attr) => ({
759
+ node: getElementXPath(node),
760
+ attr,
761
+ value: node.getAttribute(attr) || "",
762
+ tag: node.tagName.toLowerCase(),
763
+ }))
764
+ .filter((i) => i.value.length > 10)
765
+ );
766
+ }
767
+ function matchHref(node, stringToMatch) {
768
+ if (!node["href"] || typeof node["href"] !== "string") {
769
+ return;
770
+ }
771
+ const attributeValue = node["href"] || "";
772
+ let [isPartOfStringResult, matchedValue] = isPartOfString(
773
+ stringToMatch,
774
+ attributeValue
775
+ );
776
+ if (isPartOfStringResult) {
777
+ const [isExact] = isMatchExact(stringToMatch, attributeValue);
778
+ return {
779
+ attribute: "href",
780
+ fuzzy_distance: null,
781
+ match_mode: isExact ? MatchMode.FULL : MatchMode.PARTIAL,
782
+ match_source: MatchSource.ATTRIBUTE,
783
+ matched_value: matchedValue,
784
+ matched_source_value: attributeValue,
785
+ tag: node.tagName.toLowerCase(),
786
+ xpath: getElementXPath(node),
787
+ };
788
+ }
789
+ let decodedStringToMatch;
790
+ try {
791
+ decodedStringToMatch = decodeURI(stringToMatch);
792
+ } catch (e) {
793
+ console.log("failed to decode stringToMatch", stringToMatch);
794
+ return;
795
+ }
796
+ [isPartOfStringResult, matchedValue] = isPartOfString(
797
+ decodedStringToMatch,
798
+ attributeValue
799
+ );
800
+ if (isPartOfStringResult) {
801
+ const [isExact] = isMatchExact(stringToMatch, attributeValue);
802
+ return {
803
+ attribute: "href",
804
+ fuzzy_distance: null,
805
+ match_mode: isExact ? MatchMode.FULL : MatchMode.PARTIAL,
806
+ match_source: MatchSource.ATTRIBUTE,
807
+ matched_value: matchedValue,
808
+ matched_source_value: attributeValue,
809
+ tag: node.tagName.toLowerCase(),
810
+ xpath: getElementXPath(node),
811
+ };
812
+ }
813
+ }
814
+
815
+ function convertElementToMarkdown(element) {
816
+ const mdCharsMatcher = /([\\[\]()])/g;
817
+ function escapeMd(text) {
818
+ // Escapes markdown-sensitive characters within other markdown constructs.
819
+ return text.replace(mdCharsMatcher, "\\$1");
820
+ }
821
+ function listNumberingStart(attrs) {
822
+ const start = attrs.getNamedItem("start")?.value;
823
+ if (start) {
824
+ return parseInt(start, 10) - 1;
825
+ } else {
826
+ return 0;
827
+ }
828
+ }
829
+ // Define the characters that require escaping
830
+ const slashChars = "\\`*_{}[]()#+-.!";
831
+ // Escape any special regex characters in slashChars
832
+ const escapedSlashChars = slashChars.replace(
833
+ /[-/\\^$*+?.()|[\]{}]/g,
834
+ "\\$&"
835
+ );
836
+ // Create the regular expression
837
+ const mdBackslashMatcher = new RegExp(
838
+ `\\\\(?=[${escapedSlashChars}])`,
839
+ "g"
840
+ );
841
+ const mdDotMatcher = new RegExp(`^(\\s*\\d+)(\\.)(?=\\s)`, "gm");
842
+ const mdPlusMatcher = new RegExp(`^(\\s*)(\\+)(?=\\s)`, "gm");
843
+ const mdDashMatcher = new RegExp(`^(\\s*)(-)(?=\\s|-)`, "gm");
844
+ function escapeMdSection(text) {
845
+ text = text.replace(mdBackslashMatcher, "\\\\");
846
+ text = text.replace(mdDotMatcher, "$1\\$2");
847
+ text = text.replace(mdPlusMatcher, "$1\\$2");
848
+ text = text.replace(mdDashMatcher, "$1\\$2");
849
+ return text;
850
+ }
851
+ function isFirstTbody(element) {
852
+ const previousSibling = element.previousSibling;
853
+ return (
854
+ element.nodeName === "TBODY" &&
855
+ (!previousSibling ||
856
+ (previousSibling.nodeName === "THEAD" &&
857
+ /^\s*$/i.test(previousSibling.textContent ?? "")))
858
+ );
859
+ }
860
+ function isHeadingRow(tr) {
861
+ const parentNode = tr.parentNode;
862
+ return (
863
+ parentNode.nodeName === "THEAD" ||
864
+ (parentNode.firstChild === tr &&
865
+ (parentNode.nodeName === "TABLE" || isFirstTbody(parentNode)) &&
866
+ Array.from(tr.childNodes).every(function (n) {
867
+ return n.nodeName === "TH";
868
+ }))
869
+ );
870
+ }
871
+ class Html2Text {
872
+ p_p = 0;
873
+ abbrData; // last inner HTML (for abbr being defined)
874
+ pre = false;
875
+ code = false;
876
+ startPre = false;
877
+ blockquote = 0;
878
+ list = [];
879
+ start = true;
880
+ breakToggle = "";
881
+ space;
882
+ lastWasNewLine = false;
883
+ a = null;
884
+ outCount = 0;
885
+ baseurl;
886
+ abbrList = {};
887
+ outText = "";
888
+ outTextList = [];
889
+ abbr_title;
890
+ skipInternalLinks = true;
891
+ aStack = [];
892
+ maybeAutomaticLink;
893
+ lastWasList = false;
894
+ absoluteUrlMatcher = new RegExp("^[a-zA-Z+]+://");
895
+ emphasis_mark = "_";
896
+ strong_mark = "**";
897
+ break() {
898
+ if (this.p_p === 0) {
899
+ this.p_p = 1;
900
+ }
901
+ }
902
+ softBreak() {
903
+ this.break();
904
+ this.breakToggle = " ";
905
+ }
906
+ processOutput(data, pureData = 0, force = 0) {
907
+ if (this.abbrData !== undefined) {
908
+ this.abbrData += data;
909
+ }
910
+ if (pureData && !this.pre) {
911
+ data = data.replace(/\s+/g, " ");
912
+ if (data && data[0] === " ") {
913
+ this.space = 1;
914
+ data = data.substring(1);
915
+ }
916
+ }
917
+ if (!data && force !== "end") return;
918
+ if (this.startPre) {
919
+ if (!data.startsWith("\n")) {
920
+ data = "\n" + data;
921
+ }
922
+ }
923
+ let newLineIndent = ">".repeat(this.blockquote ?? 0);
924
+ if (!(force === "end" && data && data[0] === ">") && this.blockquote) {
925
+ newLineIndent += " ";
926
+ }
927
+ if (this.pre) {
928
+ if (this.list.length === 0) {
929
+ newLineIndent += " ";
930
+ } else {
931
+ for (let i = 0; i < this.list.length + 1; i++) {
932
+ newLineIndent += " ";
933
+ }
934
+ }
935
+ data = data.replace(/\n/g, `\n${newLineIndent}`);
936
+ }
937
+ if (this.startPre) {
938
+ this.startPre = false;
939
+ if (this.list.length > 0) {
940
+ data = data.trimStart();
941
+ }
942
+ }
943
+ if (this.start) {
944
+ this.space = 0;
945
+ this.p_p = 0;
946
+ this.start = false;
947
+ }
948
+ if (force === "end") {
949
+ this.p_p = 0;
950
+ this.out("\n");
951
+ this.space = 0;
952
+ }
953
+ if (this.p_p) {
954
+ this.out((this.breakToggle + "\n" + newLineIndent).repeat(this.p_p));
955
+ this.space = 0;
956
+ this.breakToggle = "";
957
+ }
958
+ if (this.space) {
959
+ if (!this.lastWasNewLine) {
960
+ this.out(" ");
961
+ }
962
+ this.space = 0;
963
+ }
964
+ if (this.a && force === "end") {
965
+ if (force === "end") {
966
+ this.out("\n");
967
+ }
968
+ const newA = this.a.filter((link) => {
969
+ if (this.outCount > link.outcount) {
970
+ this.out(
971
+ " [" +
972
+ link.count +
973
+ "]: " +
974
+ new URL(link.href, this.baseurl).toString()
975
+ );
976
+ if (link.title) {
977
+ this.out(" (" + link.title + ")");
978
+ }
979
+ this.out("\n");
980
+ return false;
981
+ }
982
+ return true;
983
+ });
984
+ if (this.a.length !== newA.length) {
985
+ this.out("\n");
986
+ }
987
+ this.a = newA;
988
+ }
989
+ if (this.abbrList && force === "end") {
990
+ for (const [abbr, definition] of Object.entries(this.abbrList)) {
991
+ this.out("\n *[" + abbr + "]: " + definition + "\n");
992
+ }
993
+ }
994
+ this.p_p = 0;
995
+ this.out(data);
996
+ this.outCount++;
997
+ }
998
+ out(string) {
999
+ this.outTextList.push(string);
1000
+ if (string) {
1001
+ this.lastWasNewLine = string.charAt(string.length - 1) === "\n";
1002
+ }
1003
+ }
1004
+ getResult() {
1005
+ this.processOutput("", 0, "end");
1006
+ this.outText = this.outTextList.join("");
1007
+ this.outText = this.outText.replace("&nbsp_place_holder;", " ");
1008
+ return this.outText;
1009
+ }
1010
+ getHeadingLevel(tag) {
1011
+ if (tag[0] === "h" && tag.length === 2) {
1012
+ try {
1013
+ const n = parseInt(tag[1]);
1014
+ if (!isNaN(n) && n >= 1 && n <= 9) {
1015
+ return n;
1016
+ }
1017
+ } catch (error) {
1018
+ return 0;
1019
+ }
1020
+ }
1021
+ return 0;
1022
+ }
1023
+ padding() {
1024
+ this.p_p = 2;
1025
+ }
1026
+ handleData(node) {
1027
+ if (this.maybeAutomaticLink) {
1028
+ const href = this.maybeAutomaticLink;
1029
+ if (
1030
+ href?.value === node.nodeValue &&
1031
+ this.absoluteUrlMatcher.test(href.value)
1032
+ ) {
1033
+ this.processOutput(`<${node.nodeValue}>`);
1034
+ return;
1035
+ } else {
1036
+ this.processOutput("[");
1037
+ this.maybeAutomaticLink = null;
1038
+ }
1039
+ }
1040
+ if (!this.code && !this.pre && node.nodeValue) {
1041
+ const data = escapeMdSection(node.nodeValue);
1042
+ this.processOutput(data, 1);
1043
+ return;
1044
+ }
1045
+ this.processOutput(node.textContent || "", 1);
1046
+ }
1047
+ handleTag(node) {
1048
+ const tag = node.nodeName.toLowerCase();
1049
+ if (["head", "style", "script"].includes(tag)) {
1050
+ return;
1051
+ }
1052
+ if (this.getHeadingLevel(tag)) {
1053
+ this.padding();
1054
+ this.processOutput("#".repeat(this.getHeadingLevel(tag)) + " ");
1055
+ }
1056
+ if (tag == "br") this.processOutput(" \n");
1057
+ if (tag == "hr") {
1058
+ this.padding();
1059
+ this.processOutput("---");
1060
+ this.padding();
1061
+ }
1062
+ if (tag == "blockquote") {
1063
+ this.padding();
1064
+ this.processOutput("> ", 0, 1);
1065
+ }
1066
+ }
1067
+ handleTagPrefix(node) {
1068
+ const nodeName = node.nodeName.toLowerCase();
1069
+ let attrs =
1070
+ node.nodeType === node.ELEMENT_NODE ? node.attributes : null;
1071
+ if (["table"].includes(nodeName)) {
1072
+ this.padding();
1073
+ }
1074
+ if (nodeName == "td" || nodeName == "th") {
1075
+ const index = Array.from(node.parentNode?.children ?? []).indexOf(
1076
+ node
1077
+ );
1078
+ let prefix = " ";
1079
+ if (index === 0) prefix = "| ";
1080
+ this.processOutput(prefix);
1081
+ // this.break();
1082
+ }
1083
+ if (["div", "p"].includes(nodeName)) {
1084
+ this.padding();
1085
+ }
1086
+ if (nodeName === "blockquote") {
1087
+ this.blockquote += 1;
1088
+ }
1089
+ if (nodeName === "pre") {
1090
+ this.pre = true;
1091
+ this.startPre = true;
1092
+ this.padding();
1093
+ }
1094
+ if (["code", "tt"].includes(nodeName)) {
1095
+ this.processOutput("`");
1096
+ }
1097
+ if (["em", "i", "u"].includes(nodeName)) {
1098
+ this.processOutput(this.emphasis_mark);
1099
+ }
1100
+ if (["strong", "b"].includes(nodeName)) {
1101
+ this.processOutput(this.strong_mark);
1102
+ }
1103
+ if (["del", "strike", "s"].includes(nodeName)) {
1104
+ this.processOutput("<" + nodeName + ">");
1105
+ }
1106
+ if (nodeName === "abbr") {
1107
+ this.abbr_title = null;
1108
+ this.abbrData = "";
1109
+ const title = attrs && attrs.getNamedItem("title");
1110
+ if (attrs && title) {
1111
+ this.abbr_title = title.value;
1112
+ }
1113
+ }
1114
+ if (nodeName === "dl") {
1115
+ this.padding();
1116
+ }
1117
+ if (nodeName === "dd") {
1118
+ this.processOutput(" ");
1119
+ }
1120
+ if (nodeName == "a") {
1121
+ const href = attrs ? attrs.getNamedItem("href") : null;
1122
+ if (href && !(this.skipInternalLinks && href.value.startsWith("#"))) {
1123
+ this.aStack.push(attrs);
1124
+ this.maybeAutomaticLink = href;
1125
+ } else {
1126
+ this.aStack.push(null);
1127
+ }
1128
+ }
1129
+ if (nodeName === "img") {
1130
+ const src = attrs ? attrs.getNamedItem("src") : null;
1131
+ if (src) {
1132
+ node.setAttribute("href", src.value);
1133
+ attrs = node.attributes;
1134
+ const alt = attrs.getNamedItem("alt")?.value;
1135
+ this.processOutput("![" + escapeMd(alt ?? "") + "]");
1136
+ this.processOutput(
1137
+ "(" + escapeMd(attrs.getNamedItem("href")?.value ?? "") + ")"
1138
+ );
1139
+ }
1140
+ }
1141
+ if (["ul", "ol"].includes(nodeName)) {
1142
+ const listStyle = nodeName;
1143
+ const numberingStart = listNumberingStart(node.attributes);
1144
+ this.list.push({ name: listStyle, num: numberingStart });
1145
+ this.lastWasList = true;
1146
+ } else {
1147
+ this.lastWasList = false;
1148
+ }
1149
+ if (nodeName === "li") {
1150
+ let li;
1151
+ this.break();
1152
+ if (this.list.length > 0) {
1153
+ li = this.list[this.list.length - 1];
1154
+ } else {
1155
+ li = { name: "ul", num: 0 };
1156
+ }
1157
+ const nestCount = this.list.length;
1158
+ this.processOutput(" ".repeat(nestCount));
1159
+ if (li["name"] == "ul") this.processOutput("*" + " ");
1160
+ else if (li["name"] == "ol") {
1161
+ li["num"] += 1;
1162
+ this.processOutput(li["num"] + ". ");
1163
+ }
1164
+ this.start = true;
1165
+ }
1166
+ }
1167
+ handleTagSuffix(node) {
1168
+ const nodeName = node.nodeName.toLowerCase();
1169
+ if (nodeName === "blockquote") {
1170
+ this.blockquote -= 1;
1171
+ }
1172
+ if (nodeName == "td" || nodeName == "th") {
1173
+ this.processOutput(" |");
1174
+ }
1175
+ if (nodeName == "tr") {
1176
+ const cell = (content, node) => {
1177
+ const index = Array.from(node.parentNode.childNodes).indexOf(node);
1178
+ let prefix = " ";
1179
+ if (index === 0) prefix = "| ";
1180
+ return prefix + content + " |";
1181
+ };
1182
+ let borderCells = "";
1183
+ const alignMap = { left: ":--", right: "--:", center: ":-:" };
1184
+ if (isHeadingRow(node)) {
1185
+ for (let i = 0; i < node.children.length; i++) {
1186
+ let border = "---";
1187
+ const align = (
1188
+ node.children[i].getAttribute("align") || ""
1189
+ ).toLowerCase();
1190
+ if (align) border = alignMap[align] || border;
1191
+ borderCells += cell(border, node.childNodes[i]);
1192
+ }
1193
+ }
1194
+ this.processOutput(borderCells ? "\n" + borderCells + "\n" : "\n");
1195
+ }
1196
+ if (nodeName === "pre") {
1197
+ this.pre = false;
1198
+ this.padding();
1199
+ }
1200
+ if (["code", "tt"].includes(nodeName)) {
1201
+ this.processOutput("`");
1202
+ }
1203
+ if (["em", "i", "u"].includes(nodeName)) {
1204
+ this.processOutput(this.emphasis_mark);
1205
+ }
1206
+ if (["strong", "b"].includes(nodeName)) {
1207
+ this.processOutput(this.strong_mark);
1208
+ }
1209
+ if (["div", "p"].includes(nodeName)) {
1210
+ this.padding();
1211
+ }
1212
+ if (["del", "strike", "s"].includes(nodeName)) {
1213
+ this.processOutput("</" + nodeName + ">");
1214
+ }
1215
+ if (nodeName === "abbr") {
1216
+ if (this.abbr_title && this.abbrData) {
1217
+ this.abbrList[this.abbrData] = this.abbr_title;
1218
+ this.abbr_title = null;
1219
+ }
1220
+ this.abbrData = "";
1221
+ }
1222
+ if (nodeName === "dt") {
1223
+ this.break();
1224
+ }
1225
+ if (nodeName === "dd") {
1226
+ this.break();
1227
+ }
1228
+ if (nodeName === "a") {
1229
+ if (this.aStack.length > 0) {
1230
+ const a = this.aStack.pop();
1231
+ if (this.maybeAutomaticLink) {
1232
+ this.maybeAutomaticLink = null;
1233
+ } else if (a) {
1234
+ this.processOutput(
1235
+ `](${escapeMd(a.getNamedItem("href")?.value || "")})`
1236
+ );
1237
+ }
1238
+ }
1239
+ }
1240
+ if (["ul", "ol"].includes(nodeName)) {
1241
+ if (this.list.length > 0) this.list.pop();
1242
+ this.lastWasList = true;
1243
+ } else {
1244
+ this.lastWasList = false;
1245
+ }
1246
+ if (nodeName === "li") {
1247
+ this.break();
1248
+ }
1249
+ }
1250
+ previousIndex(attrs) {
1251
+ // Returns the index of a certain set of attributes (of a link) in the
1252
+ // this.a list.
1253
+ // If the set of attributes is not found, returns null.
1254
+ const href = attrs.getNamedItem("href");
1255
+ if (!attrs.getNamedItem("href")) return null;
1256
+ let itemIndex = -1;
1257
+ for (const a of this.a ?? []) {
1258
+ itemIndex += 1;
1259
+ let match = false;
1260
+ if (a.getNamedItem("href") === href) {
1261
+ if (a.getNamedItem("title") || attrs.getNamedItem("title")) {
1262
+ if (
1263
+ a.getNamedItem("title") &&
1264
+ attrs.getNamedItem("title") &&
1265
+ a.getNamedItem("title") === attrs.getNamedItem("title")
1266
+ ) {
1267
+ match = true;
1268
+ }
1269
+ } else {
1270
+ match = true;
1271
+ }
1272
+ }
1273
+ if (match) return itemIndex;
1274
+ }
1275
+ return null;
1276
+ }
1277
+ handle(htmlElement) {
1278
+ // jsdom failed to parse hilton page due to invalid stylesheet
1279
+ // Nodes to be removed
1280
+ const filteredNodes = ["style", "script", "noscript"];
1281
+ for (const node of filteredNodes) {
1282
+ const nodeSelectors = htmlElement.querySelectorAll(node);
1283
+ nodeSelectors.forEach((nodeSelector) => {
1284
+ if (nodeSelector && nodeSelector.parentNode) {
1285
+ nodeSelector.parentNode.removeChild(nodeSelector);
1286
+ }
1287
+ });
1288
+ }
1289
+ // Get the cleaned-up HTML content
1290
+ const htmlContent = htmlElement.outerHTML;
1291
+ const parser = new DOMParser();
1292
+ const doc = parser.parseFromString(htmlContent, "text/html");
1293
+ const traverseDOM = (node) => {
1294
+ const tag = node.nodeName.toLowerCase();
1295
+ if (node.nodeType === node.TEXT_NODE) {
1296
+ const element = node;
1297
+ this.handleData(element);
1298
+ return;
1299
+ }
1300
+ if (node.nodeType === node.ELEMENT_NODE) {
1301
+ const element = node;
1302
+ this.handleTag(element);
1303
+ }
1304
+ if (!["head", "style", "script"].includes(tag)) {
1305
+ this.handleTagPrefix(node);
1306
+ node.childNodes.forEach((child) => traverseDOM(child));
1307
+ this.handleTagSuffix(node);
1308
+ }
1309
+ };
1310
+ traverseDOM(doc.documentElement);
1311
+ return this.getResult();
1312
+ }
1313
+ }
1314
+ const converter = new Html2Text();
1315
+ const result = converter.handle(element);
1316
+ return result;
1317
+ }
1318
+
1319
+ var node = {};
1320
+
1321
+ var htmlToMarkdownAST$1 = {};
1322
+
1323
+ var ElementNode = {};
1324
+
1325
+ Object.defineProperty(ElementNode, "__esModule", { value: true });
1326
+ ElementNode._Node = void 0;
1327
+ // this is by value copy of the global Node
1328
+ ElementNode._Node = {
1329
+ /** node is an element. */
1330
+ ELEMENT_NODE: 1,
1331
+ ATTRIBUTE_NODE: 2,
1332
+ /** node is a Text node. */
1333
+ TEXT_NODE: 3,
1334
+ /** node is a CDATASection node. */
1335
+ CDATA_SECTION_NODE: 4,
1336
+ ENTITY_REFERENCE_NODE: 5,
1337
+ ENTITY_NODE: 6,
1338
+ /** node is a ProcessingInstruction node. */
1339
+ PROCESSING_INSTRUCTION_NODE: 7,
1340
+ /** node is a Comment node. */
1341
+ COMMENT_NODE: 8,
1342
+ /** node is a document. */
1343
+ DOCUMENT_NODE: 9,
1344
+ /** node is a doctype. */
1345
+ DOCUMENT_TYPE_NODE: 10,
1346
+ /** node is a DocumentFragment node. */
1347
+ DOCUMENT_FRAGMENT_NODE: 11,
1348
+ NOTATION_NODE: 12,
1349
+ /** Set when node and other are not in the same tree. */
1350
+ DOCUMENT_POSITION_DISCONNECTED: 0x01,
1351
+ /** Set when other is preceding node. */
1352
+ DOCUMENT_POSITION_PRECEDING: 0x02,
1353
+ /** Set when other is following node. */
1354
+ DOCUMENT_POSITION_FOLLOWING: 0x04,
1355
+ /** Set when other is an ancestor of node. */
1356
+ DOCUMENT_POSITION_CONTAINS: 0x08,
1357
+ /** Set when other is a descendant of node. */
1358
+ DOCUMENT_POSITION_CONTAINED_BY: 0x10,
1359
+ DOCUMENT_POSITION_IMPLEMENTATION_SPECIFIC: 0x20,
1360
+ };
1361
+
1362
+ Object.defineProperty(htmlToMarkdownAST$1, "__esModule", { value: true });
1363
+ htmlToMarkdownAST$1.htmlToMarkdownAST = htmlToMarkdownAST;
1364
+ const ElementNode_1$1 = ElementNode;
1365
+ function htmlToMarkdownAST(element, options, indentLevel = 0) {
1366
+ let result = [];
1367
+ const debugLog = (message) => {
1368
+ if (options?.debug) {
1369
+ console.log(message);
1370
+ }
1371
+ };
1372
+ element.childNodes.forEach((childElement) => {
1373
+ const overriddenElementProcessing = options?.overrideElementProcessing?.(
1374
+ childElement,
1375
+ options,
1376
+ indentLevel
1377
+ );
1378
+ if (overriddenElementProcessing) {
1379
+ debugLog(`Element Processing Overridden: '${childElement.nodeType}'`);
1380
+ result.push(...overriddenElementProcessing);
1381
+ } else if (childElement.nodeType === ElementNode_1$1._Node.TEXT_NODE) {
1382
+ const textContent = escapeMarkdownCharacters(
1383
+ childElement.textContent?.trim() ?? ""
1384
+ );
1385
+ if (textContent && !!childElement.textContent) {
1386
+ debugLog(`Text Node: '${textContent}'`);
1387
+ // preserve whitespaces when text childElement is not empty
1388
+ result.push({
1389
+ type: "text",
1390
+ content: childElement.textContent?.trim(),
1391
+ });
1392
+ }
1393
+ } else if (childElement.nodeType === ElementNode_1$1._Node.ELEMENT_NODE) {
1394
+ const elem = childElement;
1395
+ if (/^h[1-6]$/i.test(elem.tagName)) {
1396
+ const level = parseInt(elem.tagName.substring(1));
1397
+ const content = escapeMarkdownCharacters(
1398
+ elem.textContent || ""
1399
+ ).trim();
1400
+ if (content) {
1401
+ debugLog(`Heading ${level}: '${elem.textContent}'`);
1402
+ result.push({ type: "heading", level, content });
1403
+ }
1404
+ } else if (elem.tagName.toLowerCase() === "p") {
1405
+ debugLog("Paragraph");
1406
+ result.push(...htmlToMarkdownAST(elem, options));
1407
+ // Add a new line after the paragraph
1408
+ result.push({ type: "text", content: "\n\n" });
1409
+ } else if (elem.tagName.toLowerCase() === "a") {
1410
+ debugLog(
1411
+ `Link: '${elem.getAttribute("href")}' with text '${
1412
+ elem.textContent
1413
+ }'`
1414
+ );
1415
+ // Check if the href is a data URL for an image
1416
+ if (
1417
+ typeof elem.getAttribute("href") === "string" &&
1418
+ elem.getAttribute("href").startsWith("data:image")
1419
+ ) {
1420
+ // If it's a data URL for an image, skip this link
1421
+ result.push({
1422
+ type: "link",
1423
+ href: "-",
1424
+ content: htmlToMarkdownAST(elem, options),
1425
+ });
1426
+ } else {
1427
+ // Process the link as usual
1428
+ let href = elem.getAttribute("href");
1429
+ if (typeof href === "string") {
1430
+ href =
1431
+ options?.websiteDomain && href.startsWith(options.websiteDomain)
1432
+ ? href.substring(options.websiteDomain.length)
1433
+ : href;
1434
+ } else {
1435
+ href = "#"; // Use a default value when href is not a string
1436
+ }
1437
+ // if all children are text,
1438
+ if (
1439
+ Array.from(elem.childNodes).every(
1440
+ (_) => _.nodeType === ElementNode_1$1._Node.TEXT_NODE
1441
+ )
1442
+ ) {
1443
+ result.push({
1444
+ type: "link",
1445
+ href: href,
1446
+ content: [
1447
+ { type: "text", content: elem.textContent?.trim() ?? "" },
1448
+ ],
1449
+ });
1450
+ } else {
1451
+ result.push({
1452
+ type: "link",
1453
+ href: href,
1454
+ content: htmlToMarkdownAST(elem, options),
1455
+ });
1456
+ }
1457
+ }
1458
+ } else if (elem.tagName.toLowerCase() === "img") {
1459
+ debugLog(`Image: src='${elem.src}', alt='${elem.alt}'`);
1460
+ if (elem.src?.startsWith("data:image")) {
1461
+ result.push({
1462
+ type: "image",
1463
+ src: "-",
1464
+ alt: escapeMarkdownCharacters(elem.alt),
1465
+ });
1466
+ } else {
1467
+ const src =
1468
+ options?.websiteDomain &&
1469
+ elem.src?.startsWith(options.websiteDomain)
1470
+ ? elem.src?.substring(options.websiteDomain.length)
1471
+ : elem.src;
1472
+ result.push({
1473
+ type: "image",
1474
+ src,
1475
+ alt: escapeMarkdownCharacters(elem.alt),
1476
+ });
1477
+ }
1478
+ } else if (elem.tagName.toLowerCase() === "video") {
1479
+ debugLog(
1480
+ `Video: src='${elem.src}', poster='${elem.poster}', controls='${elem.controls}'`
1481
+ );
1482
+ result.push({
1483
+ type: "video",
1484
+ src: elem.src,
1485
+ poster: escapeMarkdownCharacters(elem.poster),
1486
+ controls: elem.controls,
1487
+ });
1488
+ } else if (
1489
+ elem.tagName.toLowerCase() === "ul" ||
1490
+ elem.tagName.toLowerCase() === "ol"
1491
+ ) {
1492
+ debugLog(
1493
+ `${
1494
+ elem.tagName.toLowerCase() === "ul" ? "Unordered" : "Ordered"
1495
+ } List`
1496
+ );
1497
+ result.push({
1498
+ type: "list",
1499
+ ordered: elem.tagName.toLowerCase() === "ol",
1500
+ items: Array.from(elem.children).map((li) => ({
1501
+ type: "listItem",
1502
+ content: htmlToMarkdownAST(li, options, indentLevel + 1),
1503
+ })),
1504
+ });
1505
+ } else if (elem.tagName.toLowerCase() === "br") {
1506
+ debugLog("Line Break");
1507
+ result.push({ type: "text", content: "\n" });
1508
+ } else if (elem.tagName.toLowerCase() === "table") {
1509
+ debugLog("Table");
1510
+ let colIds = [];
1511
+ if (options?.enableTableColumnTracking) {
1512
+ // Generate unique column IDs
1513
+ const headerCells = Array.from(elem.querySelectorAll("th, td"));
1514
+ headerCells.forEach((_, index) => {
1515
+ colIds.push(`col-${index}`);
1516
+ });
1517
+ }
1518
+ const tableRows = Array.from(elem.querySelectorAll("tr"));
1519
+ const markdownTableRows = tableRows.map((row) => {
1520
+ let columnIndex = 0;
1521
+ const cells = Array.from(row.querySelectorAll("th, td")).map(
1522
+ (cell) => {
1523
+ const colspan = parseInt(
1524
+ cell.getAttribute("colspan") || "1",
1525
+ 10
1526
+ );
1527
+ const rowspan = parseInt(
1528
+ cell.getAttribute("rowspan") || "1",
1529
+ 10
1530
+ );
1531
+ const cellNode = {
1532
+ type: "tableCell",
1533
+ content:
1534
+ cell.nodeType === ElementNode_1$1._Node.TEXT_NODE
1535
+ ? escapeMarkdownCharacters(cell.textContent?.trim() ?? "")
1536
+ : htmlToMarkdownAST(cell, options, indentLevel + 1),
1537
+ colId: colIds[columnIndex],
1538
+ colspan: colspan > 1 ? colspan : undefined,
1539
+ rowspan: rowspan > 1 ? rowspan : undefined,
1540
+ };
1541
+ columnIndex += colspan;
1542
+ return cellNode;
1543
+ }
1544
+ );
1545
+ return { type: "tableRow", cells };
1546
+ });
1547
+ if (markdownTableRows.length > 0) {
1548
+ // Check if the first row contains header cells
1549
+ const hasHeaders = tableRows[0].querySelector("th") !== null;
1550
+ if (hasHeaders) {
1551
+ // Create a header separator row
1552
+ const headerSeparatorCells = Array.from(
1553
+ tableRows[0].querySelectorAll("th, td")
1554
+ ).map(() => ({
1555
+ type: "tableCell",
1556
+ content: "---",
1557
+ colId: undefined,
1558
+ colspan: undefined,
1559
+ rowspan: undefined,
1560
+ }));
1561
+ const headerSeparatorRow = {
1562
+ type: "tableRow",
1563
+ cells: headerSeparatorCells,
1564
+ };
1565
+ markdownTableRows.splice(1, 0, headerSeparatorRow);
1566
+ }
1567
+ }
1568
+ result.push({ type: "table", rows: markdownTableRows, colIds });
1569
+ } else if (
1570
+ elem.tagName.toLowerCase() === "head" &&
1571
+ !!options?.includeMetaData
1572
+ ) {
1573
+ const node = {
1574
+ type: "meta",
1575
+ content: {
1576
+ standard: {},
1577
+ openGraph: {},
1578
+ twitter: {},
1579
+ },
1580
+ };
1581
+ elem.querySelectorAll("title").forEach((titleElem) => {
1582
+ node.content.standard["title"] = escapeMarkdownCharacters(
1583
+ titleElem.text
1584
+ );
1585
+ });
1586
+ // Extract meta tags
1587
+ const metaTags = elem.querySelectorAll("meta");
1588
+ const nonSemanticTagNames = [
1589
+ "viewport",
1590
+ "referrer",
1591
+ "Content-Security-Policy",
1592
+ ];
1593
+ metaTags.forEach((metaTag) => {
1594
+ const name = metaTag.getAttribute("name");
1595
+ const property = metaTag.getAttribute("property");
1596
+ const content = metaTag.getAttribute("content");
1597
+ if (property && property.startsWith("og:") && content) {
1598
+ if (options.includeMetaData === "extended") {
1599
+ node.content.openGraph[property.substring(3)] = content;
1600
+ }
1601
+ } else if (name && name.startsWith("twitter:") && content) {
1602
+ if (options.includeMetaData === "extended") {
1603
+ node.content.twitter[name.substring(8)] = content;
1604
+ }
1605
+ } else if (name && !nonSemanticTagNames.includes(name) && content) {
1606
+ node.content.standard[name] = content;
1607
+ }
1608
+ });
1609
+ // Extract JSON-LD data
1610
+ if (options.includeMetaData === "extended") {
1611
+ const jsonLdData = [];
1612
+ const jsonLDScripts = elem.querySelectorAll(
1613
+ 'script[type="application/ld+json"]'
1614
+ );
1615
+ jsonLDScripts.forEach((script) => {
1616
+ try {
1617
+ const jsonContent = script.textContent;
1618
+ if (jsonContent) {
1619
+ const parsedData = JSON.parse(jsonContent);
1620
+ jsonLdData.push(parsedData);
1621
+ }
1622
+ } catch (error) {
1623
+ console.error("Failed to parse JSON-LD", error);
1624
+ }
1625
+ });
1626
+ node.content.jsonLd = jsonLdData;
1627
+ }
1628
+ result.push(node);
1629
+ } else {
1630
+ const content = escapeMarkdownCharacters(elem.textContent || "");
1631
+ switch (elem.tagName.toLowerCase()) {
1632
+ case "noscript":
1633
+ case "script":
1634
+ case "style":
1635
+ case "html":
1636
+ // blackhole..
1637
+ break;
1638
+ case "strong":
1639
+ case "b":
1640
+ if (content) {
1641
+ debugLog(`Bold: '${content}'`);
1642
+ result.push({
1643
+ type: "bold",
1644
+ content: htmlToMarkdownAST(elem, options, indentLevel + 1),
1645
+ });
1646
+ }
1647
+ break;
1648
+ case "em":
1649
+ case "i":
1650
+ if (content) {
1651
+ debugLog(`Italic: '${content}'`);
1652
+ result.push({
1653
+ type: "italic",
1654
+ content: htmlToMarkdownAST(elem, options, indentLevel + 1),
1655
+ });
1656
+ }
1657
+ break;
1658
+ case "s":
1659
+ case "strike":
1660
+ if (content) {
1661
+ debugLog(`Strikethrough: '${content}'`);
1662
+ result.push({
1663
+ type: "strikethrough",
1664
+ content: htmlToMarkdownAST(elem, options, indentLevel + 1),
1665
+ });
1666
+ }
1667
+ break;
1668
+ case "code":
1669
+ if (content) {
1670
+ // Handling inline code differently
1671
+ const isCodeBlock =
1672
+ elem.parentNode &&
1673
+ elem.parentNode.nodeName.toLowerCase() === "pre";
1674
+ debugLog(
1675
+ `${isCodeBlock ? "Code Block" : "Inline Code"}: '${content}'`
1676
+ );
1677
+ const languageClass = elem.className
1678
+ ?.split(" ")
1679
+ .find((cls) => cls.startsWith("language-"));
1680
+ const language = languageClass
1681
+ ? languageClass.replace("language-", "")
1682
+ : "";
1683
+ result.push({
1684
+ type: "code",
1685
+ content: elem.textContent?.trim() ?? "",
1686
+ language,
1687
+ inline: !isCodeBlock,
1688
+ });
1689
+ }
1690
+ break;
1691
+ case "blockquote":
1692
+ debugLog(`Blockquote`);
1693
+ result.push({
1694
+ type: "blockquote",
1695
+ content: htmlToMarkdownAST(elem, options),
1696
+ });
1697
+ break;
1698
+ case "article":
1699
+ case "aside":
1700
+ case "details":
1701
+ case "figcaption":
1702
+ case "figure":
1703
+ case "footer":
1704
+ case "header":
1705
+ case "main":
1706
+ case "mark":
1707
+ case "nav":
1708
+ case "section":
1709
+ case "summary":
1710
+ case "time":
1711
+ debugLog(`Semantic HTML Element: '${elem.tagName}'`);
1712
+ result.push({
1713
+ type: "semanticHtml",
1714
+ htmlType: elem.tagName.toLowerCase(),
1715
+ content: htmlToMarkdownAST(elem, options),
1716
+ });
1717
+ break;
1718
+ default:
1719
+ const unhandledElementProcessing =
1720
+ options?.processUnhandledElement?.(elem, options, indentLevel);
1721
+ if (unhandledElementProcessing) {
1722
+ debugLog(`Processing Unhandled Element: '${elem.tagName}'`);
1723
+ result.push(...unhandledElementProcessing);
1724
+ } else {
1725
+ debugLog(`Generic HTMLElement: '${elem.tagName}'`);
1726
+ result.push(
1727
+ ...htmlToMarkdownAST(elem, options, indentLevel + 1)
1728
+ );
1729
+ }
1730
+ break;
1731
+ }
1732
+ }
1733
+ }
1734
+ });
1735
+ return result;
1736
+ }
1737
+ function escapeMarkdownCharacters(text, isInlineCode = false) {
1738
+ if (isInlineCode || !text?.trim()) {
1739
+ // In inline code, we don't escape any characters
1740
+ return text;
1741
+ }
1742
+ // First, replace special HTML characters with their entity equivalents
1743
+ let escapedText = text
1744
+ .replace(/&/g, "&amp;") // Replace & first
1745
+ .replace(/</g, "&lt;")
1746
+ .replace(/>/g, "&gt;");
1747
+ // Then escape characters that have special meaning in Markdown
1748
+ escapedText = escapedText.replace(/([\\`*_{}[\]#+!|])/g, "\\$1");
1749
+ return escapedText;
1750
+ }
1751
+
1752
+ var markdownASTToString = {};
1753
+
1754
+ var hasRequiredMarkdownASTToString;
1755
+
1756
+ function requireMarkdownASTToString() {
1757
+ if (hasRequiredMarkdownASTToString) return markdownASTToString;
1758
+ hasRequiredMarkdownASTToString = 1;
1759
+ Object.defineProperty(markdownASTToString, "__esModule", { value: true });
1760
+ markdownASTToString.markdownASTToString = markdownASTToString$1;
1761
+ const index_1 = requireNode();
1762
+ function markdownASTToString$1(nodes, options, indentLevel = 0) {
1763
+ let markdownString = "";
1764
+ markdownString += markdownMetaASTToString(nodes, options, indentLevel);
1765
+ markdownString += markdownContentASTToString(nodes, options, indentLevel);
1766
+ return markdownString;
1767
+ }
1768
+ function markdownMetaASTToString(nodes, options, indentLevel = 0) {
1769
+ let markdownString = "";
1770
+ if (options?.includeMetaData) {
1771
+ // include meta-data
1772
+ markdownString += "---\n";
1773
+ const node = (0, index_1.findInMarkdownAST)(
1774
+ nodes,
1775
+ (_) => _.type === "meta"
1776
+ );
1777
+ if (node?.type === "meta") {
1778
+ if (node.content.standard) {
1779
+ Object.keys(node.content.standard).forEach((key) => {
1780
+ markdownString += `${key}: "${node.content.standard[key]}"\n`;
1781
+ });
1782
+ }
1783
+ if (options.includeMetaData === "extended") {
1784
+ if (node.content.openGraph) {
1785
+ if (Object.keys(node.content.openGraph).length > 0) {
1786
+ markdownString += "openGraph:\n";
1787
+ for (const [key, value] of Object.entries(
1788
+ node.content.openGraph
1789
+ )) {
1790
+ markdownString += ` ${key}: "${value}"\n`;
1791
+ }
1792
+ }
1793
+ }
1794
+ if (node.content.twitter) {
1795
+ if (Object.keys(node.content.twitter).length > 0) {
1796
+ markdownString += "twitter:\n";
1797
+ for (const [key, value] of Object.entries(
1798
+ node.content.twitter
1799
+ )) {
1800
+ markdownString += ` ${key}: "${value}"\n`;
1801
+ }
1802
+ }
1803
+ }
1804
+ if (node.content.jsonLd && node.content.jsonLd.length > 0) {
1805
+ markdownString += "schema:\n";
1806
+ node.content.jsonLd.forEach((item) => {
1807
+ const {
1808
+ "@context": jldContext,
1809
+ "@type": jldType,
1810
+ ...semanticData
1811
+ } = item;
1812
+ markdownString += ` ${jldType ?? "(unknown type)"}:\n`;
1813
+ Object.keys(semanticData).forEach((key) => {
1814
+ markdownString += ` ${key}: ${JSON.stringify(
1815
+ semanticData[key]
1816
+ )}\n`;
1817
+ });
1818
+ });
1819
+ }
1820
+ }
1821
+ }
1822
+ markdownString += "---\n\n";
1823
+ }
1824
+ return markdownString;
1825
+ }
1826
+ function markdownContentASTToString(nodes, options, indentLevel = 0) {
1827
+ let markdownString = "";
1828
+ nodes.forEach((node) => {
1829
+ const indent = " ".repeat(indentLevel * 2); // Adjust the multiplier for different indent sizes
1830
+ const nodeRenderingOverride = options?.overrideNodeRenderer?.(
1831
+ node,
1832
+ options,
1833
+ indentLevel
1834
+ );
1835
+ if (nodeRenderingOverride) {
1836
+ markdownString += nodeRenderingOverride;
1837
+ } else {
1838
+ switch (node.type) {
1839
+ case "text":
1840
+ case "bold":
1841
+ case "italic":
1842
+ case "strikethrough":
1843
+ case "link":
1844
+ let content = node.content; // might be a nodes array but we take care of that below
1845
+ if (Array.isArray(node.content)) {
1846
+ content = markdownContentASTToString(
1847
+ node.content,
1848
+ options,
1849
+ indentLevel
1850
+ );
1851
+ }
1852
+ const isMarkdownStringNotEmpty = markdownString.length > 0;
1853
+ const isFirstCharOfContentWhitespace = /\s/.test(
1854
+ content.slice(0, 1)
1855
+ );
1856
+ const isLastCharOfMarkdownWhitespace = /\s/.test(
1857
+ markdownString.slice(-1)
1858
+ );
1859
+ const isContentPunctuation =
1860
+ content.length === 1 && /^[.,!?;:]/.test(content);
1861
+ if (
1862
+ isMarkdownStringNotEmpty &&
1863
+ !isContentPunctuation &&
1864
+ !isFirstCharOfContentWhitespace &&
1865
+ !isLastCharOfMarkdownWhitespace
1866
+ ) {
1867
+ markdownString += " ";
1868
+ }
1869
+ if (node.type === "text") {
1870
+ markdownString += `${indent}${content}`;
1871
+ } else {
1872
+ if (node.type === "bold") {
1873
+ markdownString += `**${content}**`;
1874
+ } else if (node.type === "italic") {
1875
+ markdownString += `*${content}*`;
1876
+ } else if (node.type === "strikethrough") {
1877
+ markdownString += `~~${content}~~`;
1878
+ } else if (node.type === "link") {
1879
+ // check if the link contains only text
1880
+ if (
1881
+ node.content.length === 1 &&
1882
+ node.content[0].type === "text"
1883
+ ) {
1884
+ // use native markdown syntax for text-only links
1885
+ markdownString += `[${content}](${encodeURI(node.href)})`;
1886
+ } else {
1887
+ // Use HTML <a> tag for links with rich content
1888
+ markdownString += `<a href="${node.href}">${content}</a>`;
1889
+ }
1890
+ }
1891
+ }
1892
+ break;
1893
+ case "heading":
1894
+ const isEndsWithNewLine = markdownString.slice(-1) === "\n";
1895
+ if (!isEndsWithNewLine) {
1896
+ markdownString += "\n";
1897
+ }
1898
+ markdownString += `${"#".repeat(node.level)} ${node.content}\n\n`;
1899
+ break;
1900
+ case "image":
1901
+ if (!node.alt?.trim() || !!node.src?.trim()) {
1902
+ markdownString += `![${node.alt || ""}](${node.src})`;
1903
+ }
1904
+ break;
1905
+ case "list":
1906
+ node.items.forEach((item, i) => {
1907
+ const listItemPrefix = node.ordered ? `${i + 1}.` : "-";
1908
+ const contents = markdownContentASTToString(
1909
+ item.content,
1910
+ options,
1911
+ indentLevel + 1
1912
+ ).trim();
1913
+ if (markdownString.slice(-1) !== "\n") {
1914
+ markdownString += "\n";
1915
+ }
1916
+ if (contents) {
1917
+ markdownString += `${indent}${listItemPrefix} ${contents}\n`;
1918
+ }
1919
+ });
1920
+ markdownString += "\n";
1921
+ break;
1922
+ case "video":
1923
+ markdownString += `\n![Video](${node.src})\n`;
1924
+ if (node.poster) {
1925
+ markdownString += `![Poster](${node.poster})\n`;
1926
+ }
1927
+ if (node.controls) {
1928
+ markdownString += `Controls: ${node.controls}\n`;
1929
+ }
1930
+ markdownString += "\n";
1931
+ break;
1932
+ case "table":
1933
+ const maxColumns = Math.max(
1934
+ ...node.rows.map((row) =>
1935
+ row.cells.reduce((sum, cell) => sum + (cell.colspan || 1), 0)
1936
+ )
1937
+ );
1938
+ node.rows.forEach((row) => {
1939
+ let currentColumn = 0;
1940
+ row.cells.forEach((cell) => {
1941
+ let cellContent =
1942
+ typeof cell.content === "string"
1943
+ ? cell.content
1944
+ : markdownContentASTToString(
1945
+ cell.content,
1946
+ options,
1947
+ indentLevel + 1
1948
+ ).trim();
1949
+ if (cell.colId) {
1950
+ cellContent += ` <!-- ${cell.colId} -->`;
1951
+ }
1952
+ if (cell.colspan && cell.colspan > 1) {
1953
+ cellContent += ` <!-- colspan: ${cell.colspan} -->`;
1954
+ }
1955
+ if (cell.rowspan && cell.rowspan > 1) {
1956
+ cellContent += ` <!-- rowspan: ${cell.rowspan} -->`;
1957
+ }
1958
+ markdownString += `| ${cellContent} `;
1959
+ currentColumn += cell.colspan || 1;
1960
+ // Add empty cells for colspan
1961
+ for (let i = 1; i < (cell.colspan || 1); i++) {
1962
+ markdownString += "| ";
1963
+ }
1964
+ });
1965
+ // Fill remaining columns with empty cells
1966
+ while (currentColumn < maxColumns) {
1967
+ markdownString += "| ";
1968
+ currentColumn++;
1969
+ }
1970
+ markdownString += "|\n";
1971
+ });
1972
+ markdownString += "\n";
1973
+ break;
1974
+ case "code":
1975
+ if (node.inline) {
1976
+ const isLsatWhitespace = /\s/.test(markdownString.slice(-1));
1977
+ if (!isLsatWhitespace) {
1978
+ markdownString += " ";
1979
+ }
1980
+ markdownString += `\`${node.content}\``;
1981
+ } else {
1982
+ // For code blocks, we do not escape characters and preserve formatting
1983
+ markdownString += "\n```" + (node.language ?? "") + "\n";
1984
+ markdownString += `${node.content}\n`;
1985
+ markdownString += "```\n\n";
1986
+ }
1987
+ break;
1988
+ case "blockquote":
1989
+ markdownString += `> ${markdownContentASTToString(
1990
+ node.content,
1991
+ options
1992
+ ).trim()}\n\n`;
1993
+ break;
1994
+ case "meta":
1995
+ // already handled
1996
+ break;
1997
+ case "semanticHtml":
1998
+ switch (node.htmlType) {
1999
+ case "article":
2000
+ markdownString +=
2001
+ "\n\n" + markdownContentASTToString(node.content, options);
2002
+ break;
2003
+ case "summary":
2004
+ case "time":
2005
+ case "aside":
2006
+ case "nav":
2007
+ case "figcaption":
2008
+ case "main":
2009
+ case "mark":
2010
+ case "header":
2011
+ case "footer":
2012
+ case "details":
2013
+ case "figure":
2014
+ markdownString +=
2015
+ `\n\n<-${node.htmlType}->\n` +
2016
+ markdownContentASTToString(node.content, options) +
2017
+ `\n\n</-${node.htmlType}->\n`;
2018
+ break;
2019
+ case "section":
2020
+ markdownString += "---\n\n";
2021
+ markdownString += markdownContentASTToString(
2022
+ node.content,
2023
+ options
2024
+ );
2025
+ markdownString += "\n\n";
2026
+ markdownString += "---\n\n";
2027
+ break;
2028
+ }
2029
+ break;
2030
+ case "custom":
2031
+ const customNodeRendering = options?.renderCustomNode?.(
2032
+ node,
2033
+ options,
2034
+ indentLevel
2035
+ );
2036
+ if (customNodeRendering) {
2037
+ markdownString += customNodeRendering;
2038
+ }
2039
+ break;
2040
+ }
2041
+ }
2042
+ });
2043
+ return markdownString;
2044
+ }
2045
+ return markdownASTToString;
2046
+ }
2047
+
2048
+ var domUtils = {};
2049
+
2050
+ Object.defineProperty(domUtils, "__esModule", { value: true });
2051
+ domUtils.findMainContent = findMainContent;
2052
+ domUtils.wrapMainContent = wrapMainContent;
2053
+ domUtils.isElementVisible = isElementVisible;
2054
+ domUtils.getVisibleText = getVisibleText;
2055
+ const ElementNode_1 = ElementNode;
2056
+ const debugMessage = (message) => {};
2057
+ /**
2058
+ * Attempts to find the main content of a web page.
2059
+ * @param document The Document object to search.
2060
+ * @returns The Element containing the main content, or the body if no main content is found.
2061
+ */
2062
+ function findMainContent(document) {
2063
+ const mainElement = document.querySelector("main");
2064
+ if (mainElement) {
2065
+ return mainElement;
2066
+ }
2067
+ if (!document.body) {
2068
+ return document.documentElement;
2069
+ }
2070
+ return detectMainContent(document.body);
2071
+ }
2072
+ function wrapMainContent(mainContentElement, document) {
2073
+ if (mainContentElement.tagName.toLowerCase() !== "main") {
2074
+ const mainElement = document.createElement("main");
2075
+ mainContentElement.before(mainElement);
2076
+ mainElement.appendChild(mainContentElement);
2077
+ mainElement.id = "detected-main-content";
2078
+ }
2079
+ }
2080
+ function detectMainContent(rootElement) {
2081
+ const candidates = [];
2082
+ const minScore = 20;
2083
+ collectCandidates(rootElement, candidates, minScore);
2084
+ if (candidates.length === 0) {
2085
+ return rootElement;
2086
+ }
2087
+ candidates.sort((a, b) => calculateScore(b) - calculateScore(a));
2088
+ let bestIndependentCandidate = candidates[0];
2089
+ for (let i = 1; i < candidates.length; i++) {
2090
+ if (
2091
+ !candidates.some(
2092
+ (otherCandidate, j) =>
2093
+ j !== i && otherCandidate.contains(candidates[i])
2094
+ )
2095
+ ) {
2096
+ if (
2097
+ calculateScore(candidates[i]) >
2098
+ calculateScore(bestIndependentCandidate)
2099
+ ) {
2100
+ bestIndependentCandidate = candidates[i];
2101
+ debugMessage(
2102
+ `New best independent candidate found: ${elementToString(
2103
+ bestIndependentCandidate
2104
+ )}`
2105
+ );
2106
+ }
2107
+ }
2108
+ }
2109
+ debugMessage(
2110
+ `Final main content candidate: ${elementToString(
2111
+ bestIndependentCandidate
2112
+ )}`
2113
+ );
2114
+ return bestIndependentCandidate;
2115
+ }
2116
+ function elementToString(element) {
2117
+ if (!element) {
2118
+ return "No element";
2119
+ }
2120
+ return `${element.tagName}#${element.id || "no-id"}.${Array.from(
2121
+ element.classList
2122
+ ).join(".")}`;
2123
+ }
2124
+ function collectCandidates(element, candidates, minScore) {
2125
+ const score = calculateScore(element);
2126
+ if (score >= minScore) {
2127
+ candidates.push(element);
2128
+ debugMessage(
2129
+ `Candidate found: ${elementToString(element)}, score: ${score}`
2130
+ );
2131
+ }
2132
+ Array.from(element.children).forEach((child) => {
2133
+ collectCandidates(child, candidates, minScore);
2134
+ });
2135
+ }
2136
+ function calculateScore(element) {
2137
+ let score = 0;
2138
+ let scoreLog = [];
2139
+ // High impact attributes
2140
+ const highImpactAttributes = [
2141
+ "article",
2142
+ "content",
2143
+ "main-container",
2144
+ "main",
2145
+ "main-content",
2146
+ ];
2147
+ highImpactAttributes.forEach((attr) => {
2148
+ if (element.classList.contains(attr) || element.id.includes(attr)) {
2149
+ score += 10;
2150
+ scoreLog.push(
2151
+ `High impact attribute found: ${attr}, score increased by 10`
2152
+ );
2153
+ }
2154
+ });
2155
+ // High impact tags
2156
+ const highImpactTags = ["article", "main", "section"];
2157
+ if (highImpactTags.includes(element.tagName.toLowerCase())) {
2158
+ score += 5;
2159
+ scoreLog.push(
2160
+ `High impact tag found: ${element.tagName}, score increased by 5`
2161
+ );
2162
+ }
2163
+ // Paragraph count
2164
+ const paragraphCount = element.getElementsByTagName("p").length;
2165
+ const paragraphScore = Math.min(paragraphCount, 5);
2166
+ if (paragraphScore > 0) {
2167
+ score += paragraphScore;
2168
+ scoreLog.push(
2169
+ `Paragraph count: ${paragraphCount}, score increased by ${paragraphScore}`
2170
+ );
2171
+ }
2172
+ // Text content length
2173
+ const textContentLength = element.textContent?.trim().length || 0;
2174
+ if (textContentLength > 200) {
2175
+ const textScore = Math.min(Math.floor(textContentLength / 200), 5);
2176
+ score += textScore;
2177
+ scoreLog.push(
2178
+ `Text content length: ${textContentLength}, score increased by ${textScore}`
2179
+ );
2180
+ }
2181
+ // Link density
2182
+ const linkDensity = calculateLinkDensity(element);
2183
+ if (linkDensity < 0.3) {
2184
+ score += 5;
2185
+ scoreLog.push(
2186
+ `Link density: ${linkDensity.toFixed(2)}, score increased by 5`
2187
+ );
2188
+ }
2189
+ // Data attributes
2190
+ if (
2191
+ element.hasAttribute("data-main") ||
2192
+ element.hasAttribute("data-content")
2193
+ ) {
2194
+ score += 10;
2195
+ scoreLog.push(
2196
+ "Data attribute for main content found, score increased by 10"
2197
+ );
2198
+ }
2199
+ // Role attribute
2200
+ if (element.getAttribute("role")?.includes("main")) {
2201
+ score += 10;
2202
+ scoreLog.push(
2203
+ "Role attribute indicating main content found, score increased by 10"
2204
+ );
2205
+ }
2206
+ if (scoreLog.length > 0) {
2207
+ debugMessage(`Scoring for ${elementToString(element)}:`);
2208
+ }
2209
+ return score;
2210
+ }
2211
+ function calculateLinkDensity(element) {
2212
+ const linkLength = Array.from(element.getElementsByTagName("a")).reduce(
2213
+ (sum, link) => sum + (link.textContent?.length || 0),
2214
+ 0
2215
+ );
2216
+ const textLength = element.textContent?.length || 1; // Avoid division by zero
2217
+ return linkLength / textLength;
2218
+ }
2219
+ function isElementVisible(element) {
2220
+ if (!(element instanceof HTMLElement)) {
2221
+ return true; // Non-HTMLElements are considered visible
2222
+ }
2223
+ const style = window.getComputedStyle(element);
2224
+ return (
2225
+ style.display !== "none" &&
2226
+ style.visibility !== "hidden" &&
2227
+ style.opacity !== "0"
2228
+ );
2229
+ }
2230
+ function getVisibleText(element) {
2231
+ if (!isElementVisible(element)) {
2232
+ return "";
2233
+ }
2234
+ let text = "";
2235
+ for (const child of Array.from(element.childNodes)) {
2236
+ if (child.nodeType === ElementNode_1._Node.TEXT_NODE) {
2237
+ text += child.textContent;
2238
+ } else if (child.nodeType === ElementNode_1._Node.ELEMENT_NODE) {
2239
+ text += getVisibleText(child);
2240
+ }
2241
+ }
2242
+ return text.trim();
2243
+ }
2244
+
2245
+ var urlUtils = {};
2246
+
2247
+ Object.defineProperty(urlUtils, "__esModule", { value: true });
2248
+ urlUtils.refifyUrls = refifyUrls;
2249
+ const mediaSuffixes = [
2250
+ "jpeg",
2251
+ "jpg",
2252
+ "png",
2253
+ "gif",
2254
+ "bmp",
2255
+ "tiff",
2256
+ "tif",
2257
+ "svg",
2258
+ "webp",
2259
+ "ico",
2260
+ "avi",
2261
+ "mov",
2262
+ "mp4",
2263
+ "mkv",
2264
+ "flv",
2265
+ "wmv",
2266
+ "webm",
2267
+ "mpeg",
2268
+ "mpg",
2269
+ "mp3",
2270
+ "wav",
2271
+ "aac",
2272
+ "ogg",
2273
+ "flac",
2274
+ "m4a",
2275
+ "pdf",
2276
+ "doc",
2277
+ "docx",
2278
+ "ppt",
2279
+ "pptx",
2280
+ "xls",
2281
+ "xlsx",
2282
+ "txt",
2283
+ "css",
2284
+ "js",
2285
+ "xml",
2286
+ "json",
2287
+ "html",
2288
+ "htm",
2289
+ ];
2290
+ const addRefPrefix = (prefix, prefixesToRefs) => {
2291
+ if (!prefixesToRefs[prefix]) {
2292
+ prefixesToRefs[prefix] = "ref" + Object.values(prefixesToRefs).length;
2293
+ }
2294
+ return prefixesToRefs[prefix];
2295
+ };
2296
+ const processUrl = (url, prefixesToRefs) => {
2297
+ if (!url.startsWith("http")) {
2298
+ return url;
2299
+ } else {
2300
+ const mediaSuffix = url.split(".").slice(-1)[0];
2301
+ if (mediaSuffix && mediaSuffixes.includes(mediaSuffix)) {
2302
+ const parts = url.split("/"); // Split URL keeping the slash before text
2303
+ const prefix = parts.slice(0, -1).join("/"); // Get the prefix by removing last part
2304
+ const refPrefix = addRefPrefix(prefix, prefixesToRefs);
2305
+ return `${refPrefix}://${parts.slice(-1).join("")}`;
2306
+ } else {
2307
+ if (url.split("/").length > 4) {
2308
+ return addRefPrefix(url, prefixesToRefs);
2309
+ } else {
2310
+ return url;
2311
+ }
2312
+ }
2313
+ }
2314
+ };
2315
+ function refifyUrls(markdownElement, prefixesToRefs = {}) {
2316
+ if (Array.isArray(markdownElement)) {
2317
+ markdownElement.forEach((element) => refifyUrls(element, prefixesToRefs));
2318
+ } else {
2319
+ switch (markdownElement.type) {
2320
+ case "link":
2321
+ markdownElement.href = processUrl(
2322
+ markdownElement.href,
2323
+ prefixesToRefs
2324
+ );
2325
+ refifyUrls(markdownElement.content, prefixesToRefs);
2326
+ break;
2327
+ case "image":
2328
+ case "video":
2329
+ markdownElement.src = processUrl(markdownElement.src, prefixesToRefs);
2330
+ break;
2331
+ case "list":
2332
+ markdownElement.items.forEach((item) =>
2333
+ item.content.forEach((_) => refifyUrls(_, prefixesToRefs))
2334
+ );
2335
+ break;
2336
+ case "table":
2337
+ markdownElement.rows.forEach((row) =>
2338
+ row.cells.forEach((cell) =>
2339
+ typeof cell.content === "string"
2340
+ ? null
2341
+ : refifyUrls(cell.content, prefixesToRefs)
2342
+ )
2343
+ );
2344
+ break;
2345
+ case "blockquote":
2346
+ case "semanticHtml":
2347
+ refifyUrls(markdownElement.content, prefixesToRefs);
2348
+ break;
2349
+ }
2350
+ }
2351
+ return prefixesToRefs;
2352
+ }
2353
+
2354
+ var astUtils = {};
2355
+
2356
+ (function (exports) {
2357
+ Object.defineProperty(exports, "__esModule", { value: true });
2358
+ exports.isNot = exports.getMainContent = void 0;
2359
+ exports.findInAST = findInAST;
2360
+ exports.findAllInAST = findAllInAST;
2361
+ const getMainContent = (markdownStr) => {
2362
+ if (markdownStr.includes("<-main->")) {
2363
+ const regex = /(?<=<-main->)[\s\S]*?(?=<\/-main->)/;
2364
+ const match = markdownStr.match(regex);
2365
+ return match?.[0] ?? "";
2366
+ } else {
2367
+ const removeSectionsRegex =
2368
+ /(<-nav->[\s\S]*?<\/-nav->)|(<-footer->[\s\S]*?<\/-footer->)|(<-header->[\s\S]*?<\/-header->)|(<-aside->[\s\S]*?<\/-aside->)/g;
2369
+ return markdownStr.replace(removeSectionsRegex, "");
2370
+ }
2371
+ };
2372
+ exports.getMainContent = getMainContent;
2373
+ const isNot = (tPred) => (t) => !tPred(t);
2374
+ exports.isNot = isNot;
2375
+ const isString = (x) => typeof x === "string";
2376
+ function findInAST(markdownElement, checker) {
2377
+ const loopCheck = (z) => {
2378
+ for (const element of z) {
2379
+ const found = findInAST(element, checker);
2380
+ if (found) {
2381
+ return found;
2382
+ }
2383
+ }
2384
+ return undefined;
2385
+ };
2386
+ if (Array.isArray(markdownElement)) {
2387
+ return loopCheck(markdownElement);
2388
+ } else {
2389
+ if (checker(markdownElement)) {
2390
+ return markdownElement;
2391
+ }
2392
+ switch (markdownElement.type) {
2393
+ case "link":
2394
+ return loopCheck(markdownElement.content);
2395
+ case "list":
2396
+ return loopCheck(
2397
+ markdownElement.items.map((_) => _.content).flat()
2398
+ );
2399
+ case "table":
2400
+ return loopCheck(
2401
+ markdownElement.rows
2402
+ .map((row) =>
2403
+ row.cells
2404
+ .map((_) => _.content)
2405
+ .filter((0, exports.isNot)(isString))
2406
+ )
2407
+ .flat()
2408
+ );
2409
+ case "blockquote":
2410
+ case "semanticHtml":
2411
+ return loopCheck(markdownElement.content);
2412
+ }
2413
+ return undefined;
2414
+ }
2415
+ }
2416
+ function findAllInAST(markdownElement, checker) {
2417
+ const loopCheck = (z) => {
2418
+ let out = [];
2419
+ for (const element of z) {
2420
+ const found = findAllInAST(element, checker);
2421
+ out = [...out, ...found];
2422
+ }
2423
+ return out;
2424
+ };
2425
+ if (Array.isArray(markdownElement)) {
2426
+ return loopCheck(markdownElement);
2427
+ } else {
2428
+ if (checker(markdownElement)) {
2429
+ return [markdownElement];
2430
+ }
2431
+ switch (markdownElement.type) {
2432
+ case "link":
2433
+ return loopCheck(markdownElement.content);
2434
+ case "list":
2435
+ return loopCheck(
2436
+ markdownElement.items.map((_) => _.content).flat()
2437
+ );
2438
+ case "table":
2439
+ return loopCheck(
2440
+ markdownElement.rows
2441
+ .map((row) =>
2442
+ row.cells
2443
+ .map((_) => _.content)
2444
+ .filter((0, exports.isNot)(isString))
2445
+ )
2446
+ .flat()
2447
+ );
2448
+ case "blockquote":
2449
+ case "semanticHtml":
2450
+ return loopCheck(markdownElement.content);
2451
+ }
2452
+ return [];
2453
+ }
2454
+ }
2455
+ })(astUtils);
2456
+
2457
+ var hasRequiredNode;
2458
+
2459
+ function requireNode() {
2460
+ if (hasRequiredNode) return node;
2461
+ hasRequiredNode = 1;
2462
+ (function (exports) {
2463
+ Object.defineProperty(exports, "__esModule", { value: true });
2464
+ exports.wrapMainContent =
2465
+ exports.refifyUrls =
2466
+ exports.findMainContent =
2467
+ exports.markdownASTToString =
2468
+ exports.htmlToMarkdownAST =
2469
+ void 0;
2470
+ exports.convertHtmlToMarkdown = convertHtmlToMarkdown;
2471
+ exports.convertElementToMarkdown = convertElementToMarkdown;
2472
+ exports.findInMarkdownAST = findInMarkdownAST;
2473
+ exports.findAllInMarkdownAST = findAllInMarkdownAST;
2474
+ const htmlToMarkdownAST_1 = htmlToMarkdownAST$1;
2475
+ Object.defineProperty(exports, "htmlToMarkdownAST", {
2476
+ enumerable: true,
2477
+ get: function () {
2478
+ return htmlToMarkdownAST_1.htmlToMarkdownAST;
2479
+ },
2480
+ });
2481
+ const markdownASTToString_1 = requireMarkdownASTToString();
2482
+ Object.defineProperty(exports, "markdownASTToString", {
2483
+ enumerable: true,
2484
+ get: function () {
2485
+ return markdownASTToString_1.markdownASTToString;
2486
+ },
2487
+ });
2488
+ const domUtils_1 = domUtils;
2489
+ Object.defineProperty(exports, "findMainContent", {
2490
+ enumerable: true,
2491
+ get: function () {
2492
+ return domUtils_1.findMainContent;
2493
+ },
2494
+ });
2495
+ Object.defineProperty(exports, "wrapMainContent", {
2496
+ enumerable: true,
2497
+ get: function () {
2498
+ return domUtils_1.wrapMainContent;
2499
+ },
2500
+ });
2501
+ const urlUtils_1 = urlUtils;
2502
+ Object.defineProperty(exports, "refifyUrls", {
2503
+ enumerable: true,
2504
+ get: function () {
2505
+ return urlUtils_1.refifyUrls;
2506
+ },
2507
+ });
2508
+ const astUtils_1 = astUtils;
2509
+ /**
2510
+ * Converts an HTML string to Markdown.
2511
+ * @param html The HTML string to convert.
2512
+ * @param options Conversion options.
2513
+ * @returns The converted Markdown string.
2514
+ */
2515
+ function convertHtmlToMarkdown(html, options) {
2516
+ const parser =
2517
+ options?.overrideDOMParser ??
2518
+ (typeof DOMParser !== "undefined" ? new DOMParser() : null);
2519
+ if (!parser) {
2520
+ throw new Error(
2521
+ "DOMParser is not available. Please provide an overrideDOMParser in options."
2522
+ );
2523
+ }
2524
+ const doc = parser.parseFromString(html, "text/html");
2525
+ let element;
2526
+ if (options?.extractMainContent) {
2527
+ element = (0, domUtils_1.findMainContent)(doc);
2528
+ if (
2529
+ options.includeMetaData &&
2530
+ !!doc.querySelector("head")?.innerHTML &&
2531
+ !element.querySelector("head")
2532
+ ) {
2533
+ // content container was found and extracted, re-attaching the head for meta-data extraction
2534
+ element = parser.parseFromString(
2535
+ `<html>${doc.head.outerHTML}${element.outerHTML}`,
2536
+ "text/html"
2537
+ ).documentElement;
2538
+ }
2539
+ } else {
2540
+ // If there's a body, use it; otherwise, use the document element
2541
+ if (
2542
+ options?.includeMetaData &&
2543
+ !!doc.querySelector("head")?.innerHTML
2544
+ ) {
2545
+ element = doc.documentElement;
2546
+ } else {
2547
+ element = doc.body || doc.documentElement;
2548
+ }
2549
+ }
2550
+ return convertElementToMarkdown(element, options);
2551
+ }
2552
+ /**
2553
+ * Converts an HTML Element to Markdown.
2554
+ * @param element The HTML Element to convert.
2555
+ * @param options Conversion options.
2556
+ * @returns The converted Markdown string.
2557
+ */
2558
+ function convertElementToMarkdown(element, options) {
2559
+ let ast = (0, htmlToMarkdownAST_1.htmlToMarkdownAST)(element, options);
2560
+ if (options?.refifyUrls) {
2561
+ options.urlMap = (0, urlUtils_1.refifyUrls)(ast);
2562
+ }
2563
+ return (0, markdownASTToString_1.markdownASTToString)(ast, options);
2564
+ }
2565
+ /**
2566
+ * Finds a node in the Markdown AST that matches the given predicate.
2567
+ * @param ast The Markdown AST to search.
2568
+ * @param predicate A function that returns true for the desired node.
2569
+ * @returns The first matching node, or undefined if not found.
2570
+ */
2571
+ function findInMarkdownAST(ast, predicate) {
2572
+ return (0, astUtils_1.findInAST)(ast, predicate);
2573
+ }
2574
+ /**
2575
+ * Finds all nodes in the Markdown AST that match the given predicate.
2576
+ * @param ast The Markdown AST to search.
2577
+ * @param predicate A function that returns true for the desired nodes.
2578
+ * @returns An array of all matching nodes.
2579
+ */
2580
+ function findAllInMarkdownAST(ast, predicate) {
2581
+ return (0, astUtils_1.findAllInAST)(ast, predicate);
2582
+ }
2583
+ })(node);
2584
+ return node;
2585
+ }
2586
+
2587
+ var nodeExports = requireNode();
2588
+
2589
+ //@ts-ignore
2590
+ window.__INTUNED__ = {
2591
+ matchStringsWithDomContent,
2592
+ convertElementToMarkdown,
2593
+ convertHtmlStringToSemanticMarkdown: nodeExports.convertHtmlToMarkdown,
2594
+ getElementXPath: getElementXPath,
2595
+ };
2596
+ })();