@zeldrisho/pi-web-fetch 0.5.1 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/package.json +4 -4
- package/src/extract.ts +22 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,4 +1,11 @@
|
|
|
1
1
|
# Changelog
|
|
2
|
+
|
|
3
|
+
## [0.5.2](https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.5.1...pi-web-fetch-v0.5.2) (2026-08-10)
|
|
4
|
+
|
|
5
|
+
### Bug fixes
|
|
6
|
+
|
|
7
|
+
- **web-fetch:** Discard malformed schema metadata ([2ac7b02](https://github.com/zeldrisho/pi-packages/commit/2ac7b02f8952950aa60d4d22f080ab0774cdf50a))
|
|
8
|
+
|
|
2
9
|
## [0.5.1](https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.5.0...pi-web-fetch-v0.5.1) (2026-08-03)
|
|
3
10
|
|
|
4
11
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zeldrisho/pi-web-fetch",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.2",
|
|
4
4
|
"description": "Pi extension for secure, bounded public web page fetching and Markdown extraction",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"pi",
|
|
@@ -35,10 +35,10 @@
|
|
|
35
35
|
"linkedom": "^0.18.13"
|
|
36
36
|
},
|
|
37
37
|
"devDependencies": {
|
|
38
|
-
"@earendil-works/pi-coding-agent": "^0.
|
|
39
|
-
"@earendil-works/pi-tui": "^0.
|
|
38
|
+
"@earendil-works/pi-coding-agent": "^0.84.0",
|
|
39
|
+
"@earendil-works/pi-tui": "^0.84.0",
|
|
40
40
|
"typebox": "^1.1.24",
|
|
41
|
-
"typescript": "^
|
|
41
|
+
"typescript": "^7.0.0",
|
|
42
42
|
"vite-plus": "0.2.5"
|
|
43
43
|
},
|
|
44
44
|
"peerDependencies": {
|
package/src/extract.ts
CHANGED
|
@@ -2,6 +2,24 @@ import { parseHTML } from "linkedom";
|
|
|
2
2
|
|
|
3
3
|
const RAW_ID_SELECTOR_SAFE = /^-?[_a-zA-Z][-_a-zA-Z0-9]*$/;
|
|
4
4
|
|
|
5
|
+
/** Removes schema.org scripts that Defuddle would report directly to the process console. */
|
|
6
|
+
function removeMalformedSchemaOrgData(document: Document): void {
|
|
7
|
+
for (const script of document.querySelectorAll<HTMLScriptElement>(
|
|
8
|
+
'script[type="application/ld+json"]',
|
|
9
|
+
)) {
|
|
10
|
+
const jsonContent = (script.textContent || "")
|
|
11
|
+
.replace(/\/\*[\s\S]*?\*\/|^\s*\/\/.*$/gm, "")
|
|
12
|
+
.replace(/^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/, "$1")
|
|
13
|
+
.replace(/^\s*(\*\/|\/\*)\s*|\s*(\*\/|\/\*)\s*$/g, "")
|
|
14
|
+
.trim();
|
|
15
|
+
try {
|
|
16
|
+
if (JSON.parse(jsonContent) === null) script.remove();
|
|
17
|
+
} catch {
|
|
18
|
+
script.remove();
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
5
23
|
/**
|
|
6
24
|
* Replaces element IDs that are unsafe for CSS selectors and updates matching fragment links.
|
|
7
25
|
*
|
|
@@ -71,8 +89,10 @@ export async function extractHtmlToMarkdown(
|
|
|
71
89
|
try {
|
|
72
90
|
const { Defuddle } = await import("defuddle/node");
|
|
73
91
|
const { document } = parseHTML(html);
|
|
74
|
-
|
|
75
|
-
|
|
92
|
+
const defuddleDocument = document as unknown as Document;
|
|
93
|
+
removeMalformedSchemaOrgData(defuddleDocument);
|
|
94
|
+
normalizeSelectorUnsafeIds(defuddleDocument);
|
|
95
|
+
const result = await Defuddle(defuddleDocument, baseUrl.toString(), {
|
|
76
96
|
markdown: true,
|
|
77
97
|
useAsync: false,
|
|
78
98
|
});
|