dsh-web-fetch-playwright 0.2.8 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +11 -11
- package/lib/index.js +108 -4
- package/lib/types/host-resolve-compat.d.ts +38 -0
- package/lib/types/markdown.d.ts +4 -1
- package/package.json +1 -1
- package/src/host-resolve-compat.ts +102 -0
- package/src/markdown.ts +33 -2
package/lib/client.js
CHANGED
|
@@ -412,11 +412,11 @@ window.__ModuleLoader__.load({
|
|
|
412
412
|
document.head.appendChild(tag);
|
|
413
413
|
}
|
|
414
414
|
var PluginCard_module_css_default = {
|
|
415
|
-
"footer": "UM8-Rq_footer",
|
|
416
|
-
"form": "UM8-Rq_form",
|
|
417
415
|
"discard": "UM8-Rq_discard",
|
|
418
416
|
"save": "UM8-Rq_save",
|
|
417
|
+
"footer": "UM8-Rq_footer",
|
|
419
418
|
"failed": "UM8-Rq_failed",
|
|
419
|
+
"form": "UM8-Rq_form",
|
|
420
420
|
"readOnly": "UM8-Rq_readOnly"
|
|
421
421
|
};
|
|
422
422
|
//#endregion
|
|
@@ -477,25 +477,25 @@ window.__ModuleLoader__.load({
|
|
|
477
477
|
document.head.appendChild(tag);
|
|
478
478
|
}
|
|
479
479
|
var fields_module_css_default = {
|
|
480
|
-
"radioContent": "CFSaVG_radioContent",
|
|
481
|
-
"field": "CFSaVG_field",
|
|
482
480
|
"checkboxRow": "CFSaVG_checkboxRow",
|
|
483
|
-
"radioHint": "CFSaVG_radioHint",
|
|
484
|
-
"reset": "CFSaVG_reset",
|
|
485
|
-
"badges": "CFSaVG_badges",
|
|
486
481
|
"radioGroup": "CFSaVG_radioGroup",
|
|
487
482
|
"fieldEmbedded": "CFSaVG_fieldEmbedded",
|
|
483
|
+
"radioContent": "CFSaVG_radioContent",
|
|
488
484
|
"radioOption": "CFSaVG_radioOption",
|
|
489
|
-
"radioLabel": "CFSaVG_radioLabel",
|
|
490
485
|
"inputInvalid": "CFSaVG_inputInvalid",
|
|
491
486
|
"radioText": "CFSaVG_radioText",
|
|
492
|
-
"label": "CFSaVG_label",
|
|
493
|
-
"invalid": "CFSaVG_invalid",
|
|
494
|
-
"hintIndented": "CFSaVG_hintIndented",
|
|
495
487
|
"badge": "CFSaVG_badge",
|
|
488
|
+
"radioLabel": "CFSaVG_radioLabel",
|
|
489
|
+
"field": "CFSaVG_field",
|
|
496
490
|
"input": "CFSaVG_input",
|
|
491
|
+
"badges": "CFSaVG_badges",
|
|
492
|
+
"invalid": "CFSaVG_invalid",
|
|
497
493
|
"head": "CFSaVG_head",
|
|
494
|
+
"label": "CFSaVG_label",
|
|
495
|
+
"reset": "CFSaVG_reset",
|
|
496
|
+
"hintIndented": "CFSaVG_hintIndented",
|
|
498
497
|
"radioPick": "CFSaVG_radioPick",
|
|
498
|
+
"radioHint": "CFSaVG_radioHint",
|
|
499
499
|
"hint": "CFSaVG_hint"
|
|
500
500
|
};
|
|
501
501
|
//#endregion
|
package/lib/index.js
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
|
-
import { createRequire } from "node:module";
|
|
1
|
+
import { createRequire, isBuiltin } from "node:module";
|
|
2
2
|
import z from "@deepseek-ai/schemastery";
|
|
3
3
|
import { WebError } from "@deepseek-ai/dsh-web";
|
|
4
|
-
import { JSDOM, VirtualConsole } from "jsdom";
|
|
5
4
|
import createDOMPurify from "dompurify";
|
|
6
5
|
import { Readability } from "@mozilla/readability";
|
|
7
6
|
import TurndownService from "turndown";
|
|
8
7
|
import { gfm } from "@joplin/turndown-plugin-gfm";
|
|
8
|
+
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
9
9
|
import { accessSync, constants, readFileSync, realpathSync, statSync } from "node:fs";
|
|
10
10
|
import { delimiter, dirname, join } from "node:path";
|
|
11
|
-
import { pathToFileURL } from "node:url";
|
|
12
11
|
//#region src/config.ts
|
|
13
12
|
/** Default CDP endpoint when the settings section leaves it blank. */
|
|
14
13
|
const DEFAULT_CDP_ENDPOINT = "127.0.0.1:9222";
|
|
@@ -518,6 +517,87 @@ var CdpConnectionPool = class {
|
|
|
518
517
|
}
|
|
519
518
|
};
|
|
520
519
|
//#endregion
|
|
520
|
+
//#region src/host-resolve-compat.ts
|
|
521
|
+
/**
|
|
522
|
+
* Host resolve compat for the `punycode/` builtin-slash idiom.
|
|
523
|
+
*
|
|
524
|
+
* dsh web (runtime resolution in `dsh-app-boot`) intercepts Node's CommonJS
|
|
525
|
+
* resolver for every package installed in a profile. Its router classifies a
|
|
526
|
+
* bare request by stripping one trailing path segment, which turns tr46's
|
|
527
|
+
* `require("punycode/")` — the standard npm-package-over-deprecated-builtin
|
|
528
|
+
* idiom — into the builtin name `punycode`, and `require.resolve.paths()` of a
|
|
529
|
+
* builtin returns `null`; the unguarded iteration then throws
|
|
530
|
+
* `TypeError: createRequire.resolve.paths is not a function or its return
|
|
531
|
+
* value is not iterable`, killing the whole `jsdom` import chain (jsdom →
|
|
532
|
+
* whatwg-url → tr46) and with it this plugin's activation.
|
|
533
|
+
*
|
|
534
|
+
* The shim below is installed lazily, immediately before the first `jsdom`
|
|
535
|
+
* load, and only when the running resolver actually exhibits the failure
|
|
536
|
+
* (probed with the very request that breaks). It wraps the public
|
|
537
|
+
* `Module._resolveFilename` — whatever it currently is, native or already
|
|
538
|
+
* host-intercepted — and re-dispatches exactly the affected request shape
|
|
539
|
+
* (`<deprecated builtin>/`) with an explicit `paths` list. An explicit
|
|
540
|
+
* `options.paths` is the documented bypass of every resolution interceptor in
|
|
541
|
+
* the chain (the host wrapper delegates straight to the native resolver for
|
|
542
|
+
* it), so the request lands on the real npm package — the same file plain
|
|
543
|
+
* Node would have found. Hosts without the bug never see the wrapper at all;
|
|
544
|
+
* the probe itself changes nothing.
|
|
545
|
+
*
|
|
546
|
+
* The ESM flavor of the same router path exists in theory (`import
|
|
547
|
+
* 'punycode/'`), but nothing in this plugin's tree imports it that way —
|
|
548
|
+
* tr46 is CommonJS — so the CommonJS hook is the complete fix here.
|
|
549
|
+
*
|
|
550
|
+
* @module dsh-web-fetch-playwright/host-resolve-compat
|
|
551
|
+
*/
|
|
552
|
+
/** `createRequire` anchored at this module, wherever the bundle is installed. */
|
|
553
|
+
const require_$1 = createRequire(import.meta.url);
|
|
554
|
+
/** This file's own path, the anchor the probe resolves from. */
|
|
555
|
+
const SELF_FILENAME = fileURLToPath(import.meta.url);
|
|
556
|
+
/**
|
|
557
|
+
* Whether a request is the npm-over-builtin idiom: one bare package name with
|
|
558
|
+
* a single trailing slash, whose stem is a Node builtin (today, in practice:
|
|
559
|
+
* `punycode/`).
|
|
560
|
+
* @param request - the module request seen by `Module._resolveFilename`.
|
|
561
|
+
* @returns `true` for `punycode/`-shaped requests; relative paths, scoped
|
|
562
|
+
* names, subpaths, and ordinary packages all answer `false`.
|
|
563
|
+
*/
|
|
564
|
+
function isBuiltinSlashRequest(request) {
|
|
565
|
+
if (request.length < 2 || !request.endsWith("/")) return false;
|
|
566
|
+
const first = request.charAt(0);
|
|
567
|
+
if (first === "" || "./#@".includes(first)) return false;
|
|
568
|
+
const stem = request.slice(0, -1);
|
|
569
|
+
return stem !== "" && !stem.includes("/") && isBuiltin(stem);
|
|
570
|
+
}
|
|
571
|
+
/** Probe outcome once known: `true` = shim installed or verified unneeded. */
|
|
572
|
+
let settled = false;
|
|
573
|
+
/**
|
|
574
|
+
* Make `require("<deprecated builtin>/")` resolvable under hosts whose
|
|
575
|
+
* profile-scope resolution interceptor mishandles it. Idempotent, synchronous,
|
|
576
|
+
* and a no-op on healthy resolvers (the probe resolves successfully and no
|
|
577
|
+
* wrapper is installed). Must run before the first `require('jsdom')`.
|
|
578
|
+
*/
|
|
579
|
+
function installBuiltinSlashResolveCompat() {
|
|
580
|
+
if (settled) return;
|
|
581
|
+
settled = true;
|
|
582
|
+
const ModuleClass = require_$1("node:module").Module;
|
|
583
|
+
const original = ModuleClass._resolveFilename;
|
|
584
|
+
if (typeof original !== "function") return;
|
|
585
|
+
try {
|
|
586
|
+
original.call(ModuleClass, "punycode/", { filename: SELF_FILENAME });
|
|
587
|
+
return;
|
|
588
|
+
} catch (error) {
|
|
589
|
+
if (!(error instanceof TypeError)) return;
|
|
590
|
+
}
|
|
591
|
+
const searchPaths = require_$1.resolve.paths("jsdom") ?? [];
|
|
592
|
+
ModuleClass._resolveFilename = function patchedResolveFilename(request, parent, main, options) {
|
|
593
|
+
if (options?.paths === void 0 && typeof request === "string" && isBuiltinSlashRequest(request)) return original.call(ModuleClass, request, parent, main, {
|
|
594
|
+
...options,
|
|
595
|
+
paths: searchPaths
|
|
596
|
+
});
|
|
597
|
+
return original.call(ModuleClass, request, parent, main, options);
|
|
598
|
+
};
|
|
599
|
+
}
|
|
600
|
+
//#endregion
|
|
521
601
|
//#region src/markdown.ts
|
|
522
602
|
/**
|
|
523
603
|
* The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
|
|
@@ -532,10 +612,33 @@ var CdpConnectionPool = class {
|
|
|
532
612
|
* options and span-safe table rules as the shipped `dsh-tool-web` renderer,
|
|
533
613
|
* so output is consistent with what `web_fetch` produces elsewhere.
|
|
534
614
|
*
|
|
535
|
-
*
|
|
615
|
+
* Synchronous for its callers and unit-tested against fixture pages: jsdom
|
|
616
|
+
* itself is loaded lazily on the first call — and, inside a dsh profile, only
|
|
617
|
+
* after `installBuiltinSlashResolveCompat`, so a host resolver that
|
|
618
|
+
* mishandles tr46's `require("punycode/")` cannot take the plugin down.
|
|
536
619
|
*
|
|
537
620
|
* @module dsh-web-fetch-playwright/markdown
|
|
538
621
|
*/
|
|
622
|
+
/** `createRequire` anchored at this module, so jsdom resolves from this install. */
|
|
623
|
+
const require_ = createRequire(import.meta.url);
|
|
624
|
+
/** Loaded jsdom module, cached after the first denoise. */
|
|
625
|
+
let jsdom;
|
|
626
|
+
/**
|
|
627
|
+
* Load jsdom on first use, installing the host resolve compat first.
|
|
628
|
+
*
|
|
629
|
+
* Deferral serves two purposes: plugin activation never depends on jsdom's
|
|
630
|
+
* import chain surviving the host's module-resolution interception, and the
|
|
631
|
+
* compat shim — which must wrap the resolver *before* tr46's
|
|
632
|
+
* `require("punycode/")` runs — gets a guaranteed installation point.
|
|
633
|
+
* @returns the jsdom module exports.
|
|
634
|
+
*/
|
|
635
|
+
function loadJsdom() {
|
|
636
|
+
if (jsdom === void 0) {
|
|
637
|
+
installBuiltinSlashResolveCompat();
|
|
638
|
+
jsdom = require_("jsdom");
|
|
639
|
+
}
|
|
640
|
+
return jsdom;
|
|
641
|
+
}
|
|
539
642
|
/** Layout/noise tags DOMPurify removes outright (with their content). */
|
|
540
643
|
const FORBID_TAGS = [
|
|
541
644
|
"nav",
|
|
@@ -666,6 +769,7 @@ turndown.addRule("tableRowWithoutSpanExpansion", {
|
|
|
666
769
|
* @returns the markdown and the extraction mode used.
|
|
667
770
|
*/
|
|
668
771
|
function htmlToMarkdown(html, url) {
|
|
772
|
+
const { JSDOM, VirtualConsole } = loadJsdom();
|
|
669
773
|
const dom = new JSDOM(html, {
|
|
670
774
|
url,
|
|
671
775
|
virtualConsole: new VirtualConsole()
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Host resolve compat for the `punycode/` builtin-slash idiom.
|
|
3
|
+
*
|
|
4
|
+
* dsh web (runtime resolution in `dsh-app-boot`) intercepts Node's CommonJS
|
|
5
|
+
* resolver for every package installed in a profile. Its router classifies a
|
|
6
|
+
* bare request by stripping one trailing path segment, which turns tr46's
|
|
7
|
+
* `require("punycode/")` — the standard npm-package-over-deprecated-builtin
|
|
8
|
+
* idiom — into the builtin name `punycode`, and `require.resolve.paths()` of a
|
|
9
|
+
* builtin returns `null`; the unguarded iteration then throws
|
|
10
|
+
* `TypeError: createRequire.resolve.paths is not a function or its return
|
|
11
|
+
* value is not iterable`, killing the whole `jsdom` import chain (jsdom →
|
|
12
|
+
* whatwg-url → tr46) and with it this plugin's activation.
|
|
13
|
+
*
|
|
14
|
+
* The shim below is installed lazily, immediately before the first `jsdom`
|
|
15
|
+
* load, and only when the running resolver actually exhibits the failure
|
|
16
|
+
* (probed with the very request that breaks). It wraps the public
|
|
17
|
+
* `Module._resolveFilename` — whatever it currently is, native or already
|
|
18
|
+
* host-intercepted — and re-dispatches exactly the affected request shape
|
|
19
|
+
* (`<deprecated builtin>/`) with an explicit `paths` list. An explicit
|
|
20
|
+
* `options.paths` is the documented bypass of every resolution interceptor in
|
|
21
|
+
* the chain (the host wrapper delegates straight to the native resolver for
|
|
22
|
+
* it), so the request lands on the real npm package — the same file plain
|
|
23
|
+
* Node would have found. Hosts without the bug never see the wrapper at all;
|
|
24
|
+
* the probe itself changes nothing.
|
|
25
|
+
*
|
|
26
|
+
* The ESM flavor of the same router path exists in theory (`import
|
|
27
|
+
* 'punycode/'`), but nothing in this plugin's tree imports it that way —
|
|
28
|
+
* tr46 is CommonJS — so the CommonJS hook is the complete fix here.
|
|
29
|
+
*
|
|
30
|
+
* @module dsh-web-fetch-playwright/host-resolve-compat
|
|
31
|
+
*/
|
|
32
|
+
/**
|
|
33
|
+
* Make `require("<deprecated builtin>/")` resolvable under hosts whose
|
|
34
|
+
* profile-scope resolution interceptor mishandles it. Idempotent, synchronous,
|
|
35
|
+
* and a no-op on healthy resolvers (the probe resolves successfully and no
|
|
36
|
+
* wrapper is installed). Must run before the first `require('jsdom')`.
|
|
37
|
+
*/
|
|
38
|
+
export declare function installBuiltinSlashResolveCompat(): void;
|
package/lib/types/markdown.d.ts
CHANGED
|
@@ -11,7 +11,10 @@
|
|
|
11
11
|
* options and span-safe table rules as the shipped `dsh-tool-web` renderer,
|
|
12
12
|
* so output is consistent with what `web_fetch` produces elsewhere.
|
|
13
13
|
*
|
|
14
|
-
*
|
|
14
|
+
* Synchronous for its callers and unit-tested against fixture pages: jsdom
|
|
15
|
+
* itself is loaded lazily on the first call — and, inside a dsh profile, only
|
|
16
|
+
* after `installBuiltinSlashResolveCompat`, so a host resolver that
|
|
17
|
+
* mishandles tr46's `require("punycode/")` cannot take the plugin down.
|
|
15
18
|
*
|
|
16
19
|
* @module dsh-web-fetch-playwright/markdown
|
|
17
20
|
*/
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-web-fetch-playwright",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.9",
|
|
4
4
|
"description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Host resolve compat for the `punycode/` builtin-slash idiom.
|
|
3
|
+
*
|
|
4
|
+
* dsh web (runtime resolution in `dsh-app-boot`) intercepts Node's CommonJS
|
|
5
|
+
* resolver for every package installed in a profile. Its router classifies a
|
|
6
|
+
* bare request by stripping one trailing path segment, which turns tr46's
|
|
7
|
+
* `require("punycode/")` — the standard npm-package-over-deprecated-builtin
|
|
8
|
+
* idiom — into the builtin name `punycode`, and `require.resolve.paths()` of a
|
|
9
|
+
* builtin returns `null`; the unguarded iteration then throws
|
|
10
|
+
* `TypeError: createRequire.resolve.paths is not a function or its return
|
|
11
|
+
* value is not iterable`, killing the whole `jsdom` import chain (jsdom →
|
|
12
|
+
* whatwg-url → tr46) and with it this plugin's activation.
|
|
13
|
+
*
|
|
14
|
+
* The shim below is installed lazily, immediately before the first `jsdom`
|
|
15
|
+
* load, and only when the running resolver actually exhibits the failure
|
|
16
|
+
* (probed with the very request that breaks). It wraps the public
|
|
17
|
+
* `Module._resolveFilename` — whatever it currently is, native or already
|
|
18
|
+
* host-intercepted — and re-dispatches exactly the affected request shape
|
|
19
|
+
* (`<deprecated builtin>/`) with an explicit `paths` list. An explicit
|
|
20
|
+
* `options.paths` is the documented bypass of every resolution interceptor in
|
|
21
|
+
* the chain (the host wrapper delegates straight to the native resolver for
|
|
22
|
+
* it), so the request lands on the real npm package — the same file plain
|
|
23
|
+
* Node would have found. Hosts without the bug never see the wrapper at all;
|
|
24
|
+
* the probe itself changes nothing.
|
|
25
|
+
*
|
|
26
|
+
* The ESM flavor of the same router path exists in theory (`import
|
|
27
|
+
* 'punycode/'`), but nothing in this plugin's tree imports it that way —
|
|
28
|
+
* tr46 is CommonJS — so the CommonJS hook is the complete fix here.
|
|
29
|
+
*
|
|
30
|
+
* @module dsh-web-fetch-playwright/host-resolve-compat
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
import { createRequire, isBuiltin } from 'node:module'
|
|
34
|
+
import { fileURLToPath } from 'node:url'
|
|
35
|
+
|
|
36
|
+
/** `createRequire` anchored at this module, wherever the bundle is installed. */
|
|
37
|
+
const require_ = createRequire(import.meta.url)
|
|
38
|
+
|
|
39
|
+
/** This file's own path, the anchor the probe resolves from. */
|
|
40
|
+
const SELF_FILENAME = fileURLToPath(import.meta.url)
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Whether a request is the npm-over-builtin idiom: one bare package name with
|
|
44
|
+
* a single trailing slash, whose stem is a Node builtin (today, in practice:
|
|
45
|
+
* `punycode/`).
|
|
46
|
+
* @param request - the module request seen by `Module._resolveFilename`.
|
|
47
|
+
* @returns `true` for `punycode/`-shaped requests; relative paths, scoped
|
|
48
|
+
* names, subpaths, and ordinary packages all answer `false`.
|
|
49
|
+
*/
|
|
50
|
+
function isBuiltinSlashRequest(request: string): boolean {
|
|
51
|
+
if (request.length < 2 || !request.endsWith('/')) return false
|
|
52
|
+
const first = request.charAt(0)
|
|
53
|
+
if (first === '' || './#@'.includes(first)) return false
|
|
54
|
+
const stem = request.slice(0, -1)
|
|
55
|
+
return stem !== '' && !stem.includes('/') && isBuiltin(stem)
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Probe outcome once known: `true` = shim installed or verified unneeded. */
|
|
59
|
+
let settled = false
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Make `require("<deprecated builtin>/")` resolvable under hosts whose
|
|
63
|
+
* profile-scope resolution interceptor mishandles it. Idempotent, synchronous,
|
|
64
|
+
* and a no-op on healthy resolvers (the probe resolves successfully and no
|
|
65
|
+
* wrapper is installed). Must run before the first `require('jsdom')`.
|
|
66
|
+
*/
|
|
67
|
+
export function installBuiltinSlashResolveCompat(): void {
|
|
68
|
+
if (settled) return
|
|
69
|
+
settled = true
|
|
70
|
+
const ModuleClass = require_('node:module').Module as {
|
|
71
|
+
_resolveFilename?: (request: string, parent: unknown, main?: boolean, options?: { paths?: string[] }) => string
|
|
72
|
+
}
|
|
73
|
+
const original = ModuleClass._resolveFilename
|
|
74
|
+
if (typeof original !== 'function') return
|
|
75
|
+
|
|
76
|
+
// Probe with the exact request that breaks. Resolving it (or failing with
|
|
77
|
+
// plain MODULE_NOT_FOUND on a tree that lacks the npm twin — where a
|
|
78
|
+
// wrapper could not help either) means the host needs no help.
|
|
79
|
+
try {
|
|
80
|
+
original.call(ModuleClass, 'punycode/', { filename: SELF_FILENAME })
|
|
81
|
+
return
|
|
82
|
+
} catch (error) {
|
|
83
|
+
if (!(error instanceof TypeError)) return
|
|
84
|
+
/* the compat case: install the wrapper below */
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// The anchor list a native lookup would use from this module's position:
|
|
88
|
+
// this plugin's nested node_modules first, then the profile's hoisted root
|
|
89
|
+
// where `punycode` actually lives in a pnpm install.
|
|
90
|
+
const searchPaths = require_.resolve.paths('jsdom') ?? []
|
|
91
|
+
|
|
92
|
+
ModuleClass._resolveFilename = function patchedResolveFilename(
|
|
93
|
+
this: unknown, request: string, parent: unknown, main?: boolean, options?: { paths?: string[] },
|
|
94
|
+
): string {
|
|
95
|
+
if (options?.paths === undefined && typeof request === 'string' && isBuiltinSlashRequest(request)) {
|
|
96
|
+
// `paths` set → every layer in the chain (including the host's) defers
|
|
97
|
+
// to the native resolver against exactly these anchors.
|
|
98
|
+
return original.call(ModuleClass, request, parent, main, { ...options, paths: searchPaths })
|
|
99
|
+
}
|
|
100
|
+
return original.call(ModuleClass, request, parent, main, options)
|
|
101
|
+
}
|
|
102
|
+
}
|
package/src/markdown.ts
CHANGED
|
@@ -11,16 +11,46 @@
|
|
|
11
11
|
* options and span-safe table rules as the shipped `dsh-tool-web` renderer,
|
|
12
12
|
* so output is consistent with what `web_fetch` produces elsewhere.
|
|
13
13
|
*
|
|
14
|
-
*
|
|
14
|
+
* Synchronous for its callers and unit-tested against fixture pages: jsdom
|
|
15
|
+
* itself is loaded lazily on the first call — and, inside a dsh profile, only
|
|
16
|
+
* after `installBuiltinSlashResolveCompat`, so a host resolver that
|
|
17
|
+
* mishandles tr46's `require("punycode/")` cannot take the plugin down.
|
|
15
18
|
*
|
|
16
19
|
* @module dsh-web-fetch-playwright/markdown
|
|
17
20
|
*/
|
|
18
21
|
|
|
19
|
-
import {
|
|
22
|
+
import { createRequire } from 'node:module'
|
|
20
23
|
import createDOMPurify from 'dompurify'
|
|
21
24
|
import { Readability } from '@mozilla/readability'
|
|
22
25
|
import TurndownService from 'turndown'
|
|
23
26
|
import { gfm } from '@joplin/turndown-plugin-gfm'
|
|
27
|
+
import { installBuiltinSlashResolveCompat } from './host-resolve-compat.ts'
|
|
28
|
+
|
|
29
|
+
/** `createRequire` anchored at this module, so jsdom resolves from this install. */
|
|
30
|
+
const require_ = createRequire(import.meta.url)
|
|
31
|
+
|
|
32
|
+
/** The lazily loaded jsdom exports; jsdom is CommonJS, so loading stays synchronous. */
|
|
33
|
+
type JsdomModule = typeof import('jsdom')
|
|
34
|
+
|
|
35
|
+
/** Loaded jsdom module, cached after the first denoise. */
|
|
36
|
+
let jsdom: JsdomModule | undefined
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Load jsdom on first use, installing the host resolve compat first.
|
|
40
|
+
*
|
|
41
|
+
* Deferral serves two purposes: plugin activation never depends on jsdom's
|
|
42
|
+
* import chain surviving the host's module-resolution interception, and the
|
|
43
|
+
* compat shim — which must wrap the resolver *before* tr46's
|
|
44
|
+
* `require("punycode/")` runs — gets a guaranteed installation point.
|
|
45
|
+
* @returns the jsdom module exports.
|
|
46
|
+
*/
|
|
47
|
+
function loadJsdom(): JsdomModule {
|
|
48
|
+
if (jsdom === undefined) {
|
|
49
|
+
installBuiltinSlashResolveCompat()
|
|
50
|
+
jsdom = require_('jsdom') as JsdomModule
|
|
51
|
+
}
|
|
52
|
+
return jsdom
|
|
53
|
+
}
|
|
24
54
|
|
|
25
55
|
/** Layout/noise tags DOMPurify removes outright (with their content). */
|
|
26
56
|
const FORBID_TAGS = [
|
|
@@ -157,6 +187,7 @@ export interface DenoiseResult {
|
|
|
157
187
|
* @returns the markdown and the extraction mode used.
|
|
158
188
|
*/
|
|
159
189
|
export function htmlToMarkdown(html: string, url: string): DenoiseResult {
|
|
190
|
+
const { JSDOM, VirtualConsole } = loadJsdom()
|
|
160
191
|
// jsdom's virtual console defaults to forwarding parse noise; a silent one
|
|
161
192
|
// keeps broken inline CSS on random pages out of host logs.
|
|
162
193
|
const dom = new JSDOM(html, { url, virtualConsole: new VirtualConsole() })
|