@pagepocket/lighterceptor 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +137 -0
- package/dist/examples/aggregate-requests.d.ts +2 -0
- package/dist/examples/aggregate-requests.d.ts.map +1 -0
- package/dist/examples/aggregate-requests.js +34 -0
- package/dist/examples/aggregate-requests.js.map +1 -0
- package/dist/examples/basic-lighterceptor.d.ts +2 -0
- package/dist/examples/basic-lighterceptor.d.ts.map +1 -0
- package/dist/examples/basic-lighterceptor.js +43 -0
- package/dist/examples/basic-lighterceptor.js.map +1 -0
- package/dist/examples/custom-interceptor.d.ts +2 -0
- package/dist/examples/custom-interceptor.d.ts.map +1 -0
- package/dist/examples/custom-interceptor.js +57 -0
- package/dist/examples/custom-interceptor.js.map +1 -0
- package/dist/examples/real-world-moon.d.ts +2 -0
- package/dist/examples/real-world-moon.d.ts.map +1 -0
- package/dist/examples/real-world-moon.js +69 -0
- package/dist/examples/real-world-moon.js.map +1 -0
- package/dist/examples/recursive-crawl.d.ts +2 -0
- package/dist/examples/recursive-crawl.d.ts.map +1 -0
- package/dist/examples/recursive-crawl.js +87 -0
- package/dist/examples/recursive-crawl.js.map +1 -0
- package/dist/src/dom.d.ts +9 -0
- package/dist/src/dom.d.ts.map +1 -0
- package/dist/src/dom.js +608 -0
- package/dist/src/dom.js.map +1 -0
- package/dist/src/index.d.ts +6 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/index.js +3 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/lighterceptor.d.ts +40 -0
- package/dist/src/lighterceptor.d.ts.map +1 -0
- package/dist/src/lighterceptor.js +697 -0
- package/dist/src/lighterceptor.js.map +1 -0
- package/dist/src/resource-loader.d.ts +8 -0
- package/dist/src/resource-loader.d.ts.map +1 -0
- package/dist/src/resource-loader.js +43 -0
- package/dist/src/resource-loader.js.map +1 -0
- package/dist/src/types.d.ts +10 -0
- package/dist/src/types.d.ts.map +1 -0
- package/dist/src/types.js +2 -0
- package/dist/src/types.js.map +1 -0
- package/package.json +30 -0
package/README.md
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# lighterceptor
|
|
2
|
+
|
|
3
|
+
Capture the outbound requests an HTML/JS/CSS payload would trigger in a
|
|
4
|
+
jsdom environment, without hitting the network.
|
|
5
|
+
|
|
6
|
+
## Install
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
pnpm add lighterceptor
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## Quick Start
|
|
13
|
+
|
|
14
|
+
```ts
|
|
15
|
+
import { Lighterceptor } from "lighterceptor";
|
|
16
|
+
|
|
17
|
+
const html = `
|
|
18
|
+
<!doctype html>
|
|
19
|
+
<html>
|
|
20
|
+
<head>
|
|
21
|
+
<link rel="stylesheet" href="https://cdn.example.com/site.css" />
|
|
22
|
+
</head>
|
|
23
|
+
<body>
|
|
24
|
+
<img src="https://cdn.example.com/logo.png" />
|
|
25
|
+
<script>
|
|
26
|
+
fetch("https://api.example.com/search?q=lighterceptor");
|
|
27
|
+
</script>
|
|
28
|
+
</body>
|
|
29
|
+
</html>
|
|
30
|
+
`;
|
|
31
|
+
|
|
32
|
+
const result = await new Lighterceptor(html).run();
|
|
33
|
+
console.log(result.requests);
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Recursion (Dependency Graph)
|
|
37
|
+
|
|
38
|
+
Enable recursion to keep walking JS/CSS/HTML dependencies. This is useful
|
|
39
|
+
when HTML loads CSS/JS, and those assets load more assets.
|
|
40
|
+
|
|
41
|
+
```ts
|
|
42
|
+
import { Lighterceptor } from "lighterceptor";
|
|
43
|
+
|
|
44
|
+
const html = `
|
|
45
|
+
<!doctype html>
|
|
46
|
+
<link rel="stylesheet" href="https://example.com/site.css" />
|
|
47
|
+
<script src="https://example.com/app.js"></script>
|
|
48
|
+
`;
|
|
49
|
+
|
|
50
|
+
const result = await new Lighterceptor(html, { recursion: true }).run();
|
|
51
|
+
console.log(result.requests.map((item) => item.url));
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Recursion uses the global `fetch` to load discovered resources. In tests or
|
|
55
|
+
offline usage, stub `fetch` to return deterministic content.
|
|
56
|
+
|
|
57
|
+
## API
|
|
58
|
+
|
|
59
|
+
### Lighterceptor
|
|
60
|
+
|
|
61
|
+
```ts
|
|
62
|
+
type LighterceptorOptions = {
|
|
63
|
+
settleTimeMs?: number;
|
|
64
|
+
recursion?: boolean;
|
|
65
|
+
requestOnly?: boolean;
|
|
66
|
+
baseUrl?: string;
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
type RequestRecord = {
|
|
70
|
+
url: string;
|
|
71
|
+
source: "resource" | "img" | "css" | "fetch" | "xhr" | "unknown";
|
|
72
|
+
timestamp: number;
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
type ResponseRecord = {
|
|
76
|
+
status: number;
|
|
77
|
+
statusText: string;
|
|
78
|
+
headers: Record<string, string>;
|
|
79
|
+
body: string;
|
|
80
|
+
bodyEncoding: "text" | "base64";
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
type NetworkRecord = {
|
|
84
|
+
url: string;
|
|
85
|
+
source: "resource" | "img" | "css" | "fetch" | "xhr" | "unknown";
|
|
86
|
+
method: string;
|
|
87
|
+
timestamp: number;
|
|
88
|
+
response?: ResponseRecord;
|
|
89
|
+
error?: string;
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
type LighterceptorResult = {
|
|
93
|
+
title?: string;
|
|
94
|
+
capturedAt: string;
|
|
95
|
+
requests: RequestRecord[];
|
|
96
|
+
networkRecords?: NetworkRecord[];
|
|
97
|
+
};
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
- `settleTimeMs`: wait time before the run finishes, so script-driven requests
|
|
101
|
+
can be captured.
|
|
102
|
+
- `recursion`: when true, fetches JS/CSS/HTML resources and applies the same
|
|
103
|
+
interception logic to their dependencies.
|
|
104
|
+
- `requestOnly`: when true, skips response capture and only records request
|
|
105
|
+
metadata.
|
|
106
|
+
- `baseUrl`: absolute URL used to resolve `/path` URLs and other relative
|
|
107
|
+
references.
|
|
108
|
+
|
|
109
|
+
### createJSDOMWithInterceptor
|
|
110
|
+
|
|
111
|
+
Use this when you need low-level access to jsdom.
|
|
112
|
+
|
|
113
|
+
```ts
|
|
114
|
+
import { createJSDOMWithInterceptor } from "lighterceptor";
|
|
115
|
+
|
|
116
|
+
const dom = createJSDOMWithInterceptor({
|
|
117
|
+
html: "<img src='https://example.com/logo.png'>",
|
|
118
|
+
domOptions: {
|
|
119
|
+
pretendToBeVisual: true,
|
|
120
|
+
runScripts: "dangerously",
|
|
121
|
+
},
|
|
122
|
+
interceptor: (url, options) => {
|
|
123
|
+
console.log("Intercepted", url, options.source);
|
|
124
|
+
return "";
|
|
125
|
+
},
|
|
126
|
+
});
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
## Examples
|
|
130
|
+
|
|
131
|
+
See the `examples/` directory for more:
|
|
132
|
+
|
|
133
|
+
- `examples/basic-lighterceptor.ts`
|
|
134
|
+
- `examples/custom-interceptor.ts`
|
|
135
|
+
- `examples/aggregate-requests.ts`
|
|
136
|
+
- `examples/recursive-crawl.ts`
|
|
137
|
+
- `examples/real-world-moon.ts`
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"aggregate-requests.d.ts","sourceRoot":"","sources":["../../examples/aggregate-requests.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { Lighterceptor } from "../src/index";
|
|
2
|
+
const html = `
|
|
3
|
+
<!doctype html>
|
|
4
|
+
<html>
|
|
5
|
+
<head>
|
|
6
|
+
<link rel="stylesheet" href="https://cdn.example.com/base.css" />
|
|
7
|
+
</head>
|
|
8
|
+
<body>
|
|
9
|
+
<img src="https://cdn.example.com/icons/icon-1.png" />
|
|
10
|
+
<img src="https://cdn.example.com/icons/icon-2.png" />
|
|
11
|
+
<script>
|
|
12
|
+
fetch("https://api.example.com/v1/search");
|
|
13
|
+
fetch("https://api.example.com/v1/profile");
|
|
14
|
+
</script>
|
|
15
|
+
</body>
|
|
16
|
+
</html>
|
|
17
|
+
`;
|
|
18
|
+
async function run() {
|
|
19
|
+
const result = await new Lighterceptor(html).run();
|
|
20
|
+
// Aggregate counts by source so you can spot the noisier channels quickly.
|
|
21
|
+
const counts = new Map();
|
|
22
|
+
for (const request of result.requests) {
|
|
23
|
+
const current = counts.get(request.source) ?? 0;
|
|
24
|
+
counts.set(request.source, current + 1);
|
|
25
|
+
}
|
|
26
|
+
// Sort the summary for predictable output in logs or docs.
|
|
27
|
+
const summary = [...counts.entries()]
|
|
28
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
29
|
+
.map(([source, total]) => `${source}: ${total}`);
|
|
30
|
+
console.log("Request summary:", summary);
|
|
31
|
+
console.log("Full request list:", result.requests);
|
|
32
|
+
}
|
|
33
|
+
void run();
|
|
34
|
+
//# sourceMappingURL=aggregate-requests.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"aggregate-requests.js","sourceRoot":"","sources":["../../examples/aggregate-requests.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,IAAI,GAAG;;;;;;;;;;;;;;;CAeZ,CAAC;AAEF,KAAK,UAAU,GAAG;IAChB,MAAM,MAAM,GAAG,MAAM,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;IAEnD,2EAA2E;IAC3E,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,2DAA2D;IAC3D,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SAClC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;SACtC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;IAEnD,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;AACrD,CAAC;AAED,KAAK,GAAG,EAAE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"basic-lighterceptor.d.ts","sourceRoot":"","sources":["../../examples/basic-lighterceptor.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { Lighterceptor } from "../src/index";
|
|
2
|
+
// This example uses the high-level Lighterceptor wrapper to scan HTML and
|
|
3
|
+
// capture any outbound requests it would have triggered in a browser.
|
|
4
|
+
const html = `
|
|
5
|
+
<!doctype html>
|
|
6
|
+
<html>
|
|
7
|
+
<head>
|
|
8
|
+
<title>Landing Page</title>
|
|
9
|
+
<link rel="stylesheet" href="https://cdn.example.com/site.css" />
|
|
10
|
+
<style>
|
|
11
|
+
@import url("https://cdn.example.com/fonts.css");
|
|
12
|
+
.hero {
|
|
13
|
+
background-image: url("https://cdn.example.com/hero-bg.jpg");
|
|
14
|
+
}
|
|
15
|
+
</style>
|
|
16
|
+
</head>
|
|
17
|
+
<body>
|
|
18
|
+
<img src="https://cdn.example.com/logo.png" />
|
|
19
|
+
<script>
|
|
20
|
+
// Script-driven URLs are captured too once scripts are allowed to run.
|
|
21
|
+
fetch("https://api.example.com/search?q=lighterceptor");
|
|
22
|
+
const xhr = new XMLHttpRequest();
|
|
23
|
+
xhr.open("GET", "https://api.example.com/legacy");
|
|
24
|
+
xhr.send();
|
|
25
|
+
|
|
26
|
+
const hero = document.createElement("div");
|
|
27
|
+
hero.className = "hero";
|
|
28
|
+
hero.style.backgroundImage = "url(https://cdn.example.com/hero-2.jpg)";
|
|
29
|
+
document.body.appendChild(hero);
|
|
30
|
+
</script>
|
|
31
|
+
</body>
|
|
32
|
+
</html>
|
|
33
|
+
`;
|
|
34
|
+
async function run() {
|
|
35
|
+
// The settle time gives scripts a moment to execute before we snapshot results.
|
|
36
|
+
const interceptor = new Lighterceptor(html, { settleTimeMs: 75 });
|
|
37
|
+
const result = await interceptor.run();
|
|
38
|
+
console.log("Captured title:", result.title);
|
|
39
|
+
console.log("Captured at:", result.capturedAt);
|
|
40
|
+
console.log("Requests:", result.requests);
|
|
41
|
+
}
|
|
42
|
+
void run();
|
|
43
|
+
//# sourceMappingURL=basic-lighterceptor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"basic-lighterceptor.js","sourceRoot":"","sources":["../../examples/basic-lighterceptor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,0EAA0E;AAC1E,sEAAsE;AACtE,MAAM,IAAI,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6BZ,CAAC;AAEF,KAAK,UAAU,GAAG;IAChB,gFAAgF;IAChF,MAAM,WAAW,GAAG,IAAI,aAAa,CAAC,IAAI,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,CAAC;IAEvC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;AAC5C,CAAC;AAED,KAAK,GAAG,EAAE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"custom-interceptor.d.ts","sourceRoot":"","sources":["../../examples/custom-interceptor.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { createJSDOMWithInterceptor } from "../src/index";
|
|
2
|
+
const captured = [];
|
|
3
|
+
const dom = createJSDOMWithInterceptor({
|
|
4
|
+
html: `
|
|
5
|
+
<!doctype html>
|
|
6
|
+
<html>
|
|
7
|
+
<head>
|
|
8
|
+
<link rel="stylesheet" href="https://assets.example.com/theme.css" />
|
|
9
|
+
</head>
|
|
10
|
+
<body>
|
|
11
|
+
<p>Custom interceptor demo</p>
|
|
12
|
+
</body>
|
|
13
|
+
</html>
|
|
14
|
+
`,
|
|
15
|
+
domOptions: {
|
|
16
|
+
pretendToBeVisual: true,
|
|
17
|
+
runScripts: "dangerously",
|
|
18
|
+
beforeParse(window) {
|
|
19
|
+
// Provide safe stubs so fetch/XHR don't touch the network during examples.
|
|
20
|
+
window.fetch = () => Promise.resolve({ ok: true });
|
|
21
|
+
window.XMLHttpRequest.prototype.send = function send() { };
|
|
22
|
+
}
|
|
23
|
+
},
|
|
24
|
+
interceptor: (url, options) => {
|
|
25
|
+
captured.push({
|
|
26
|
+
url,
|
|
27
|
+
source: options.source ?? "unknown"
|
|
28
|
+
});
|
|
29
|
+
// Pretend we already have the stylesheet and return it immediately.
|
|
30
|
+
if (url.endsWith("/theme.css")) {
|
|
31
|
+
return `
|
|
32
|
+
body {
|
|
33
|
+
background-image: url("https://assets.example.com/paper.png");
|
|
34
|
+
}
|
|
35
|
+
`;
|
|
36
|
+
}
|
|
37
|
+
// Returning an empty string skips any network fetch for other assets.
|
|
38
|
+
return "";
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
async function run() {
|
|
42
|
+
const { document } = dom.window;
|
|
43
|
+
// This image will be intercepted when its src is assigned.
|
|
44
|
+
const badge = document.createElement("img");
|
|
45
|
+
badge.setAttribute("src", "https://assets.example.com/badge.png");
|
|
46
|
+
document.body.appendChild(badge);
|
|
47
|
+
// Setting a style property with a url() also triggers interception.
|
|
48
|
+
document.body.style.setProperty("background-image", "url(https://assets.example.com/texture.png)");
|
|
49
|
+
// Explicit fetch and XHR calls are still captured by the interceptor.
|
|
50
|
+
await dom.window.fetch("https://api.example.com/custom");
|
|
51
|
+
const xhr = new dom.window.XMLHttpRequest();
|
|
52
|
+
xhr.open("GET", "https://api.example.com/xhr");
|
|
53
|
+
xhr.send();
|
|
54
|
+
console.log("Captured requests:", captured);
|
|
55
|
+
}
|
|
56
|
+
void run();
|
|
57
|
+
//# sourceMappingURL=custom-interceptor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"custom-interceptor.js","sourceRoot":"","sources":["../../examples/custom-interceptor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,0BAA0B,EAAE,MAAM,cAAc,CAAC;AAO1D,MAAM,QAAQ,GAAsB,EAAE,CAAC;AAEvC,MAAM,GAAG,GAAG,0BAA0B,CAAC;IACrC,IAAI,EAAE;;;;;;;;;;GAUL;IACD,UAAU,EAAE;QACV,iBAAiB,EAAE,IAAI;QACvB,UAAU,EAAE,aAAa;QACzB,WAAW,CAAC,MAAM;YAChB,2EAA2E;YAC3E,MAAM,CAAC,KAAK,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAiC,CAAC;YACnF,MAAM,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,GAAG,SAAS,IAAI,KAAI,CAAC,CAAC;QAC5D,CAAC;KACF;IACD,WAAW,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;QAC5B,QAAQ,CAAC,IAAI,CAAC;YACZ,GAAG;YACH,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,SAAS;SACpC,CAAC,CAAC;QAEH,oEAAoE;QACpE,IAAI,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;YAC/B,OAAO;;;;OAIN,CAAC;QACJ,CAAC;QAED,sEAAsE;QACtE,OAAO,EAAE,CAAC;IACZ,CAAC;CACF,CAAC,CAAC;AAEH,KAAK,UAAU,GAAG;IAChB,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;IAEhC,2DAA2D;IAC3D,MAAM,KAAK,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC5C,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,sCAAsC,CAAC,CAAC;IAClE,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;IAEjC,oEAAoE;IACpE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAC7B,kBAAkB,EAClB,6CAA6C,CAC9C,CAAC;IAEF,sEAAsE;IACtE,MAAM,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACzD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;IAC5C,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,6BAA6B,CAAC,CAAC;IAC/C,GAAG,CAAC,IAAI,EAAE,CAAC;IAEX,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,QAAQ,CAAC,CAAC;AAC9C,CAAC;AAED,KAAK,GAAG,EAAE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"real-world-moon.d.ts","sourceRoot":"","sources":["../../examples/real-world-moon.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import { Lighterceptor } from "../src/index";
|
|
3
|
+
const targetUrl = "https://ciechanow.ski/moon/";
|
|
4
|
+
const snapshotPath = process.env.MOON_SNAPSHOT_PATH ??
|
|
5
|
+
"/home/mist/Repositories/pagepocket/resources/Moon_Bartosz_Ciechanowski.requests.json";
|
|
6
|
+
const outputPath = process.env.MOON_OUTPUT_PATH ??
|
|
7
|
+
"/home/mist/Repositories/lighterceptor/examples/Moon_Bartosz_Ciechanowski.requests.json";
|
|
8
|
+
function normalizeUrl(url) {
|
|
9
|
+
try {
|
|
10
|
+
const parsed = new URL(url);
|
|
11
|
+
parsed.hash = "";
|
|
12
|
+
return parsed.toString();
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
return url;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
function collectSnapshotUrls(snapshot) {
|
|
19
|
+
const urls = new Set();
|
|
20
|
+
const recordGroups = [snapshot.networkRecords, snapshot.fetchXhrRecords, snapshot.resources];
|
|
21
|
+
for (const group of recordGroups) {
|
|
22
|
+
if (!group) {
|
|
23
|
+
continue;
|
|
24
|
+
}
|
|
25
|
+
for (const record of group) {
|
|
26
|
+
if (!record?.url) {
|
|
27
|
+
continue;
|
|
28
|
+
}
|
|
29
|
+
urls.add(normalizeUrl(record.url));
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return urls;
|
|
33
|
+
}
|
|
34
|
+
async function run() {
|
|
35
|
+
const snapshotRaw = await readFile(snapshotPath, "utf8");
|
|
36
|
+
const snapshot = JSON.parse(snapshotRaw);
|
|
37
|
+
const expectedUrls = collectSnapshotUrls(snapshot);
|
|
38
|
+
const result = await new Lighterceptor(targetUrl, {
|
|
39
|
+
recursion: true
|
|
40
|
+
}).run();
|
|
41
|
+
const actualUrls = new Set(result.requests.map((item) => normalizeUrl(item.url)));
|
|
42
|
+
const intersection = new Set();
|
|
43
|
+
for (const url of actualUrls) {
|
|
44
|
+
if (expectedUrls.has(url)) {
|
|
45
|
+
intersection.add(url);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
const coverage = expectedUrls.size ? intersection.size / expectedUrls.size : 0;
|
|
49
|
+
const missing = [...expectedUrls].filter((url) => !actualUrls.has(url));
|
|
50
|
+
const extra = [...actualUrls].filter((url) => !expectedUrls.has(url));
|
|
51
|
+
console.log("Target URL:", targetUrl);
|
|
52
|
+
console.log("Snapshot:", snapshotPath);
|
|
53
|
+
console.log("Expected URLs:", expectedUrls.size);
|
|
54
|
+
console.log("Captured URLs:", actualUrls.size);
|
|
55
|
+
console.log("Overlap:", intersection.size);
|
|
56
|
+
console.log("Coverage:", `${(coverage * 100).toFixed(2)}%`);
|
|
57
|
+
console.log("Missing sample:", missing.slice(0, 10));
|
|
58
|
+
console.log("Extra sample:", extra.slice(0, 10));
|
|
59
|
+
const networkWithResponse = result.networkRecords?.filter((record) => record.response) ?? [];
|
|
60
|
+
await writeFile(outputPath, JSON.stringify({
|
|
61
|
+
url: targetUrl,
|
|
62
|
+
title: result.title,
|
|
63
|
+
capturedAt: result.capturedAt,
|
|
64
|
+
networkRecords: networkWithResponse
|
|
65
|
+
}, null, 2), "utf8");
|
|
66
|
+
console.log("Wrote:", outputPath);
|
|
67
|
+
}
|
|
68
|
+
void run();
|
|
69
|
+
//# sourceMappingURL=real-world-moon.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"real-world-moon.js","sourceRoot":"","sources":["../../examples/real-world-moon.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAEvD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAS7C,MAAM,SAAS,GAAG,6BAA6B,CAAC;AAChD,MAAM,YAAY,GAChB,OAAO,CAAC,GAAG,CAAC,kBAAkB;IAC9B,sFAAsF,CAAC;AACzF,MAAM,UAAU,GACd,OAAO,CAAC,GAAG,CAAC,gBAAgB;IAC5B,wFAAwF,CAAC;AAE3F,SAAS,YAAY,CAAC,GAAW;IAC/B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACjB,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAkB;IAC7C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,YAAY,GAAG,CAAC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;IAE7F,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,SAAS;QACX,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC;gBACjB,SAAS;YACX,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAa,CAAC;IACrD,MAAM,YAAY,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,MAAM,IAAI,aAAa,CAAC,SAAS,EAAE;QAChD,SAAS,EAAE,IAAI;KAChB,CAAC,CAAC,GAAG,EAAE,CAAC;IACT,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAElF,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,OAAO,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,MAAM,KAAK,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAEtE,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;IACtC,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAEjD,MAAM,mBAAmB,GAAG,MAAM,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IAE7F,MAAM,SAAS,CACb,UAAU,EACV,IAAI,CAAC,SAAS,CACZ;QACE,GAAG,EAAE,SAAS;QACd,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,cAAc,EAAE,mBAAmB;KACpC,EACD,IAAI,EACJ,CAAC,CACF,EACD,MAAM,CACP,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;AACpC,CAAC;AAED,KAAK,GAAG,EAAE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recursive-crawl.d.ts","sourceRoot":"","sources":["../../examples/recursive-crawl.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { Lighterceptor } from "../src/index";
|
|
2
|
+
// This example demonstrates recursive discovery. When recursion is enabled,
|
|
3
|
+
// the interceptor will fetch JS/CSS/HTML resources and walk their dependencies.
|
|
4
|
+
const html = `
|
|
5
|
+
<!doctype html>
|
|
6
|
+
<html>
|
|
7
|
+
<head>
|
|
8
|
+
<link rel="stylesheet" href="https://example.com/site.css" />
|
|
9
|
+
<script src="https://example.com/app.js"></script>
|
|
10
|
+
</head>
|
|
11
|
+
<body>
|
|
12
|
+
<iframe src="https://example.com/frame.html"></iframe>
|
|
13
|
+
</body>
|
|
14
|
+
</html>
|
|
15
|
+
`;
|
|
16
|
+
const resources = new Map([
|
|
17
|
+
[
|
|
18
|
+
"https://example.com/site.css",
|
|
19
|
+
{
|
|
20
|
+
body: '@import url("./theme.css"); .hero{background:url("/hero.png");}',
|
|
21
|
+
contentType: "text/css"
|
|
22
|
+
}
|
|
23
|
+
],
|
|
24
|
+
[
|
|
25
|
+
"https://example.com/theme.css",
|
|
26
|
+
{
|
|
27
|
+
body: ".card{background-image:url(https://example.com/card.png);}",
|
|
28
|
+
contentType: "text/css"
|
|
29
|
+
}
|
|
30
|
+
],
|
|
31
|
+
[
|
|
32
|
+
"https://example.com/app.js",
|
|
33
|
+
{
|
|
34
|
+
body: 'import "./feature.js"; fetch("https://example.com/api/data");',
|
|
35
|
+
contentType: "application/javascript"
|
|
36
|
+
}
|
|
37
|
+
],
|
|
38
|
+
[
|
|
39
|
+
"https://example.com/feature.js",
|
|
40
|
+
{
|
|
41
|
+
body: 'fetch("https://example.com/api/feature");',
|
|
42
|
+
contentType: "application/javascript"
|
|
43
|
+
}
|
|
44
|
+
],
|
|
45
|
+
[
|
|
46
|
+
"https://example.com/frame.html",
|
|
47
|
+
{
|
|
48
|
+
body: '<!doctype html><link rel="stylesheet" href="/frame.css"><img src="/frame.png">',
|
|
49
|
+
contentType: "text/html"
|
|
50
|
+
}
|
|
51
|
+
],
|
|
52
|
+
[
|
|
53
|
+
"https://example.com/frame.css",
|
|
54
|
+
{
|
|
55
|
+
body: ".frame{background:url(https://example.com/frame-bg.png)}",
|
|
56
|
+
contentType: "text/css"
|
|
57
|
+
}
|
|
58
|
+
]
|
|
59
|
+
]);
|
|
60
|
+
async function run() {
|
|
61
|
+
const originalFetch = globalThis.fetch;
|
|
62
|
+
globalThis.fetch = async (input) => {
|
|
63
|
+
const url = typeof input === "string" ? input : input.toString();
|
|
64
|
+
const entry = resources.get(url);
|
|
65
|
+
if (!entry) {
|
|
66
|
+
return new Response("", { status: 404 });
|
|
67
|
+
}
|
|
68
|
+
return new Response(entry.body, {
|
|
69
|
+
status: 200,
|
|
70
|
+
headers: {
|
|
71
|
+
"content-type": entry.contentType
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
};
|
|
75
|
+
try {
|
|
76
|
+
const interceptor = new Lighterceptor(html, { recursion: true });
|
|
77
|
+
const result = await interceptor.run();
|
|
78
|
+
console.log("Requests:", result.requests.map((item) => item.url));
|
|
79
|
+
}
|
|
80
|
+
finally {
|
|
81
|
+
if (originalFetch) {
|
|
82
|
+
globalThis.fetch = originalFetch;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
void run();
|
|
87
|
+
//# sourceMappingURL=recursive-crawl.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recursive-crawl.js","sourceRoot":"","sources":["../../examples/recursive-crawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,4EAA4E;AAC5E,gFAAgF;AAChF,MAAM,IAAI,GAAG;;;;;;;;;;;CAWZ,CAAC;AAEF,MAAM,SAAS,GAAG,IAAI,GAAG,CAAgD;IACvE;QACE,8BAA8B;QAC9B;YACE,IAAI,EAAE,iEAAiE;YACvE,WAAW,EAAE,UAAU;SACxB;KACF;IACD;QACE,+BAA+B;QAC/B;YACE,IAAI,EAAE,4DAA4D;YAClE,WAAW,EAAE,UAAU;SACxB;KACF;IACD;QACE,4BAA4B;QAC5B;YACE,IAAI,EAAE,+DAA+D;YACrE,WAAW,EAAE,wBAAwB;SACtC;KACF;IACD;QACE,gCAAgC;QAChC;YACE,IAAI,EAAE,2CAA2C;YACjD,WAAW,EAAE,wBAAwB;SACtC;KACF;IACD;QACE,gCAAgC;QAChC;YACE,IAAI,EAAE,gFAAgF;YACtF,WAAW,EAAE,WAAW;SACzB;KACF;IACD;QACE,+BAA+B;QAC/B;YACE,IAAI,EAAE,0DAA0D;YAChE,WAAW,EAAE,UAAU;SACxB;KACF;CACF,CAAC,CAAC;AAEH,KAAK,UAAU,GAAG;IAChB,MAAM,aAAa,GAAG,UAAU,CAAC,KAAK,CAAC;IACvC,UAAU,CAAC,KAAK,GAAG,KAAK,EAAE,KAAwB,EAAE,EAAE;QACpD,MAAM,GAAG,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QACjE,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,QAAQ,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE;YAC9B,MAAM,EAAE,GAAG;YACX,OAAO,EAAE;gBACP,cAAc,EAAE,KAAK,CAAC,WAAW;aAClC;SACF,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,IAAI,aAAa,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,CAAC;QACvC,OAAO,CAAC,GAAG,CACT,WAAW,EACX,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CACxC,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,IAAI,aAAa,EAAE,CAAC;YAClB,UAAU,CAAC,KAAK,GAAG,aAAa,CAAC;QACnC,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,GAAG,EAAE,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { JSDOM } from "jsdom";
|
|
2
|
+
import type { RequestInterceptor } from "./types.js";
|
|
3
|
+
export type InterceptorOptions = {
|
|
4
|
+
html: string;
|
|
5
|
+
domOptions?: ConstructorParameters<typeof JSDOM>[1];
|
|
6
|
+
interceptor: RequestInterceptor;
|
|
7
|
+
};
|
|
8
|
+
export declare function createJSDOMWithInterceptor(options: InterceptorOptions): JSDOM;
|
|
9
|
+
//# sourceMappingURL=dom.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dom.d.ts","sourceRoot":"","sources":["../../src/dom.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAkC,MAAM,OAAO,CAAC;AAW9D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAErD,MAAM,MAAM,kBAAkB,GAAG;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,qBAAqB,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,WAAW,EAAE,kBAAkB,CAAC;CACjC,CAAC;AA0HF,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,kBAAkB,SAsdrE"}
|