@pagepocket/lib 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -17
- package/dist/completion.d.ts +4 -0
- package/dist/completion.js +29 -0
- package/dist/content-store.d.ts +21 -0
- package/dist/content-store.js +96 -0
- package/dist/css-rewrite.d.ts +3 -4
- package/dist/css-rewrite.js +48 -49
- package/dist/download-resources.js +2 -5
- package/dist/hack-html.d.ts +1 -1
- package/dist/hack-html.js +1 -1
- package/dist/hackers/replay-xhr.js +26 -10
- package/dist/index.d.ts +8 -3
- package/dist/index.js +19 -5
- package/dist/network-records.d.ts +2 -2
- package/dist/network-records.js +3 -3
- package/dist/network-store.d.ts +51 -0
- package/dist/network-store.js +159 -0
- package/dist/pagepocket.d.ts +6 -15
- package/dist/pagepocket.js +97 -62
- package/dist/path-resolver.d.ts +5 -0
- package/dist/path-resolver.js +92 -0
- package/dist/replay-script.d.ts +11 -1
- package/dist/replay-script.js +156 -173
- package/dist/resource-filter.d.ts +2 -0
- package/dist/resource-filter.js +34 -0
- package/dist/rewrite-links.d.ts +12 -14
- package/dist/rewrite-links.js +185 -197
- package/dist/snapshot-builder.d.ts +15 -0
- package/dist/snapshot-builder.js +275 -0
- package/dist/snapshot.d.ts +2 -0
- package/dist/snapshot.js +12 -0
- package/dist/types.d.ts +182 -36
- package/dist/utils.d.ts +19 -0
- package/dist/utils.js +109 -0
- package/dist/writers.d.ts +3 -0
- package/dist/writers.js +175 -0
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# @pagepocket/lib
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
Core library for capturing a page via NetworkInterceptorAdapter events and
|
|
4
|
+
producing a virtual snapshot (HTML/CSS/JS rewritten to absolute snapshot paths).
|
|
5
|
+
No network fetch happens in the core library.
|
|
6
6
|
|
|
7
7
|
## Install
|
|
8
8
|
|
|
@@ -14,32 +14,81 @@ pnpm add @pagepocket/lib
|
|
|
14
14
|
|
|
15
15
|
```ts
|
|
16
16
|
import { PagePocket } from "@pagepocket/lib";
|
|
17
|
+
import { CdpAdapter } from "@pagepocket/cdp-adapter";
|
|
17
18
|
|
|
18
|
-
const
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
requestsPath: "example.requests.json"
|
|
19
|
+
const interceptor = new CdpAdapter();
|
|
20
|
+
const snapshot = await PagePocket.fromURL("https://example.com").capture({
|
|
21
|
+
interceptor
|
|
22
22
|
});
|
|
23
23
|
|
|
24
|
-
|
|
24
|
+
await snapshot.toDirectory("./out");
|
|
25
25
|
```
|
|
26
26
|
|
|
27
27
|
## API
|
|
28
28
|
|
|
29
29
|
```ts
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
}
|
|
30
|
+
class PagePocket {
|
|
31
|
+
static fromURL(url: string, options?: PagePocketOptions): PagePocket;
|
|
32
|
+
static fromTarget(target: InterceptTarget, options?: PagePocketOptions): PagePocket;
|
|
33
|
+
capture(options?: CaptureOptions): Promise<PageSnapshot>;
|
|
34
|
+
}
|
|
35
35
|
```
|
|
36
36
|
|
|
37
|
-
|
|
38
|
-
- `baseUrl`: used to resolve relative URLs.
|
|
39
|
-
- `requestsPath`: path to the `*.requests.json` file referenced by replay.
|
|
37
|
+
### CaptureOptions (core)
|
|
40
38
|
|
|
41
|
-
|
|
39
|
+
```ts
|
|
40
|
+
interface CaptureOptions {
|
|
41
|
+
interceptor: NetworkInterceptorAdapter;
|
|
42
|
+
completion?: CompletionStrategy | CompletionStrategy[];
|
|
43
|
+
filter?: ResourceFilter;
|
|
44
|
+
pathResolver?: PathResolver;
|
|
45
|
+
contentStore?: ContentStore;
|
|
46
|
+
rewriteEntry?: boolean;
|
|
47
|
+
rewriteCSS?: boolean;
|
|
48
|
+
limits?: {
|
|
49
|
+
maxTotalBytes?: number;
|
|
50
|
+
maxSingleResourceBytes?: number;
|
|
51
|
+
maxResources?: number;
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
### PageSnapshot output
|
|
57
|
+
|
|
58
|
+
```ts
|
|
59
|
+
interface PageSnapshot {
|
|
60
|
+
version: "1.0";
|
|
61
|
+
createdAt: number;
|
|
62
|
+
url: string;
|
|
63
|
+
entry: string;
|
|
64
|
+
files: SnapshotFile[];
|
|
65
|
+
toDirectory(outDir: string, options?: WriteFSOptions): Promise<WriteResult>;
|
|
66
|
+
toZip(options?: ZipOptions): Promise<Uint8Array | Blob>;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
interface WriteFSOptions {
|
|
70
|
+
clearCache?: boolean;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
interface ZipOptions {
|
|
74
|
+
asBlob?: boolean;
|
|
75
|
+
clearCache?: boolean;
|
|
76
|
+
}
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Snapshot layout:
|
|
80
|
+
|
|
81
|
+
```
|
|
82
|
+
/index.html
|
|
83
|
+
/api.json
|
|
84
|
+
/<same-origin paths>
|
|
85
|
+
/external_resources/<cross-origin paths>
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
If multiple documents are captured, each document is written to its own output
|
|
89
|
+
directory based on the document URL path (e.g. `foo/bar/index.html`).
|
|
42
90
|
|
|
43
91
|
## Notes
|
|
44
92
|
|
|
45
93
|
- Uses `@pagepocket/uni-fs` for file IO so it works in Node and OPFS contexts.
|
|
94
|
+
- Network data comes only from the interceptor events.
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { CompletionStrategy } from "./types";
|
|
2
|
+
export declare const timeout: (ms: number) => CompletionStrategy;
|
|
3
|
+
export declare const networkIdle: (ms: number, checkInterval?: number) => CompletionStrategy;
|
|
4
|
+
export declare const normalizeCompletion: (completion?: CompletionStrategy | CompletionStrategy[]) => CompletionStrategy[];
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.normalizeCompletion = exports.networkIdle = exports.timeout = void 0;
|
|
4
|
+
const utils_1 = require("./utils");
|
|
5
|
+
const timeout = (ms) => ({
|
|
6
|
+
async wait() {
|
|
7
|
+
await (0, utils_1.sleep)(ms);
|
|
8
|
+
}
|
|
9
|
+
});
|
|
10
|
+
exports.timeout = timeout;
|
|
11
|
+
const networkIdle = (ms, checkInterval = 100) => ({
|
|
12
|
+
async wait(ctx) {
|
|
13
|
+
while (true) {
|
|
14
|
+
const stats = ctx.getStats();
|
|
15
|
+
const idleFor = ctx.now() - stats.lastNetworkTs;
|
|
16
|
+
if (stats.inflightRequests === 0 && idleFor >= ms) {
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
19
|
+
await (0, utils_1.sleep)(Math.min(checkInterval, ms));
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
});
|
|
23
|
+
exports.networkIdle = networkIdle;
|
|
24
|
+
const normalizeCompletion = (completion) => {
|
|
25
|
+
if (!completion)
|
|
26
|
+
return [];
|
|
27
|
+
return Array.isArray(completion) ? completion : [completion];
|
|
28
|
+
};
|
|
29
|
+
exports.normalizeCompletion = normalizeCompletion;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { BodySource, ContentRef, ContentStore } from "./types";
|
|
2
|
+
type HybridContentStoreOptions = {
|
|
3
|
+
thresholdBytes?: number;
|
|
4
|
+
baseDir?: string;
|
|
5
|
+
};
|
|
6
|
+
export declare class HybridContentStore implements ContentStore {
|
|
7
|
+
name: string;
|
|
8
|
+
private thresholdBytes;
|
|
9
|
+
private baseDir;
|
|
10
|
+
private storedIds;
|
|
11
|
+
constructor(options?: HybridContentStoreOptions);
|
|
12
|
+
put(body: BodySource, meta: {
|
|
13
|
+
url: string;
|
|
14
|
+
mimeType?: string;
|
|
15
|
+
sizeHint?: number;
|
|
16
|
+
}): Promise<ContentRef>;
|
|
17
|
+
open(ref: ContentRef): Promise<ReadableStream<Uint8Array>>;
|
|
18
|
+
dispose(): Promise<void>;
|
|
19
|
+
private removeBaseDir;
|
|
20
|
+
}
|
|
21
|
+
export {};
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.HybridContentStore = void 0;
|
|
37
|
+
const uni_fs_1 = require("@pagepocket/uni-fs");
|
|
38
|
+
const utils_1 = require("./utils");
|
|
39
|
+
const DEFAULT_THRESHOLD = 256 * 1024;
|
|
40
|
+
const DEFAULT_BASE_DIR = ".pagepocket_store";
|
|
41
|
+
const nowId = () => `${Date.now()}_${Math.random().toString(16).slice(2)}`;
|
|
42
|
+
const streamFromBytes = (data) => new ReadableStream({
|
|
43
|
+
start(controller) {
|
|
44
|
+
controller.enqueue(data);
|
|
45
|
+
controller.close();
|
|
46
|
+
}
|
|
47
|
+
});
|
|
48
|
+
const isNodeEnvironment = () => {
|
|
49
|
+
const globalProcess = globalThis.process;
|
|
50
|
+
return typeof globalProcess?.versions?.node === "string";
|
|
51
|
+
};
|
|
52
|
+
class HybridContentStore {
|
|
53
|
+
constructor(options) {
|
|
54
|
+
this.name = "hybrid";
|
|
55
|
+
this.storedIds = new Set();
|
|
56
|
+
this.thresholdBytes = options?.thresholdBytes ?? DEFAULT_THRESHOLD;
|
|
57
|
+
this.baseDir = options?.baseDir ?? DEFAULT_BASE_DIR;
|
|
58
|
+
}
|
|
59
|
+
async put(body, meta) {
|
|
60
|
+
const data = await (0, utils_1.toUint8Array)(body);
|
|
61
|
+
const size = data.byteLength;
|
|
62
|
+
if (size <= this.thresholdBytes) {
|
|
63
|
+
return { kind: "memory", data };
|
|
64
|
+
}
|
|
65
|
+
const id = `${(0, utils_1.hashString)(meta.url)}_${nowId()}`;
|
|
66
|
+
await (0, uni_fs_1.write)(`${this.baseDir}/${id}`, "bin", data);
|
|
67
|
+
this.storedIds.add(id);
|
|
68
|
+
return { kind: "store-ref", id };
|
|
69
|
+
}
|
|
70
|
+
async open(ref) {
|
|
71
|
+
if (ref.kind === "memory") {
|
|
72
|
+
return streamFromBytes(ref.data);
|
|
73
|
+
}
|
|
74
|
+
const data = await (0, uni_fs_1.readBinary)(`${this.baseDir}/${ref.id}`, "bin");
|
|
75
|
+
return streamFromBytes(data);
|
|
76
|
+
}
|
|
77
|
+
async dispose() {
|
|
78
|
+
const entries = Array.from(this.storedIds);
|
|
79
|
+
this.storedIds.clear();
|
|
80
|
+
await Promise.all(entries.map((id) => (0, uni_fs_1.remove)(`${this.baseDir}/${id}`, "bin").catch(() => { })));
|
|
81
|
+
await this.removeBaseDir().catch(() => { });
|
|
82
|
+
}
|
|
83
|
+
async removeBaseDir() {
|
|
84
|
+
if (!isNodeEnvironment()) {
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
if (!this.baseDir || this.baseDir === "/" || this.baseDir === ".") {
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
const { resolve, isAbsolute } = await Promise.resolve().then(() => __importStar(require("node:path")));
|
|
91
|
+
const { rm } = await Promise.resolve().then(() => __importStar(require("node:fs/promises")));
|
|
92
|
+
const target = isAbsolute(this.baseDir) ? this.baseDir : resolve(this.baseDir);
|
|
93
|
+
await rm(target, { recursive: true, force: true });
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
exports.HybridContentStore = HybridContentStore;
|
package/dist/css-rewrite.d.ts
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
export type CssUrlResolver = (absoluteUrl: string) => Promise<string | null
|
|
1
|
+
export type CssUrlResolver = (absoluteUrl: string) => Promise<string | null> | string | null;
|
|
2
2
|
type RewriteCssInput = {
|
|
3
|
-
|
|
4
|
-
extension: string;
|
|
3
|
+
cssText: string;
|
|
5
4
|
cssUrl: string;
|
|
6
5
|
resolveUrl: CssUrlResolver;
|
|
7
6
|
};
|
|
8
|
-
export declare const
|
|
7
|
+
export declare const rewriteCssText: (input: RewriteCssInput) => Promise<string>;
|
|
9
8
|
export {};
|
package/dist/css-rewrite.js
CHANGED
|
@@ -1,76 +1,75 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.
|
|
3
|
+
exports.rewriteCssText = void 0;
|
|
37
4
|
const URL_PATTERN = /url\(\s*(['"]?)([^'")]+)\1\s*\)/g;
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
const
|
|
5
|
+
const IMPORT_PATTERN = /@import\s+(?:url\()?['"]?([^'")]+)['"]?\)?/g;
|
|
6
|
+
const shouldSkipValue = (value) => {
|
|
7
|
+
const trimmed = value.trim();
|
|
8
|
+
return (!trimmed ||
|
|
9
|
+
trimmed.startsWith("data:") ||
|
|
10
|
+
trimmed.startsWith("blob:") ||
|
|
11
|
+
trimmed.startsWith("mailto:") ||
|
|
12
|
+
trimmed.startsWith("tel:") ||
|
|
13
|
+
trimmed.startsWith("javascript:") ||
|
|
14
|
+
trimmed.startsWith("#"));
|
|
15
|
+
};
|
|
16
|
+
const rewriteCssText = async (input) => {
|
|
17
|
+
const { cssText, cssUrl, resolveUrl } = input;
|
|
41
18
|
let updated = "";
|
|
42
19
|
let lastIndex = 0;
|
|
43
|
-
|
|
44
|
-
for (const match of original.matchAll(URL_PATTERN)) {
|
|
20
|
+
for (const match of cssText.matchAll(URL_PATTERN)) {
|
|
45
21
|
const index = match.index ?? 0;
|
|
46
|
-
updated +=
|
|
22
|
+
updated += cssText.slice(lastIndex, index);
|
|
47
23
|
const quote = match[1] || "";
|
|
48
24
|
const rawUrl = String(match[2] || "").trim();
|
|
49
25
|
let replacement = match[0];
|
|
50
|
-
if (
|
|
26
|
+
if (!shouldSkipValue(rawUrl)) {
|
|
51
27
|
const absolute = (() => {
|
|
52
28
|
try {
|
|
53
|
-
return new URL(rawUrl,
|
|
29
|
+
return new URL(rawUrl, cssUrl).toString();
|
|
54
30
|
}
|
|
55
31
|
catch {
|
|
56
32
|
return null;
|
|
57
33
|
}
|
|
58
34
|
})();
|
|
59
35
|
if (absolute) {
|
|
60
|
-
const resolved = await
|
|
36
|
+
const resolved = await resolveUrl(absolute);
|
|
61
37
|
if (resolved) {
|
|
62
38
|
replacement = `url(${quote}${resolved}${quote})`;
|
|
63
|
-
changed = true;
|
|
64
39
|
}
|
|
65
40
|
}
|
|
66
41
|
}
|
|
67
42
|
updated += replacement;
|
|
68
43
|
lastIndex = index + match[0].length;
|
|
69
44
|
}
|
|
70
|
-
updated +=
|
|
71
|
-
|
|
72
|
-
|
|
45
|
+
updated += cssText.slice(lastIndex);
|
|
46
|
+
let final = "";
|
|
47
|
+
lastIndex = 0;
|
|
48
|
+
for (const match of updated.matchAll(IMPORT_PATTERN)) {
|
|
49
|
+
const index = match.index ?? 0;
|
|
50
|
+
final += updated.slice(lastIndex, index);
|
|
51
|
+
const rawUrl = String(match[1] || "").trim();
|
|
52
|
+
let replacement = match[0];
|
|
53
|
+
if (!shouldSkipValue(rawUrl)) {
|
|
54
|
+
const absolute = (() => {
|
|
55
|
+
try {
|
|
56
|
+
return new URL(rawUrl, cssUrl).toString();
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
})();
|
|
62
|
+
if (absolute) {
|
|
63
|
+
const resolved = await resolveUrl(absolute);
|
|
64
|
+
if (resolved) {
|
|
65
|
+
replacement = match[0].replace(rawUrl, resolved);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
final += replacement;
|
|
70
|
+
lastIndex = index + match[0].length;
|
|
73
71
|
}
|
|
74
|
-
|
|
72
|
+
final += updated.slice(lastIndex);
|
|
73
|
+
return final;
|
|
75
74
|
};
|
|
76
|
-
exports.
|
|
75
|
+
exports.rewriteCssText = rewriteCssText;
|
|
@@ -98,16 +98,13 @@ const collectSrcsetUrls = (items, baseUrl) => {
|
|
|
98
98
|
return urls;
|
|
99
99
|
};
|
|
100
100
|
const downloadResources = async (input) => {
|
|
101
|
-
const { write, exists } = await Promise.resolve().then(() => __importStar(require("uni-fs")));
|
|
101
|
+
const { write, exists } = await Promise.resolve().then(() => __importStar(require("@pagepocket/uni-fs")));
|
|
102
102
|
const resourceMap = new Map();
|
|
103
103
|
const resourceMeta = [];
|
|
104
104
|
let downloadedCount = 0;
|
|
105
105
|
let failedCount = 0;
|
|
106
106
|
const srcsetUrls = collectSrcsetUrls(input.srcsetItems, input.baseUrl);
|
|
107
|
-
const candidateUrls = [
|
|
108
|
-
...input.resourceUrls.map((resource) => resource.url),
|
|
109
|
-
...srcsetUrls
|
|
110
|
-
];
|
|
107
|
+
const candidateUrls = [...input.resourceUrls.map((resource) => resource.url), ...srcsetUrls];
|
|
111
108
|
for (const candidate of candidateUrls) {
|
|
112
109
|
if (shouldSkipUrl(candidate)) {
|
|
113
110
|
continue;
|
package/dist/hack-html.d.ts
CHANGED
package/dist/hack-html.js
CHANGED
|
@@ -4,7 +4,7 @@ exports.hackHtml = void 0;
|
|
|
4
4
|
const preload_1 = require("./preload");
|
|
5
5
|
const replay_script_1 = require("./replay-script");
|
|
6
6
|
const hackHtml = (input) => {
|
|
7
|
-
const replayScript = (0, replay_script_1.buildReplayScript)(input.
|
|
7
|
+
const replayScript = (0, replay_script_1.buildReplayScript)(input.apiPath, input.baseUrl);
|
|
8
8
|
const preloadScript = `<script>${(0, preload_1.buildPreloadScript)()}</script>`;
|
|
9
9
|
const head = input.$("head");
|
|
10
10
|
const root = input.$.root();
|
|
@@ -51,13 +51,21 @@ exports.replayXhrResponder = {
|
|
|
51
51
|
defineProp(xhr, "response", responseText);
|
|
52
52
|
defineProp(xhr, "responseText", responseText);
|
|
53
53
|
}
|
|
54
|
+
const makeEvent = (type) => {
|
|
55
|
+
const event = new Event(type);
|
|
56
|
+
try {
|
|
57
|
+
defineProp(event, "target", xhr);
|
|
58
|
+
defineProp(event, "currentTarget", xhr);
|
|
59
|
+
} catch {}
|
|
60
|
+
return event;
|
|
61
|
+
};
|
|
54
62
|
if (typeof xhr.onreadystatechange === "function") xhr.onreadystatechange();
|
|
55
|
-
if (typeof xhr.onload === "function") xhr.onload(
|
|
56
|
-
if (typeof xhr.onloadend === "function") xhr.onloadend(
|
|
63
|
+
if (typeof xhr.onload === "function") xhr.onload(makeEvent("load"));
|
|
64
|
+
if (typeof xhr.onloadend === "function") xhr.onloadend(makeEvent("loadend"));
|
|
57
65
|
if (xhr.dispatchEvent) {
|
|
58
|
-
xhr.dispatchEvent(
|
|
59
|
-
xhr.dispatchEvent(
|
|
60
|
-
xhr.dispatchEvent(
|
|
66
|
+
xhr.dispatchEvent(makeEvent("readystatechange"));
|
|
67
|
+
xhr.dispatchEvent(makeEvent("load"));
|
|
68
|
+
xhr.dispatchEvent(makeEvent("loadend"));
|
|
61
69
|
}
|
|
62
70
|
}, 0);
|
|
63
71
|
return;
|
|
@@ -72,13 +80,21 @@ exports.replayXhrResponder = {
|
|
|
72
80
|
defineProp(xhr, "statusText", statusText);
|
|
73
81
|
defineProp(xhr, "response", "");
|
|
74
82
|
defineProp(xhr, "responseText", "");
|
|
83
|
+
const makeEvent = (type) => {
|
|
84
|
+
const event = new Event(type);
|
|
85
|
+
try {
|
|
86
|
+
defineProp(event, "target", xhr);
|
|
87
|
+
defineProp(event, "currentTarget", xhr);
|
|
88
|
+
} catch {}
|
|
89
|
+
return event;
|
|
90
|
+
};
|
|
75
91
|
if (typeof xhr.onreadystatechange === "function") xhr.onreadystatechange();
|
|
76
|
-
if (typeof xhr.onload === "function") xhr.onload(
|
|
77
|
-
if (typeof xhr.onloadend === "function") xhr.onloadend(
|
|
92
|
+
if (typeof xhr.onload === "function") xhr.onload(makeEvent("load"));
|
|
93
|
+
if (typeof xhr.onloadend === "function") xhr.onloadend(makeEvent("loadend"));
|
|
78
94
|
if (xhr.dispatchEvent) {
|
|
79
|
-
xhr.dispatchEvent(
|
|
80
|
-
xhr.dispatchEvent(
|
|
81
|
-
xhr.dispatchEvent(
|
|
95
|
+
xhr.dispatchEvent(makeEvent("readystatechange"));
|
|
96
|
+
xhr.dispatchEvent(makeEvent("load"));
|
|
97
|
+
xhr.dispatchEvent(makeEvent("loadend"));
|
|
82
98
|
}
|
|
83
99
|
}, 0);
|
|
84
100
|
});
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
export { PagePocket } from "./pagepocket";
|
|
2
|
-
export type { PagePocketOptions } from "./
|
|
3
|
-
export
|
|
2
|
+
export type { ApiRecord, ApiSnapshot, CaptureOptions, CompletionContext, CompletionStrategy, ContentRef, ContentStore, ContentStoreHandle, InterceptOptions, InterceptSession, InterceptTarget, NetworkEvent, NetworkEventHandlers, NetworkInterceptorAdapter, NetworkRequestEvent, NetworkRequestFailedEvent, NetworkResponseEvent, PagePocketOptions, PageSnapshot, PathResolver, ResourceFilter, ResourceType, SnapshotFile, WriteFSOptions, WriteResult, ZipOptions } from "./types";
|
|
3
|
+
export { HybridContentStore } from "./content-store";
|
|
4
|
+
export { createDefaultPathResolver, withPrefixPathResolver } from "./path-resolver";
|
|
5
|
+
export { createDefaultResourceFilter } from "./resource-filter";
|
|
6
|
+
export { networkIdle, timeout } from "./completion";
|
|
4
7
|
export { buildReplayScript } from "./replay-script";
|
|
5
8
|
export { buildPreloadScript } from "./preload";
|
|
6
|
-
export {
|
|
9
|
+
export { rewriteEntryHtml, rewriteJsText } from "./rewrite-links";
|
|
10
|
+
export { rewriteCssText } from "./css-rewrite";
|
|
11
|
+
export { writeToFS, toZip } from "./writers";
|
package/dist/index.js
CHANGED
|
@@ -1,13 +1,27 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.toZip = exports.writeToFS = exports.rewriteCssText = exports.rewriteJsText = exports.rewriteEntryHtml = exports.buildPreloadScript = exports.buildReplayScript = exports.timeout = exports.networkIdle = exports.createDefaultResourceFilter = exports.withPrefixPathResolver = exports.createDefaultPathResolver = exports.HybridContentStore = exports.PagePocket = void 0;
|
|
4
4
|
var pagepocket_1 = require("./pagepocket");
|
|
5
5
|
Object.defineProperty(exports, "PagePocket", { enumerable: true, get: function () { return pagepocket_1.PagePocket; } });
|
|
6
|
+
var content_store_1 = require("./content-store");
|
|
7
|
+
Object.defineProperty(exports, "HybridContentStore", { enumerable: true, get: function () { return content_store_1.HybridContentStore; } });
|
|
8
|
+
var path_resolver_1 = require("./path-resolver");
|
|
9
|
+
Object.defineProperty(exports, "createDefaultPathResolver", { enumerable: true, get: function () { return path_resolver_1.createDefaultPathResolver; } });
|
|
10
|
+
Object.defineProperty(exports, "withPrefixPathResolver", { enumerable: true, get: function () { return path_resolver_1.withPrefixPathResolver; } });
|
|
11
|
+
var resource_filter_1 = require("./resource-filter");
|
|
12
|
+
Object.defineProperty(exports, "createDefaultResourceFilter", { enumerable: true, get: function () { return resource_filter_1.createDefaultResourceFilter; } });
|
|
13
|
+
var completion_1 = require("./completion");
|
|
14
|
+
Object.defineProperty(exports, "networkIdle", { enumerable: true, get: function () { return completion_1.networkIdle; } });
|
|
15
|
+
Object.defineProperty(exports, "timeout", { enumerable: true, get: function () { return completion_1.timeout; } });
|
|
6
16
|
var replay_script_1 = require("./replay-script");
|
|
7
17
|
Object.defineProperty(exports, "buildReplayScript", { enumerable: true, get: function () { return replay_script_1.buildReplayScript; } });
|
|
8
18
|
var preload_1 = require("./preload");
|
|
9
19
|
Object.defineProperty(exports, "buildPreloadScript", { enumerable: true, get: function () { return preload_1.buildPreloadScript; } });
|
|
10
|
-
var
|
|
11
|
-
Object.defineProperty(exports, "
|
|
12
|
-
Object.defineProperty(exports, "
|
|
13
|
-
|
|
20
|
+
var rewrite_links_1 = require("./rewrite-links");
|
|
21
|
+
Object.defineProperty(exports, "rewriteEntryHtml", { enumerable: true, get: function () { return rewrite_links_1.rewriteEntryHtml; } });
|
|
22
|
+
Object.defineProperty(exports, "rewriteJsText", { enumerable: true, get: function () { return rewrite_links_1.rewriteJsText; } });
|
|
23
|
+
var css_rewrite_1 = require("./css-rewrite");
|
|
24
|
+
Object.defineProperty(exports, "rewriteCssText", { enumerable: true, get: function () { return css_rewrite_1.rewriteCssText; } });
|
|
25
|
+
var writers_1 = require("./writers");
|
|
26
|
+
Object.defineProperty(exports, "writeToFS", { enumerable: true, get: function () { return writers_1.writeToFS; } });
|
|
27
|
+
Object.defineProperty(exports, "toZip", { enumerable: true, get: function () { return writers_1.toZip; } });
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { CapturedNetworkRecord, NetworkRecord } from "./types";
|
|
2
2
|
export declare const toDataUrlFromRecord: (record: NetworkRecord) => string | null;
|
|
3
3
|
export declare const findFaviconDataUrl: (records: NetworkRecord[]) => string | null;
|
|
4
|
-
export declare const
|
|
4
|
+
export declare const mapCapturedNetworkRecords: (records: CapturedNetworkRecord[] | undefined) => NetworkRecord[];
|
package/dist/network-records.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.mapCapturedNetworkRecords = exports.findFaviconDataUrl = exports.toDataUrlFromRecord = void 0;
|
|
4
4
|
const getHeaderValue = (headers, name) => {
|
|
5
5
|
for (const key in headers) {
|
|
6
6
|
if (key.toLowerCase() === name.toLowerCase()) {
|
|
@@ -61,7 +61,7 @@ const findFaviconDataUrl = (records) => {
|
|
|
61
61
|
return null;
|
|
62
62
|
};
|
|
63
63
|
exports.findFaviconDataUrl = findFaviconDataUrl;
|
|
64
|
-
const
|
|
64
|
+
const mapCapturedNetworkRecords = (records) => {
|
|
65
65
|
if (!records)
|
|
66
66
|
return [];
|
|
67
67
|
return records.map((record) => {
|
|
@@ -80,4 +80,4 @@ const mapLighterceptorRecords = (records) => {
|
|
|
80
80
|
};
|
|
81
81
|
});
|
|
82
82
|
};
|
|
83
|
-
exports.
|
|
83
|
+
exports.mapCapturedNetworkRecords = mapCapturedNetworkRecords;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { ApiRecord, ContentRef, ContentStore, NetworkEvent, NetworkRequestEvent, NetworkRequestFailedEvent, NetworkResponseEvent, ResourceFilter } from "./types";
|
|
2
|
+
type Limits = {
|
|
3
|
+
maxTotalBytes?: number;
|
|
4
|
+
maxSingleResourceBytes?: number;
|
|
5
|
+
maxResources?: number;
|
|
6
|
+
};
|
|
7
|
+
export type StoredResource = {
|
|
8
|
+
request: NetworkRequestEvent;
|
|
9
|
+
response: NetworkResponseEvent;
|
|
10
|
+
contentRef: ContentRef;
|
|
11
|
+
size: number;
|
|
12
|
+
mimeType?: string;
|
|
13
|
+
};
|
|
14
|
+
type RequestRecord = {
|
|
15
|
+
request: NetworkRequestEvent;
|
|
16
|
+
response?: NetworkResponseEvent;
|
|
17
|
+
failed?: NetworkRequestFailedEvent;
|
|
18
|
+
};
|
|
19
|
+
export type ApiEntry = {
|
|
20
|
+
record: ApiRecord;
|
|
21
|
+
request: NetworkRequestEvent;
|
|
22
|
+
};
|
|
23
|
+
export declare class NetworkStore {
|
|
24
|
+
private contentStore;
|
|
25
|
+
private filter;
|
|
26
|
+
private limits;
|
|
27
|
+
private requests;
|
|
28
|
+
private storedResources;
|
|
29
|
+
private apiEntries;
|
|
30
|
+
private apiRecordIds;
|
|
31
|
+
private warnings;
|
|
32
|
+
private totalBytes;
|
|
33
|
+
constructor(options: {
|
|
34
|
+
contentStore: ContentStore;
|
|
35
|
+
filter: ResourceFilter;
|
|
36
|
+
limits?: Limits;
|
|
37
|
+
});
|
|
38
|
+
getWarnings(): string[];
|
|
39
|
+
getTotals(): {
|
|
40
|
+
totalBytes: number;
|
|
41
|
+
totalFiles: number;
|
|
42
|
+
};
|
|
43
|
+
getResources(): StoredResource[];
|
|
44
|
+
getApiRecords(): ApiRecord[];
|
|
45
|
+
getApiEntries(): ApiEntry[];
|
|
46
|
+
getRequestRecords(): Map<string, RequestRecord>;
|
|
47
|
+
handleEvent(event: NetworkEvent): Promise<void>;
|
|
48
|
+
private recordApiFailure;
|
|
49
|
+
private recordApiResponse;
|
|
50
|
+
}
|
|
51
|
+
export {};
|