@pagepocket/lib 0.7.1 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (261) hide show
  1. package/README.md +7 -6
  2. package/dist/build-snapshot-from-bundle.d.ts +23 -0
  3. package/dist/build-snapshot-from-bundle.js +68 -0
  4. package/dist/builtin-blacklist.js +3 -6
  5. package/dist/bundle/from-network-store.d.ts +10 -0
  6. package/dist/bundle/from-network-store.js +26 -0
  7. package/dist/bundle/types.d.ts +32 -0
  8. package/dist/bundle/types.js +2 -0
  9. package/dist/capture/index.d.ts +14 -0
  10. package/dist/capture/index.js +86 -0
  11. package/dist/capture/memory-content-store.d.ts +4 -0
  12. package/dist/capture/memory-content-store.js +42 -0
  13. package/dist/capture/types.d.ts +61 -0
  14. package/dist/capture/types.js +2 -0
  15. package/dist/content-store.js +3 -8
  16. package/dist/content-type.d.ts +1 -1
  17. package/dist/content-type.js +2 -28
  18. package/dist/core/_impl/completion.d.ts +4 -0
  19. package/dist/core/_impl/completion.js +29 -0
  20. package/dist/core/_impl/content-store.d.ts +21 -0
  21. package/dist/core/_impl/content-store.js +91 -0
  22. package/dist/core/_impl/debug.d.ts +1 -0
  23. package/dist/core/_impl/debug.js +16 -0
  24. package/dist/core/_impl/inflight-tracker.d.ts +19 -0
  25. package/dist/core/_impl/inflight-tracker.js +48 -0
  26. package/dist/core/_impl/pagepocket.d.ts +27 -0
  27. package/dist/core/_impl/pagepocket.js +155 -0
  28. package/dist/core/capture/_impl/memory-content-store.d.ts +4 -0
  29. package/dist/core/capture/_impl/memory-content-store.js +42 -0
  30. package/dist/core/capture/_impl/types.d.ts +61 -0
  31. package/dist/core/capture/_impl/types.js +2 -0
  32. package/dist/core/capture/internal/memory-content-store.d.ts +4 -0
  33. package/dist/core/capture/internal/memory-content-store.js +42 -0
  34. package/dist/core/capture/internal/types.d.ts +61 -0
  35. package/dist/core/capture/internal/types.js +2 -0
  36. package/dist/core/capture/memory-content-store.d.ts +4 -0
  37. package/dist/core/capture/memory-content-store.js +38 -0
  38. package/dist/core/capture/types.d.ts +61 -0
  39. package/dist/core/capture/types.js +1 -0
  40. package/dist/core/completion.d.ts +4 -0
  41. package/dist/core/completion.js +23 -0
  42. package/dist/core/content-store.d.ts +21 -0
  43. package/dist/core/content-store.js +54 -0
  44. package/dist/core/debug.d.ts +1 -0
  45. package/dist/core/debug.js +12 -0
  46. package/dist/core/file-tree-merge.d.ts +2 -0
  47. package/dist/core/file-tree-merge.js +27 -0
  48. package/dist/core/file-tree.d.ts +36 -0
  49. package/dist/core/file-tree.js +1 -0
  50. package/dist/core/inflight-tracker.d.ts +19 -0
  51. package/dist/core/inflight-tracker.js +44 -0
  52. package/dist/core/internal/completion.d.ts +4 -0
  53. package/dist/core/internal/completion.js +29 -0
  54. package/dist/core/internal/content-store.d.ts +21 -0
  55. package/dist/core/internal/content-store.js +91 -0
  56. package/dist/core/internal/debug.d.ts +1 -0
  57. package/dist/core/internal/debug.js +16 -0
  58. package/dist/core/internal/inflight-tracker.d.ts +19 -0
  59. package/dist/core/internal/inflight-tracker.js +48 -0
  60. package/dist/core/internal/pagepocket.d.ts +27 -0
  61. package/dist/core/internal/pagepocket.js +155 -0
  62. package/dist/core/pagepocket.d.ts +38 -0
  63. package/dist/core/pagepocket.js +57 -0
  64. package/dist/core/plugin/_impl/context.d.ts +47 -0
  65. package/dist/core/plugin/_impl/context.js +142 -0
  66. package/dist/core/plugin/_impl/runner.d.ts +12 -0
  67. package/dist/core/plugin/_impl/runner.js +232 -0
  68. package/dist/core/plugin/_impl/types.d.ts +108 -0
  69. package/dist/core/plugin/_impl/types.js +2 -0
  70. package/dist/core/plugin/context.d.ts +47 -0
  71. package/dist/core/plugin/context.js +205 -0
  72. package/dist/core/plugin/internal/context.d.ts +47 -0
  73. package/dist/core/plugin/internal/context.js +142 -0
  74. package/dist/core/plugin/internal/runner.d.ts +12 -0
  75. package/dist/core/plugin/internal/runner.js +232 -0
  76. package/dist/core/plugin/internal/types.d.ts +108 -0
  77. package/dist/core/plugin/internal/types.js +2 -0
  78. package/dist/core/plugin/runner-utils.d.ts +9 -0
  79. package/dist/core/plugin/runner-utils.js +29 -0
  80. package/dist/core/plugin/runner.d.ts +12 -0
  81. package/dist/core/plugin/runner.js +118 -0
  82. package/dist/core/plugin/types.d.ts +117 -0
  83. package/dist/core/plugin/types.js +1 -0
  84. package/dist/core/runtime/types.d.ts +14 -0
  85. package/dist/core/runtime/types.js +2 -0
  86. package/dist/css-rewrite.js +1 -5
  87. package/dist/debug.d.ts +0 -1
  88. package/dist/debug.js +3 -5
  89. package/dist/files/types.d.ts +41 -0
  90. package/dist/files/types.js +2 -0
  91. package/dist/hack-html.js +20 -13
  92. package/dist/hackers/index.d.ts +1 -1
  93. package/dist/hackers/index.js +24 -27
  94. package/dist/hackers/preload-fetch.d.ts +1 -1
  95. package/dist/hackers/preload-fetch.js +1 -4
  96. package/dist/hackers/preload-xhr.d.ts +1 -1
  97. package/dist/hackers/preload-xhr.js +1 -4
  98. package/dist/hackers/replay-beacon.d.ts +1 -1
  99. package/dist/hackers/replay-beacon.js +1 -4
  100. package/dist/hackers/replay-block-text-fragment.d.ts +1 -1
  101. package/dist/hackers/replay-block-text-fragment.js +1 -4
  102. package/dist/hackers/replay-css-proxy.d.ts +1 -1
  103. package/dist/hackers/replay-css-proxy.js +9 -12
  104. package/dist/hackers/replay-dom-rewrite.d.ts +1 -1
  105. package/dist/hackers/replay-dom-rewrite.js +165 -154
  106. package/dist/hackers/replay-eventsource.d.ts +1 -1
  107. package/dist/hackers/replay-eventsource.js +1 -4
  108. package/dist/hackers/replay-fetch.d.ts +1 -1
  109. package/dist/hackers/replay-fetch.js +1 -4
  110. package/dist/hackers/replay-history-path.d.ts +1 -1
  111. package/dist/hackers/replay-history-path.js +1 -4
  112. package/dist/hackers/replay-svg-image.d.ts +1 -1
  113. package/dist/hackers/replay-svg-image.js +1 -4
  114. package/dist/hackers/replay-websocket.d.ts +1 -1
  115. package/dist/hackers/replay-websocket.js +1 -4
  116. package/dist/hackers/replay-xhr.d.ts +1 -1
  117. package/dist/hackers/replay-xhr.js +1 -4
  118. package/dist/hackers/types.js +1 -2
  119. package/dist/index.d.ts +29 -13
  120. package/dist/index.js +23 -44
  121. package/dist/kind-map.d.ts +68 -0
  122. package/dist/kind-map.js +58 -0
  123. package/dist/network-store.js +12 -1
  124. package/dist/pagepocket.d.ts +19 -4
  125. package/dist/pagepocket.js +36 -102
  126. package/dist/path-resolver.d.ts +1 -2
  127. package/dist/path-resolver.js +9 -16
  128. package/dist/plugin/builtins/build-snapshot-plugin.d.ts +5 -0
  129. package/dist/plugin/builtins/build-snapshot-plugin.js +84 -0
  130. package/dist/plugin/builtins/replace-elements-plugin.d.ts +8 -0
  131. package/dist/plugin/builtins/replace-elements-plugin.js +13 -0
  132. package/dist/plugin/builtins/to-directory-plugin.d.ts +7 -0
  133. package/dist/plugin/builtins/to-directory-plugin.js +20 -0
  134. package/dist/plugin/builtins/to-zip-plugin.d.ts +5 -0
  135. package/dist/plugin/builtins/to-zip-plugin.js +19 -0
  136. package/dist/plugin/context.d.ts +47 -0
  137. package/dist/plugin/context.js +142 -0
  138. package/dist/plugin/runner.d.ts +12 -0
  139. package/dist/plugin/runner.js +232 -0
  140. package/dist/plugin/types.d.ts +108 -0
  141. package/dist/plugin/types.js +2 -0
  142. package/dist/plugins/build-files-from-capture.d.ts +5 -0
  143. package/dist/plugins/build-files-from-capture.js +85 -0
  144. package/dist/plugins/build-warc.d.ts +5 -0
  145. package/dist/plugins/build-warc.js +225 -0
  146. package/dist/plugins/builtins/manifest.d.ts +2 -0
  147. package/dist/plugins/builtins/manifest.js +42 -0
  148. package/dist/plugins/builtins/snapshot-directory.d.ts +2 -0
  149. package/dist/plugins/builtins/snapshot-directory.js +24 -0
  150. package/dist/plugins/builtins/snapshot-zip.d.ts +2 -0
  151. package/dist/plugins/builtins/snapshot-zip.js +25 -0
  152. package/dist/plugins/capture-http-lighterceptor.d.ts +5 -0
  153. package/dist/plugins/capture-http-lighterceptor.js +85 -0
  154. package/dist/plugins/capture-http-puppeteer.d.ts +5 -0
  155. package/dist/plugins/capture-http-puppeteer.js +85 -0
  156. package/dist/plugins/host.d.ts +37 -0
  157. package/dist/plugins/host.js +105 -0
  158. package/dist/plugins/index.d.ts +6 -0
  159. package/dist/plugins/index.js +11 -0
  160. package/dist/plugins/ordering.d.ts +2 -0
  161. package/dist/plugins/ordering.js +19 -0
  162. package/dist/plugins/types.d.ts +51 -0
  163. package/dist/plugins/types.js +2 -0
  164. package/dist/preload.js +3 -7
  165. package/dist/replace-elements/actions.d.ts +5 -0
  166. package/dist/replace-elements/actions.js +86 -0
  167. package/dist/replace-elements/match.d.ts +5 -0
  168. package/dist/replace-elements/match.js +46 -0
  169. package/dist/replace-elements/normalize.d.ts +21 -0
  170. package/dist/replace-elements/normalize.js +50 -0
  171. package/dist/replace-elements.d.ts +1 -1
  172. package/dist/replace-elements.js +5 -185
  173. package/dist/replay/match-api.d.ts +10 -0
  174. package/dist/replay/match-api.js +162 -0
  175. package/dist/replay/templates/match-api-source.d.ts +1 -0
  176. package/dist/replay/templates/match-api-source.js +137 -0
  177. package/dist/replay/templates/replay-script-template.d.ts +5 -0
  178. package/dist/replay/templates/replay-script-template.js +337 -0
  179. package/dist/replay/templates/resource-proxy-script.d.ts +1 -0
  180. package/dist/replay/templates/resource-proxy-script.js +274 -0
  181. package/dist/replay-script.d.ts +3 -10
  182. package/dist/replay-script.js +11 -625
  183. package/dist/resource-filter.d.ts +1 -1
  184. package/dist/resource-filter.js +1 -5
  185. package/dist/resource-proxy/escape-percent.d.ts +1 -0
  186. package/dist/resource-proxy/escape-percent.js +12 -0
  187. package/dist/resource-proxy/multimap.d.ts +3 -0
  188. package/dist/resource-proxy/multimap.js +18 -0
  189. package/dist/resource-proxy/pathname-variants.d.ts +3 -0
  190. package/dist/resource-proxy/pathname-variants.js +54 -0
  191. package/dist/resource-proxy.d.ts +4 -2
  192. package/dist/resource-proxy.js +48 -117
  193. package/dist/resources.js +4 -42
  194. package/dist/rewrite-links/js-imports.d.ts +3 -0
  195. package/dist/rewrite-links/js-imports.js +56 -0
  196. package/dist/rewrite-links/link-rel.d.ts +2 -0
  197. package/dist/rewrite-links/link-rel.js +10 -0
  198. package/dist/rewrite-links/meta-refresh.d.ts +3 -0
  199. package/dist/rewrite-links/meta-refresh.js +22 -0
  200. package/dist/rewrite-links/skip.d.ts +1 -0
  201. package/dist/rewrite-links/skip.js +10 -0
  202. package/dist/rewrite-links/srcset.d.ts +3 -0
  203. package/dist/rewrite-links/srcset.js +63 -0
  204. package/dist/rewrite-links/url-resolve.d.ts +3 -0
  205. package/dist/rewrite-links/url-resolve.js +13 -0
  206. package/dist/rewrite-links.d.ts +3 -3
  207. package/dist/rewrite-links.js +31 -240
  208. package/dist/snapshot-builder/api.d.ts +3 -0
  209. package/dist/snapshot-builder/api.js +6 -0
  210. package/dist/snapshot-builder/build-snapshot.d.ts +3 -0
  211. package/dist/snapshot-builder/build-snapshot.js +138 -0
  212. package/dist/snapshot-builder/capture-index/index-capture.d.ts +13 -0
  213. package/dist/snapshot-builder/capture-index/index-capture.js +168 -0
  214. package/dist/snapshot-builder/capture-index/index.d.ts +2 -0
  215. package/dist/snapshot-builder/capture-index/index.js +1 -0
  216. package/dist/snapshot-builder/capture-index/types.d.ts +12 -0
  217. package/dist/snapshot-builder/capture-index/types.js +1 -0
  218. package/dist/snapshot-builder/capture-index.d.ts +12 -0
  219. package/dist/snapshot-builder/capture-index.js +173 -0
  220. package/dist/snapshot-builder/emit-document.d.ts +24 -0
  221. package/dist/snapshot-builder/emit-document.js +50 -0
  222. package/dist/snapshot-builder/grouping.d.ts +8 -0
  223. package/dist/snapshot-builder/grouping.js +87 -0
  224. package/dist/snapshot-builder/http.d.ts +6 -0
  225. package/dist/snapshot-builder/http.js +28 -0
  226. package/dist/snapshot-builder/index.d.ts +4 -0
  227. package/dist/snapshot-builder/index.js +2 -0
  228. package/dist/snapshot-builder/path-map.d.ts +3 -0
  229. package/dist/snapshot-builder/path-map.js +35 -0
  230. package/dist/snapshot-builder/resources-path.d.ts +23 -0
  231. package/dist/snapshot-builder/resources-path.js +47 -0
  232. package/dist/snapshot-builder/rewrite-resource.d.ts +18 -0
  233. package/dist/snapshot-builder/rewrite-resource.js +52 -0
  234. package/dist/snapshot-builder/types.d.ts +37 -0
  235. package/dist/snapshot-builder/types.js +2 -0
  236. package/dist/snapshot-builder.d.ts +12 -8
  237. package/dist/snapshot-builder.js +252 -27
  238. package/dist/types.d.ts +122 -78
  239. package/dist/types.js +4 -2
  240. package/dist/units/contracts-bridge.d.ts +76 -0
  241. package/dist/units/contracts-bridge.js +6 -0
  242. package/dist/units/index.d.ts +4 -0
  243. package/dist/units/index.js +2 -0
  244. package/dist/units/runner.d.ts +11 -0
  245. package/dist/units/runner.js +270 -0
  246. package/dist/units/types.d.ts +39 -0
  247. package/dist/units/types.js +1 -0
  248. package/dist/utils/streams.d.ts +2 -0
  249. package/dist/utils/streams.js +29 -0
  250. package/dist/utils.d.ts +35 -1
  251. package/dist/utils.js +107 -29
  252. package/dist/v3/contracts-bridge.d.ts +69 -0
  253. package/dist/v3/contracts-bridge.js +5 -0
  254. package/dist/v3/index.d.ts +4 -0
  255. package/dist/v3/index.js +2 -0
  256. package/dist/v3/runner.d.ts +20 -0
  257. package/dist/v3/runner.js +245 -0
  258. package/dist/v3/types.d.ts +39 -0
  259. package/dist/v3/types.js +1 -0
  260. package/dist/writers.js +3 -1
  261. package/package.json +11 -3
package/README.md CHANGED
@@ -14,14 +14,15 @@ pnpm add @pagepocket/lib
14
14
 
15
15
  ```ts
16
16
  import { PagePocket } from "@pagepocket/lib";
17
- import { CdpAdapter } from "@pagepocket/cdp-adapter";
17
+ import { CaptureHttpCdpUnit } from "@pagepocket/capture-http-cdp-unit";
18
+ import { BuildSnapshotUnit } from "@pagepocket/build-snapshot-unit";
19
+ import { WriteDownUnit } from "@pagepocket/write-down-unit";
18
20
 
19
- const interceptor = new CdpAdapter();
20
- const snapshot = await PagePocket.fromTarget({ kind: "cdp-tab", tabId: 123 }).capture({
21
- interceptor
21
+ const result = await PagePocket.fromCDPTab(123).capture({
22
+ units: [new CaptureHttpCdpUnit(), new BuildSnapshotUnit(), new WriteDownUnit({ type: "raw", outputPath: "./out" })]
22
23
  });
23
24
 
24
- await snapshot.toDirectory("./out");
25
+ // result.kind === "raw" | "zip" | ...
25
26
  ```
26
27
 
27
28
  ## API
@@ -30,7 +31,7 @@ await snapshot.toDirectory("./out");
30
31
  class PagePocket {
31
32
  static fromURL(url: string, options?: PagePocketOptions): PagePocket;
32
33
  static fromTarget(target: InterceptTarget, options?: PagePocketOptions): PagePocket;
33
- interceptedRequestEvents(): NetworkEventStream;
34
+ // Network progress events are available via the units runner channel (network@1).
34
35
  capture(options?: CaptureOptions): Promise<PageSnapshot>;
35
36
  }
36
37
  ```
@@ -0,0 +1,23 @@
1
+ import type { CaptureBundle } from "./bundle/types";
2
+ import type { ContentStore, PageSnapshot, ReplaceElementsConfig } from "./types";
3
+ export type SnapshotBuildOptions = {
4
+ rewriteEntry?: boolean;
5
+ rewriteCSS?: boolean;
6
+ replaceElements?: ReplaceElementsConfig;
7
+ /**
8
+ * Replay-mode snapshot features:
9
+ * - inject preload/replay scripts (hackers)
10
+ * - emit /resources_path.json
11
+ */
12
+ emitReplayArtifacts?: boolean;
13
+ };
14
+ type BundleContentStore = ContentStore & {
15
+ /**
16
+ * Internal-only extension used by snapshot build to store rewritten assets.
17
+ * Bundle capture itself does not require this.
18
+ */
19
+ __pagepocketKind: "bundle-content-store";
20
+ };
21
+ export declare const createBundleContentStore: (bundle: CaptureBundle) => BundleContentStore;
22
+ export declare const buildSnapshotFromBundle: (bundle: CaptureBundle, options: SnapshotBuildOptions) => Promise<PageSnapshot>;
23
+ export {};
@@ -0,0 +1,68 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.buildSnapshotFromBundle = exports.createBundleContentStore = void 0;
4
+ const snapshot_builder_1 = require("./snapshot-builder");
5
+ const createBundleContentStore = (bundle) => {
6
+ const store = {
7
+ __pagepocketKind: "bundle-content-store",
8
+ name: "bundle",
9
+ async put(body) {
10
+ // Snapshot building rewrites HTML/CSS/JS, which requires creating new ContentRef.
11
+ // In v1, we store these rewritten bytes in-memory.
12
+ // (Spec allows breaking changes; this is an internal implementation detail.)
13
+ if (body.kind === "buffer") {
14
+ return { kind: "memory", data: body.data };
15
+ }
16
+ if (body.kind === "stream") {
17
+ const reader = body.stream.getReader();
18
+ const chunks = [];
19
+ let total = 0;
20
+ while (true) {
21
+ const r = await reader.read();
22
+ if (r.done)
23
+ break;
24
+ if (r.value) {
25
+ chunks.push(r.value);
26
+ total += r.value.byteLength;
27
+ }
28
+ }
29
+ const out = new Uint8Array(total);
30
+ let offset = 0;
31
+ for (const c of chunks) {
32
+ out.set(c, offset);
33
+ offset += c.byteLength;
34
+ }
35
+ return { kind: "memory", data: out };
36
+ }
37
+ if (body.kind === "late") {
38
+ const bytes = await body.read();
39
+ return { kind: "memory", data: bytes };
40
+ }
41
+ return { kind: "memory", data: new Uint8Array() };
42
+ },
43
+ open(ref) {
44
+ return bundle.content.open(ref);
45
+ },
46
+ async dispose() {
47
+ await bundle.content.dispose?.();
48
+ }
49
+ };
50
+ return store;
51
+ };
52
+ exports.createBundleContentStore = createBundleContentStore;
53
+ const buildSnapshotFromBundle = async (bundle, options) => {
54
+ const contentStore = (0, exports.createBundleContentStore)(bundle);
55
+ return (0, snapshot_builder_1.buildSnapshot)({
56
+ entryUrl: bundle.entryUrl,
57
+ createdAt: bundle.createdAt,
58
+ resources: bundle.resources,
59
+ apiEntries: [],
60
+ contentStore,
61
+ rewriteEntry: options.rewriteEntry ?? true,
62
+ rewriteCSS: options.rewriteCSS ?? true,
63
+ replaceElements: options.replaceElements,
64
+ emitReplayArtifacts: options.emitReplayArtifacts,
65
+ warnings: bundle.warnings ?? []
66
+ });
67
+ };
68
+ exports.buildSnapshotFromBundle = buildSnapshotFromBundle;
@@ -1,6 +1,3 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ns = exports.sw_iframe = exports.ga = void 0;
4
- exports.ga = [/google-analytics/i, /analytics\.google/i];
5
- exports.sw_iframe = [/sw_iframe\.html/i];
6
- exports.ns = [/googletagmanager/i];
1
+ export const ga = [/google-analytics/i, /analytics\.google/i];
2
+ export const sw_iframe = [/sw_iframe\.html/i];
3
+ export const ns = [/googletagmanager/i];
@@ -0,0 +1,10 @@
1
+ import type { NetworkStore } from "../network-store";
2
+ import type { ContentStoreHandle } from "../types";
3
+ import type { CaptureBundle } from "./types";
4
+ export declare const captureBundleFromNetworkStore: (input: {
5
+ store: NetworkStore;
6
+ createdAt: number;
7
+ entryUrl: string;
8
+ finalUrl?: string;
9
+ content: ContentStoreHandle;
10
+ }) => CaptureBundle;
@@ -0,0 +1,26 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.captureBundleFromNetworkStore = void 0;
4
+ const captureBundleFromNetworkStore = (input) => {
5
+ const requests = input.store.getRequestRecords();
6
+ const resources = input.store.getResources();
7
+ const apiRecords = input.store.getApiRecords();
8
+ const warnings = input.store.getWarnings();
9
+ const totals = input.store.getTotals();
10
+ return {
11
+ version: "1.0",
12
+ createdAt: input.createdAt,
13
+ entryUrl: input.entryUrl,
14
+ finalUrl: input.finalUrl,
15
+ requests,
16
+ resources,
17
+ apiRecords,
18
+ warnings: warnings.length ? warnings : undefined,
19
+ totals: {
20
+ totalBytes: totals.totalBytes,
21
+ totalResources: resources.length
22
+ },
23
+ content: input.content
24
+ };
25
+ };
26
+ exports.captureBundleFromNetworkStore = captureBundleFromNetworkStore;
@@ -0,0 +1,32 @@
1
+ import type { NetworkRequestEvent, NetworkRequestFailedEvent, NetworkResponseEvent } from "@pagepocket/interceptor";
2
+ import type { ContentRef, ContentStoreHandle } from "../types";
3
+ export type RequestRecord = {
4
+ request: NetworkRequestEvent;
5
+ response?: NetworkResponseEvent;
6
+ failed?: NetworkRequestFailedEvent;
7
+ };
8
+ export type CapturedResource = {
9
+ request: NetworkRequestEvent;
10
+ response: NetworkResponseEvent;
11
+ contentRef: ContentRef;
12
+ size: number;
13
+ mimeType?: string;
14
+ };
15
+ export interface CaptureBundle {
16
+ version: "1.0";
17
+ createdAt: number;
18
+ entryUrl: string;
19
+ finalUrl?: string;
20
+ /** requestId -> record */
21
+ requests: Map<string, RequestRecord>;
22
+ /** Resources that have bodies stored in ContentStore. */
23
+ resources: CapturedResource[];
24
+ /** Structured API records derived from xhr/fetch. */
25
+ apiRecords: unknown[];
26
+ warnings?: string[];
27
+ totals: {
28
+ totalBytes: number;
29
+ totalResources: number;
30
+ };
31
+ content: ContentStoreHandle;
32
+ }
@@ -0,0 +1,2 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
@@ -0,0 +1,14 @@
1
+ import { NetworkStore } from "../network-store";
2
+ import type { CaptureArtifacts } from "./types";
3
+ /**
4
+ * Converts `ctx.capture` artifacts into a legacy NetworkStore populated with
5
+ * request/response/failed events.
6
+ *
7
+ * This is a transitional helper so builders can stay pure and reusable while
8
+ * snapshot building is still implemented in terms of NetworkStore resources.
9
+ */
10
+ export declare const captureToNetworkStore: (input: {
11
+ capture: CaptureArtifacts;
12
+ filter: import("../types").ResourceFilter;
13
+ limits?: import("../plugin/types").CaptureWithPluginsOptions["limits"];
14
+ }) => Promise<NetworkStore>;
@@ -0,0 +1,86 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.captureToNetworkStore = void 0;
4
+ const network_store_1 = require("../network-store");
5
+ const streamToUint8Array = async (stream) => {
6
+ const reader = stream.getReader();
7
+ const chunks = [];
8
+ let total = 0;
9
+ while (true) {
10
+ const result = await reader.read();
11
+ if (result.done)
12
+ break;
13
+ if (result.value) {
14
+ chunks.push(result.value);
15
+ total += result.value.byteLength;
16
+ }
17
+ }
18
+ const output = new Uint8Array(total);
19
+ let offset = 0;
20
+ for (const chunk of chunks) {
21
+ output.set(chunk, offset);
22
+ offset += chunk.byteLength;
23
+ }
24
+ return output;
25
+ };
26
+ /**
27
+ * Converts `ctx.capture` artifacts into a legacy NetworkStore populated with
28
+ * request/response/failed events.
29
+ *
30
+ * This is a transitional helper so builders can stay pure and reusable while
31
+ * snapshot building is still implemented in terms of NetworkStore resources.
32
+ */
33
+ const captureToNetworkStore = async (input) => {
34
+ const store = new network_store_1.NetworkStore({
35
+ contentStore: input.capture.contentStore,
36
+ filter: input.filter,
37
+ limits: input.limits
38
+ });
39
+ for (const event of input.capture.events) {
40
+ if (event.type === "http.request") {
41
+ await store.handleEvent({
42
+ type: "request",
43
+ requestId: event.requestId,
44
+ url: event.url,
45
+ method: event.method,
46
+ headers: Object.fromEntries(event.headers.map((h) => [h.name, h.value])),
47
+ frameId: event.frameId,
48
+ resourceType: event.resourceType,
49
+ initiator: event.initiator,
50
+ timestamp: event.timestamp
51
+ });
52
+ continue;
53
+ }
54
+ if (event.type === "http.failed") {
55
+ await store.handleEvent({
56
+ type: "failed",
57
+ requestId: event.requestId,
58
+ url: event.url,
59
+ errorText: event.errorText,
60
+ timestamp: event.timestamp
61
+ });
62
+ continue;
63
+ }
64
+ const bodySource = event.bodyRef
65
+ ? {
66
+ kind: "late",
67
+ read: async () => streamToUint8Array(await input.capture.contentStore.open(event.bodyRef))
68
+ }
69
+ : undefined;
70
+ await store.handleEvent({
71
+ type: "response",
72
+ requestId: event.requestId,
73
+ url: event.url,
74
+ status: event.status,
75
+ statusText: event.statusText,
76
+ headers: Object.fromEntries(event.headers.map((h) => [h.name, h.value])),
77
+ mimeType: event.mimeType,
78
+ fromDiskCache: event.fromDiskCache,
79
+ fromServiceWorker: event.fromServiceWorker,
80
+ timestamp: event.timestamp,
81
+ ...(bodySource ? { body: bodySource } : {})
82
+ });
83
+ }
84
+ return store;
85
+ };
86
+ exports.captureToNetworkStore = captureToNetworkStore;
@@ -0,0 +1,4 @@
1
+ import type { ContentStore } from "../types";
2
+ export declare const createMemoryContentStore: (name?: string) => (ContentStore & {
3
+ _bytes: Map<string, Uint8Array>;
4
+ }) | ContentStore;
@@ -0,0 +1,42 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.createMemoryContentStore = void 0;
4
+ const streams_1 = require("../utils/streams");
5
+ const utils_1 = require("../utils");
6
+ const createMemoryContentStore = (name = "memory") => {
7
+ const bytesById = new Map();
8
+ let nextId = 1;
9
+ const storeBytes = (bytes) => {
10
+ const id = String(nextId);
11
+ nextId += 1;
12
+ bytesById.set(id, bytes);
13
+ return id;
14
+ };
15
+ const getStoredBytes = (id) => {
16
+ const bytes = bytesById.get(id);
17
+ if (!bytes) {
18
+ throw new Error(`Missing memory content for ref id=${id}`);
19
+ }
20
+ return bytes;
21
+ };
22
+ return {
23
+ name,
24
+ _bytes: bytesById,
25
+ async put(body, _meta) {
26
+ const bytes = await (0, utils_1.toUint8Array)(body);
27
+ const id = storeBytes(bytes);
28
+ return { kind: "store-ref", id };
29
+ },
30
+ async open(ref) {
31
+ if (ref.kind === "memory") {
32
+ return (0, streams_1.streamFromUint8Array)(ref.data);
33
+ }
34
+ const bytes = getStoredBytes(ref.id);
35
+ return (0, streams_1.streamFromUint8Array)(bytes);
36
+ },
37
+ async dispose() {
38
+ bytesById.clear();
39
+ }
40
+ };
41
+ };
42
+ exports.createMemoryContentStore = createMemoryContentStore;
@@ -0,0 +1,61 @@
1
+ import type { ContentRef, ContentStore, ResourceType } from "../types";
2
+ export type Header = {
3
+ name: string;
4
+ value: string;
5
+ };
6
+ export type CaptureBodyRef = ContentRef;
7
+ export type CaptureCapabilities = {
8
+ requestHeaders: "sent" | "approx" | "none";
9
+ responseHeaders: "raw" | "approx" | "none";
10
+ requestBodies: boolean;
11
+ responseBodies: "decoded" | "wire" | "none";
12
+ httpVersion: boolean;
13
+ remoteIp: boolean;
14
+ headerOrderPreserved: boolean;
15
+ };
16
+ export type CaptureRequestEvent = {
17
+ type: "http.request";
18
+ requestId: string;
19
+ url: string;
20
+ method: string;
21
+ headers: Header[];
22
+ timestamp: number;
23
+ frameId?: string;
24
+ resourceType?: ResourceType;
25
+ initiator?: {
26
+ type?: string;
27
+ url?: string;
28
+ };
29
+ };
30
+ export type CaptureResponseEvent = {
31
+ type: "http.response";
32
+ requestId: string;
33
+ url: string;
34
+ status: number;
35
+ statusText?: string;
36
+ headers: Header[];
37
+ timestamp: number;
38
+ mimeType?: string;
39
+ fromDiskCache?: boolean;
40
+ fromServiceWorker?: boolean;
41
+ bodyRef?: CaptureBodyRef;
42
+ bodySize?: number;
43
+ };
44
+ export type CaptureFailedEvent = {
45
+ type: "http.failed";
46
+ requestId: string;
47
+ url: string;
48
+ errorText: string;
49
+ timestamp: number;
50
+ };
51
+ export type CaptureEvent = CaptureRequestEvent | CaptureResponseEvent | CaptureFailedEvent;
52
+ export type CaptureArtifacts = {
53
+ /** Ordered list of captured network events. */
54
+ events: CaptureEvent[];
55
+ /** Content store containing captured bodies referenced by events. */
56
+ contentStore: ContentStore;
57
+ capabilities: CaptureCapabilities;
58
+ };
59
+ export type CaptureContext = {
60
+ capture?: CaptureArtifacts;
61
+ };
@@ -0,0 +1,2 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
@@ -36,15 +36,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
36
36
  exports.HybridContentStore = void 0;
37
37
  const uni_fs_1 = require("@pagepocket/uni-fs");
38
38
  const utils_1 = require("./utils");
39
+ const streams_1 = require("./utils/streams");
39
40
  const DEFAULT_THRESHOLD = 256 * 1024;
40
41
  const DEFAULT_BASE_DIR = ".pagepocket_store";
41
42
  const nowId = () => `${Date.now()}_${Math.random().toString(16).slice(2)}`;
42
- const streamFromBytes = (data) => new ReadableStream({
43
- start(controller) {
44
- controller.enqueue(data);
45
- controller.close();
46
- }
47
- });
48
43
  const isNodeEnvironment = () => {
49
44
  const globalProcess = globalThis.process;
50
45
  return typeof globalProcess?.versions?.node === "string";
@@ -69,10 +64,10 @@ class HybridContentStore {
69
64
  }
70
65
  async open(ref) {
71
66
  if (ref.kind === "memory") {
72
- return streamFromBytes(ref.data);
67
+ return (0, streams_1.streamFromUint8Array)(ref.data);
73
68
  }
74
69
  const data = await (0, uni_fs_1.readBinary)(`${this.baseDir}/${ref.id}`, "bin");
75
- return streamFromBytes(data);
70
+ return (0, streams_1.streamFromUint8Array)(data);
76
71
  }
77
72
  async dispose() {
78
73
  const entries = Array.from(this.storedIds);
@@ -1,2 +1,2 @@
1
- export declare const extensionFromContentType: (contentType?: string | null) => "" | ".css" | ".js" | ".png" | ".jpg" | ".gif" | ".svg" | ".woff2" | ".woff";
1
+ export { extensionFromContentType } from "@pagepocket/shared";
2
2
  export declare const isTextResponse: (contentType: string) => boolean;
@@ -1,30 +1,5 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.isTextResponse = exports.extensionFromContentType = void 0;
4
- const extensionFromContentType = (contentType) => {
5
- if (!contentType) {
6
- return "";
7
- }
8
- if (contentType.includes("text/css"))
9
- return ".css";
10
- if (contentType.includes("javascript"))
11
- return ".js";
12
- if (contentType.includes("image/png"))
13
- return ".png";
14
- if (contentType.includes("image/jpeg"))
15
- return ".jpg";
16
- if (contentType.includes("image/gif"))
17
- return ".gif";
18
- if (contentType.includes("image/svg"))
19
- return ".svg";
20
- if (contentType.includes("font/woff2"))
21
- return ".woff2";
22
- if (contentType.includes("font/woff"))
23
- return ".woff";
24
- return "";
25
- };
26
- exports.extensionFromContentType = extensionFromContentType;
27
- const isTextResponse = (contentType) => {
1
+ export { extensionFromContentType } from "@pagepocket/shared";
2
+ export const isTextResponse = (contentType) => {
28
3
  const lowered = contentType.toLowerCase();
29
4
  return (lowered.startsWith("text/") ||
30
5
  lowered.includes("json") ||
@@ -33,4 +8,3 @@ const isTextResponse = (contentType) => {
33
8
  lowered.includes("svg") ||
34
9
  lowered.includes("html"));
35
10
  };
36
- exports.isTextResponse = isTextResponse;
@@ -0,0 +1,4 @@
1
+ import type { CompletionStrategy } from "../../types";
2
+ export declare const timeout: (ms: number) => CompletionStrategy;
3
+ export declare const networkIdle: (ms: number, checkInterval?: number) => CompletionStrategy;
4
+ export declare const normalizeCompletion: (completion?: CompletionStrategy | CompletionStrategy[]) => CompletionStrategy[];
@@ -0,0 +1,29 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.normalizeCompletion = exports.networkIdle = exports.timeout = void 0;
4
+ const utils_1 = require("../../utils");
5
+ const timeout = (ms) => ({
6
+ async wait() {
7
+ await (0, utils_1.sleep)(ms);
8
+ }
9
+ });
10
+ exports.timeout = timeout;
11
+ const networkIdle = (ms, checkInterval = 100) => ({
12
+ async wait(ctx) {
13
+ while (true) {
14
+ const stats = ctx.getStats();
15
+ const idleFor = ctx.now() - stats.lastNetworkTs;
16
+ if (stats.inflightRequests === 0 && idleFor >= ms) {
17
+ return;
18
+ }
19
+ await (0, utils_1.sleep)(Math.min(checkInterval, ms));
20
+ }
21
+ }
22
+ });
23
+ exports.networkIdle = networkIdle;
24
+ const normalizeCompletion = (completion) => {
25
+ if (!completion)
26
+ return [];
27
+ return Array.isArray(completion) ? completion : [completion];
28
+ };
29
+ exports.normalizeCompletion = normalizeCompletion;
@@ -0,0 +1,21 @@
1
+ import type { BodySource, ContentRef, ContentStore } from "../../types";
2
+ type HybridContentStoreOptions = {
3
+ thresholdBytes?: number;
4
+ baseDir?: string;
5
+ };
6
+ export declare class HybridContentStore implements ContentStore {
7
+ name: string;
8
+ private thresholdBytes;
9
+ private baseDir;
10
+ private storedIds;
11
+ constructor(options?: HybridContentStoreOptions);
12
+ put(body: BodySource, meta: {
13
+ url: string;
14
+ mimeType?: string;
15
+ sizeHint?: number;
16
+ }): Promise<ContentRef>;
17
+ open(ref: ContentRef): Promise<ReadableStream<Uint8Array>>;
18
+ dispose(): Promise<void>;
19
+ private removeBaseDir;
20
+ }
21
+ export {};
@@ -0,0 +1,91 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
35
+ Object.defineProperty(exports, "__esModule", { value: true });
36
+ exports.HybridContentStore = void 0;
37
+ const uni_fs_1 = require("@pagepocket/uni-fs");
38
+ const utils_1 = require("../../utils");
39
+ const streams_1 = require("../../utils/streams");
40
+ const DEFAULT_THRESHOLD = 256 * 1024;
41
+ const DEFAULT_BASE_DIR = ".pagepocket_store";
42
+ const nowId = () => `${Date.now()}_${Math.random().toString(16).slice(2)}`;
43
+ const isNodeEnvironment = () => {
44
+ const globalProcess = globalThis.process;
45
+ return typeof globalProcess?.versions?.node === "string";
46
+ };
47
+ class HybridContentStore {
48
+ constructor(options) {
49
+ this.name = "hybrid";
50
+ this.storedIds = new Set();
51
+ this.thresholdBytes = options?.thresholdBytes ?? DEFAULT_THRESHOLD;
52
+ this.baseDir = options?.baseDir ?? DEFAULT_BASE_DIR;
53
+ }
54
+ async put(body, meta) {
55
+ const data = await (0, utils_1.toUint8Array)(body);
56
+ const size = data.byteLength;
57
+ if (size <= this.thresholdBytes) {
58
+ return { kind: "memory", data };
59
+ }
60
+ const id = `${(0, utils_1.hashString)(meta.url)}_${nowId()}`;
61
+ await (0, uni_fs_1.write)(`${this.baseDir}/${id}`, "bin", data);
62
+ this.storedIds.add(id);
63
+ return { kind: "store-ref", id };
64
+ }
65
+ async open(ref) {
66
+ if (ref.kind === "memory") {
67
+ return (0, streams_1.streamFromUint8Array)(ref.data);
68
+ }
69
+ const data = await (0, uni_fs_1.readBinary)(`${this.baseDir}/${ref.id}`, "bin");
70
+ return (0, streams_1.streamFromUint8Array)(data);
71
+ }
72
+ async dispose() {
73
+ const entries = Array.from(this.storedIds);
74
+ this.storedIds.clear();
75
+ await Promise.all(entries.map((id) => (0, uni_fs_1.remove)(`${this.baseDir}/${id}`, "bin").catch(() => { })));
76
+ await this.removeBaseDir().catch(() => { });
77
+ }
78
+ async removeBaseDir() {
79
+ if (!isNodeEnvironment()) {
80
+ return;
81
+ }
82
+ if (!this.baseDir || this.baseDir === "/" || this.baseDir === ".") {
83
+ return;
84
+ }
85
+ const { resolve, isAbsolute } = await Promise.resolve().then(() => __importStar(require("node:path")));
86
+ const { rm } = await Promise.resolve().then(() => __importStar(require("node:fs/promises")));
87
+ const target = isAbsolute(this.baseDir) ? this.baseDir : resolve(this.baseDir);
88
+ await rm(target, { recursive: true, force: true });
89
+ }
90
+ }
91
+ exports.HybridContentStore = HybridContentStore;
@@ -0,0 +1 @@
1
+ export declare const debugLog: (...args: unknown[]) => void;
@@ -0,0 +1,16 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.debugLog = void 0;
4
+ const isDebugEnabled = () => {
5
+ const globalProcess = globalThis
6
+ .process;
7
+ const value = globalProcess?.env?.PAGEPOCKET_DEBUG;
8
+ return Boolean(value);
9
+ };
10
+ const debugLog = (...args) => {
11
+ if (!isDebugEnabled()) {
12
+ return;
13
+ }
14
+ console.log(...args);
15
+ };
16
+ exports.debugLog = debugLog;