@binary1702/site-packet 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -0
- package/bin/cli.js +75 -0
- package/package.json +36 -0
- package/src/crawler.js +116 -0
- package/src/index.js +193 -0
- package/src/output.js +192 -0
- package/src/renderer.js +239 -0
package/README.md
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# @binary1702/site-packet
|
|
2
|
+
|
|
3
|
+
Turn any website into a printable review packet.
|
|
4
|
+
|
|
5
|
+
## What it does
|
|
6
|
+
|
|
7
|
+
1. Crawls all internal pages from a starting URL
|
|
8
|
+
2. Captures full-page PNG screenshots
|
|
9
|
+
3. Generates Letter-size PDFs with backgrounds
|
|
10
|
+
4. Creates an index.html linking everything
|
|
11
|
+
|
|
12
|
+
## Usage
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
npx @binary1702/site-packet https://example.com
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
### Options
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
# Custom output directory
|
|
22
|
+
npx @binary1702/site-packet https://example.com -o ./my-review
|
|
23
|
+
|
|
24
|
+
# Limit pages captured (default: 100)
|
|
25
|
+
npx @binary1702/site-packet https://example.com --limit 20
|
|
26
|
+
|
|
27
|
+
# Password-protected sites (Vercel previews, staging sites)
|
|
28
|
+
npx @binary1702/site-packet https://staging.example.com -p 'your-password'
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Output
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
site-packet/
|
|
35
|
+
├── index.html
|
|
36
|
+
├── pdf/
|
|
37
|
+
│ ├── 01-home.pdf
|
|
38
|
+
│ ├── 02-about.pdf
|
|
39
|
+
│ └── 03-contact.pdf
|
|
40
|
+
└── screenshots/
|
|
41
|
+
├── 01-home.png
|
|
42
|
+
├── 02-about.png
|
|
43
|
+
└── 03-contact.png
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Open `index.html` in a browser to view all captured pages with links to PDFs and screenshots.
|
|
47
|
+
|
|
48
|
+
## Requirements
|
|
49
|
+
|
|
50
|
+
- Node.js 18+
|
|
51
|
+
- Chromium (downloaded automatically on first run)
|
|
52
|
+
|
|
53
|
+
## License
|
|
54
|
+
|
|
55
|
+
MIT
|
package/bin/cli.js
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import { createPacket } from '../src/index.js';
|
|
4
|
+
|
|
5
|
+
const args = process.argv.slice(2);
|
|
6
|
+
|
|
7
|
+
if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
|
|
8
|
+
console.log(`
|
|
9
|
+
site-packet
|
|
10
|
+
|
|
11
|
+
Turn a public website into a printable review packet.
|
|
12
|
+
|
|
13
|
+
Usage:
|
|
14
|
+
site-packet <url> [options]
|
|
15
|
+
|
|
16
|
+
Options:
|
|
17
|
+
--output, -o <dir> Output directory (default: ./site-packet)
|
|
18
|
+
--limit, -l <n> Maximum pages to capture (default: 100)
|
|
19
|
+
--password, -p <pass> Site password (for password-protected sites)
|
|
20
|
+
--help, -h Show this help message
|
|
21
|
+
|
|
22
|
+
Examples:
|
|
23
|
+
site-packet https://example.com
|
|
24
|
+
site-packet https://example.com -o ./my-packet
|
|
25
|
+
site-packet https://example.com --limit 50
|
|
26
|
+
site-packet https://example.com -p mysecretpassword
|
|
27
|
+
`);
|
|
28
|
+
process.exit(0);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Parse arguments
|
|
32
|
+
let url = null;
|
|
33
|
+
let outputDir = './site-packet';
|
|
34
|
+
let limit = 100;
|
|
35
|
+
let password = null;
|
|
36
|
+
|
|
37
|
+
for (let i = 0; i < args.length; i++) {
|
|
38
|
+
const arg = args[i];
|
|
39
|
+
|
|
40
|
+
if (arg === '--output' || arg === '-o') {
|
|
41
|
+
outputDir = args[++i];
|
|
42
|
+
} else if (arg === '--limit' || arg === '-l') {
|
|
43
|
+
limit = parseInt(args[++i], 10);
|
|
44
|
+
if (isNaN(limit) || limit < 1) {
|
|
45
|
+
console.error('Error: --limit must be a positive number');
|
|
46
|
+
process.exit(1);
|
|
47
|
+
}
|
|
48
|
+
} else if (arg === '--password' || arg === '-p') {
|
|
49
|
+
password = args[++i];
|
|
50
|
+
} else if (!arg.startsWith('-')) {
|
|
51
|
+
url = arg;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
if (!url) {
|
|
56
|
+
console.error('Error: URL is required');
|
|
57
|
+
console.error('Usage: site-packet <url>');
|
|
58
|
+
process.exit(1);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Validate URL
|
|
62
|
+
try {
|
|
63
|
+
new URL(url);
|
|
64
|
+
} catch {
|
|
65
|
+
console.error(`Error: Invalid URL "${url}"`);
|
|
66
|
+
process.exit(1);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Run
|
|
70
|
+
try {
|
|
71
|
+
await createPacket(url, { outputDir, limit, password });
|
|
72
|
+
} catch (error) {
|
|
73
|
+
console.error(`\nFatal error: ${error.message}`);
|
|
74
|
+
process.exit(1);
|
|
75
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@binary1702/site-packet",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Turn a public website into a printable website review packet",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"bin": {
|
|
7
|
+
"site-packet": "./bin/cli.js"
|
|
8
|
+
},
|
|
9
|
+
"main": "./src/index.js",
|
|
10
|
+
"files": [
|
|
11
|
+
"bin",
|
|
12
|
+
"src",
|
|
13
|
+
"README.md"
|
|
14
|
+
],
|
|
15
|
+
"keywords": [
|
|
16
|
+
"website",
|
|
17
|
+
"screenshot",
|
|
18
|
+
"pdf",
|
|
19
|
+
"crawler",
|
|
20
|
+
"cli",
|
|
21
|
+
"review",
|
|
22
|
+
"packet"
|
|
23
|
+
],
|
|
24
|
+
"author": "binary1702",
|
|
25
|
+
"license": "MIT",
|
|
26
|
+
"repository": {
|
|
27
|
+
"type": "git",
|
|
28
|
+
"url": "https://github.com/binary1702/site-packet"
|
|
29
|
+
},
|
|
30
|
+
"engines": {
|
|
31
|
+
"node": ">=18.0.0"
|
|
32
|
+
},
|
|
33
|
+
"dependencies": {
|
|
34
|
+
"playwright": "^1.40.0"
|
|
35
|
+
}
|
|
36
|
+
}
|
package/src/crawler.js
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* URL discovery and normalization
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Patterns to ignore in URLs
|
|
7
|
+
*/
|
|
8
|
+
const IGNORE_PATTERNS = [
|
|
9
|
+
/^mailto:/i,
|
|
10
|
+
/^tel:/i,
|
|
11
|
+
/^javascript:/i,
|
|
12
|
+
/^#/,
|
|
13
|
+
/\.(pdf|jpg|jpeg|png|gif|svg|ico|webp|mp4|mp3|wav|zip|tar|gz)$/i,
|
|
14
|
+
/\/api\//i,
|
|
15
|
+
/\/auth\//i,
|
|
16
|
+
/\/login/i,
|
|
17
|
+
/\/logout/i,
|
|
18
|
+
/\/signin/i,
|
|
19
|
+
/\/signout/i,
|
|
20
|
+
/\/signup/i,
|
|
21
|
+
/\/register/i,
|
|
22
|
+
/\/admin\//i,
|
|
23
|
+
/\/wp-admin/i,
|
|
24
|
+
/\/wp-login/i,
|
|
25
|
+
];
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Normalize a URL for deduplication
|
|
29
|
+
* Removes query strings, hashes, trailing slashes
|
|
30
|
+
* Returns null if URL should be ignored
|
|
31
|
+
*/
|
|
32
|
+
export function normalizeUrl(href, baseUrl) {
|
|
33
|
+
// Check ignore patterns on raw href
|
|
34
|
+
for (const pattern of IGNORE_PATTERNS) {
|
|
35
|
+
if (pattern.test(href)) {
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
try {
|
|
41
|
+
const url = new URL(href, baseUrl);
|
|
42
|
+
const base = new URL(baseUrl);
|
|
43
|
+
|
|
44
|
+
// Must be same origin
|
|
45
|
+
if (url.origin !== base.origin) {
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Must be http(s)
|
|
50
|
+
if (!['http:', 'https:'].includes(url.protocol)) {
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Remove hash
|
|
55
|
+
url.hash = '';
|
|
56
|
+
|
|
57
|
+
// Remove query string (duplicates)
|
|
58
|
+
url.search = '';
|
|
59
|
+
|
|
60
|
+
// Remove trailing slash (except for root)
|
|
61
|
+
let pathname = url.pathname;
|
|
62
|
+
if (pathname !== '/' && pathname.endsWith('/')) {
|
|
63
|
+
pathname = pathname.slice(0, -1);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
return `${url.origin}${pathname}`;
|
|
67
|
+
} catch {
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Extract all links from a page using Playwright
|
|
74
|
+
* Returns array of normalized URLs
|
|
75
|
+
*/
|
|
76
|
+
export async function extractLinks(page, baseUrl) {
|
|
77
|
+
const hrefs = await page.evaluate(() => {
|
|
78
|
+
return Array.from(document.querySelectorAll('a[href]'))
|
|
79
|
+
.map(a => a.getAttribute('href'))
|
|
80
|
+
.filter(Boolean);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
const normalized = new Set();
|
|
84
|
+
|
|
85
|
+
for (const href of hrefs) {
|
|
86
|
+
const url = normalizeUrl(href, baseUrl);
|
|
87
|
+
if (url) {
|
|
88
|
+
normalized.add(url);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
return Array.from(normalized);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Convert URL to readable filename
|
|
97
|
+
* / -> home
|
|
98
|
+
* /about -> about
|
|
99
|
+
* /services/web-design -> services-web-design
|
|
100
|
+
*/
|
|
101
|
+
export function urlToFilename(url) {
|
|
102
|
+
const { pathname } = new URL(url);
|
|
103
|
+
|
|
104
|
+
if (pathname === '/') {
|
|
105
|
+
return 'home';
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Remove leading slash, replace remaining slashes with dashes
|
|
109
|
+
return pathname
|
|
110
|
+
.slice(1)
|
|
111
|
+
.replace(/\//g, '-')
|
|
112
|
+
.replace(/[^a-zA-Z0-9-]/g, '-')
|
|
113
|
+
.replace(/-+/g, '-')
|
|
114
|
+
.replace(/^-|-$/g, '')
|
|
115
|
+
.toLowerCase() || 'page';
|
|
116
|
+
}
|
package/src/index.js
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Main orchestrator
|
|
3
|
+
* URL -> crawl -> render -> capture -> packet
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { normalizeUrl, extractLinks, urlToFilename } from './crawler.js';
|
|
7
|
+
import { createBrowser, createContext, capturePage, enterPassword } from './renderer.js';
|
|
8
|
+
import { ensureOutputDirs, writeCapture, writeIndex } from './output.js';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Create a website review packet
|
|
12
|
+
*/
|
|
13
|
+
export async function createPacket(seedUrl, options = {}) {
|
|
14
|
+
const { outputDir = './site-packet', limit = 100, password = null } = options;
|
|
15
|
+
|
|
16
|
+
console.log('\nsite-packet');
|
|
17
|
+
console.log(`Crawling ${seedUrl}...`);
|
|
18
|
+
console.log('');
|
|
19
|
+
|
|
20
|
+
// Normalize seed URL
|
|
21
|
+
const normalizedSeed = normalizeUrl(seedUrl, seedUrl);
|
|
22
|
+
if (!normalizedSeed) {
|
|
23
|
+
throw new Error('Invalid seed URL');
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// Setup
|
|
27
|
+
await ensureOutputDirs(outputDir);
|
|
28
|
+
const browser = await createBrowser();
|
|
29
|
+
|
|
30
|
+
// Create a shared context (for cookie persistence across pages)
|
|
31
|
+
const context = await createContext(browser);
|
|
32
|
+
|
|
33
|
+
// State
|
|
34
|
+
const visitedPathnames = new Set(); // Dedupe by pathname (origin-agnostic after redirect resolution)
|
|
35
|
+
let authenticatedPage = null; // Keep auth page open to preserve session
|
|
36
|
+
let startUrl = normalizedSeed;
|
|
37
|
+
let authSessionData = null; // sessionStorage data from authentication
|
|
38
|
+
|
|
39
|
+
// Handle password-protected sites
|
|
40
|
+
if (password) {
|
|
41
|
+
console.log('Entering site password...');
|
|
42
|
+
authenticatedPage = await context.newPage();
|
|
43
|
+
try {
|
|
44
|
+
const { finalUrl, sessionData } = await enterPassword(authenticatedPage, normalizedSeed, password);
|
|
45
|
+
// Use the URL we landed on after auth (may be different)
|
|
46
|
+
startUrl = finalUrl;
|
|
47
|
+
authSessionData = sessionData;
|
|
48
|
+
console.log('Password accepted');
|
|
49
|
+
console.log('');
|
|
50
|
+
} catch (error) {
|
|
51
|
+
await context.close();
|
|
52
|
+
await browser.close();
|
|
53
|
+
throw new Error(`Password authentication failed: ${error.message}`);
|
|
54
|
+
}
|
|
55
|
+
// Don't close the auth page - we'll use it for the first capture
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const queue = [startUrl];
|
|
59
|
+
const results = [];
|
|
60
|
+
let pageNumber = 0;
|
|
61
|
+
let successCount = 0;
|
|
62
|
+
let failCount = 0;
|
|
63
|
+
let effectiveOrigin = null; // Set after first page (handles redirects like example.com -> www.example.com)
|
|
64
|
+
|
|
65
|
+
try {
|
|
66
|
+
while (queue.length > 0 && pageNumber < limit) {
|
|
67
|
+
const url = queue.shift();
|
|
68
|
+
const pathname = new URL(url).pathname;
|
|
69
|
+
|
|
70
|
+
// Skip if already visited this pathname
|
|
71
|
+
if (visitedPathnames.has(pathname)) {
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
visitedPathnames.add(pathname);
|
|
75
|
+
|
|
76
|
+
pageNumber++;
|
|
77
|
+
const numStr = String(pageNumber).padStart(2, '0');
|
|
78
|
+
|
|
79
|
+
// For first page with password auth, reuse the authenticated page
|
|
80
|
+
// Otherwise create a fresh page
|
|
81
|
+
let page;
|
|
82
|
+
let needsNavigation = true;
|
|
83
|
+
|
|
84
|
+
if (authenticatedPage && pageNumber === 1) {
|
|
85
|
+
page = authenticatedPage;
|
|
86
|
+
authenticatedPage = null; // Only use once
|
|
87
|
+
needsNavigation = false; // Already on the page after auth
|
|
88
|
+
} else {
|
|
89
|
+
page = await context.newPage();
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
try {
|
|
93
|
+
// Capture the page
|
|
94
|
+
// First page: allow redirects to establish effective origin
|
|
95
|
+
// Subsequent pages: enforce effective origin
|
|
96
|
+
const captureOptions = effectiveOrigin ? { allowedOrigin: effectiveOrigin } : {};
|
|
97
|
+
if (!needsNavigation) {
|
|
98
|
+
captureOptions.skipNavigation = true;
|
|
99
|
+
}
|
|
100
|
+
if (authSessionData) {
|
|
101
|
+
captureOptions.sessionData = authSessionData;
|
|
102
|
+
}
|
|
103
|
+
const { title, screenshot, pdf, finalUrl } = await capturePage(page, url, captureOptions);
|
|
104
|
+
|
|
105
|
+
// After first page, lock to the resolved origin
|
|
106
|
+
if (!effectiveOrigin) {
|
|
107
|
+
effectiveOrigin = new URL(finalUrl).origin;
|
|
108
|
+
if (effectiveOrigin !== new URL(normalizedSeed).origin) {
|
|
109
|
+
console.log(`Following redirect to ${effectiveOrigin}`);
|
|
110
|
+
console.log('');
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// Extract links for crawling (using effective origin for filtering)
|
|
115
|
+
const links = await extractLinks(page, effectiveOrigin);
|
|
116
|
+
for (const link of links) {
|
|
117
|
+
const linkPathname = new URL(link).pathname;
|
|
118
|
+
const inQueue = queue.some(q => new URL(q).pathname === linkPathname);
|
|
119
|
+
if (!visitedPathnames.has(linkPathname) && !inQueue) {
|
|
120
|
+
queue.push(link);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Write files
|
|
125
|
+
const filename = `${numStr}-${urlToFilename(finalUrl)}`;
|
|
126
|
+
const { pdfPath, screenshotPath } = await writeCapture(
|
|
127
|
+
outputDir,
|
|
128
|
+
filename,
|
|
129
|
+
screenshot,
|
|
130
|
+
pdf
|
|
131
|
+
);
|
|
132
|
+
|
|
133
|
+
results.push({
|
|
134
|
+
number: pageNumber,
|
|
135
|
+
title,
|
|
136
|
+
url: finalUrl,
|
|
137
|
+
pdfPath,
|
|
138
|
+
screenshotPath,
|
|
139
|
+
error: null,
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
successCount++;
|
|
143
|
+
console.log(`✓ ${numStr} ${pathname}`);
|
|
144
|
+
} catch (error) {
|
|
145
|
+
results.push({
|
|
146
|
+
number: pageNumber,
|
|
147
|
+
title: null,
|
|
148
|
+
url,
|
|
149
|
+
pdfPath: null,
|
|
150
|
+
screenshotPath: null,
|
|
151
|
+
error: error.message,
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
failCount++;
|
|
155
|
+
console.log(`✗ ${numStr} ${pathname}`);
|
|
156
|
+
} finally {
|
|
157
|
+
await page.close();
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// Generate index
|
|
162
|
+
await writeIndex(outputDir, results, {
|
|
163
|
+
seedUrl,
|
|
164
|
+
successCount,
|
|
165
|
+
failCount,
|
|
166
|
+
timestamp: new Date().toISOString(),
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
// Summary
|
|
170
|
+
console.log('');
|
|
171
|
+
console.log(`${successCount} page${successCount !== 1 ? 's' : ''} captured`);
|
|
172
|
+
if (failCount > 0) {
|
|
173
|
+
console.log(`${failCount} page${failCount !== 1 ? 's' : ''} failed`);
|
|
174
|
+
}
|
|
175
|
+
if (pageNumber >= limit && queue.length > 0) {
|
|
176
|
+
console.log(`Stopped at limit (${limit}), ${queue.length} pages remaining`);
|
|
177
|
+
}
|
|
178
|
+
console.log('');
|
|
179
|
+
console.log(`Packet created:`);
|
|
180
|
+
console.log(`${outputDir}/`);
|
|
181
|
+
console.log('');
|
|
182
|
+
} finally {
|
|
183
|
+
await context.close();
|
|
184
|
+
await browser.close();
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
return {
|
|
188
|
+
outputDir,
|
|
189
|
+
successCount,
|
|
190
|
+
failCount,
|
|
191
|
+
totalPages: pageNumber,
|
|
192
|
+
};
|
|
193
|
+
}
|
package/src/output.js
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* File writing and index.html generation
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
6
|
+
import { join } from 'node:path';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Ensure output directories exist
|
|
10
|
+
*/
|
|
11
|
+
export async function ensureOutputDirs(outputDir) {
|
|
12
|
+
await mkdir(join(outputDir, 'pdf'), { recursive: true });
|
|
13
|
+
await mkdir(join(outputDir, 'screenshots'), { recursive: true });
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Write screenshot and PDF files
|
|
18
|
+
* Returns { pdfPath, screenshotPath }
|
|
19
|
+
*/
|
|
20
|
+
export async function writeCapture(outputDir, filename, screenshot, pdf) {
|
|
21
|
+
const pdfPath = join(outputDir, 'pdf', `${filename}.pdf`);
|
|
22
|
+
const screenshotPath = join(outputDir, 'screenshots', `${filename}.png`);
|
|
23
|
+
|
|
24
|
+
await Promise.all([
|
|
25
|
+
writeFile(pdfPath, pdf),
|
|
26
|
+
writeFile(screenshotPath, screenshot),
|
|
27
|
+
]);
|
|
28
|
+
|
|
29
|
+
return {
|
|
30
|
+
pdfPath: `pdf/${filename}.pdf`,
|
|
31
|
+
screenshotPath: `screenshots/${filename}.png`,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Generate index.html
|
|
37
|
+
* @param {Array} pages - Array of { number, title, url, pdfPath, screenshotPath, error }
|
|
38
|
+
*/
|
|
39
|
+
export async function writeIndex(outputDir, pages, metadata) {
|
|
40
|
+
const { seedUrl, successCount, failCount, timestamp } = metadata;
|
|
41
|
+
|
|
42
|
+
const successPages = pages.filter(p => !p.error);
|
|
43
|
+
const failedPages = pages.filter(p => p.error);
|
|
44
|
+
|
|
45
|
+
const html = `<!DOCTYPE html>
|
|
46
|
+
<html lang="en">
|
|
47
|
+
<head>
|
|
48
|
+
<meta charset="UTF-8">
|
|
49
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
50
|
+
<title>Site Packet: ${escapeHtml(new URL(seedUrl).hostname)}</title>
|
|
51
|
+
<style>
|
|
52
|
+
* { box-sizing: border-box; margin: 0; padding: 0; }
|
|
53
|
+
body {
|
|
54
|
+
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
|
55
|
+
line-height: 1.5;
|
|
56
|
+
padding: 2rem;
|
|
57
|
+
max-width: 1200px;
|
|
58
|
+
margin: 0 auto;
|
|
59
|
+
background: #f9fafb;
|
|
60
|
+
}
|
|
61
|
+
header {
|
|
62
|
+
margin-bottom: 2rem;
|
|
63
|
+
padding-bottom: 1rem;
|
|
64
|
+
border-bottom: 2px solid #e5e7eb;
|
|
65
|
+
}
|
|
66
|
+
h1 { font-size: 1.5rem; color: #111827; }
|
|
67
|
+
.meta { color: #6b7280; font-size: 0.875rem; margin-top: 0.5rem; }
|
|
68
|
+
.stats {
|
|
69
|
+
display: flex;
|
|
70
|
+
gap: 1rem;
|
|
71
|
+
margin-top: 0.5rem;
|
|
72
|
+
}
|
|
73
|
+
.stat {
|
|
74
|
+
padding: 0.25rem 0.75rem;
|
|
75
|
+
border-radius: 9999px;
|
|
76
|
+
font-size: 0.875rem;
|
|
77
|
+
font-weight: 500;
|
|
78
|
+
}
|
|
79
|
+
.stat-success { background: #d1fae5; color: #065f46; }
|
|
80
|
+
.stat-fail { background: #fee2e2; color: #991b1b; }
|
|
81
|
+
table {
|
|
82
|
+
width: 100%;
|
|
83
|
+
border-collapse: collapse;
|
|
84
|
+
background: white;
|
|
85
|
+
border-radius: 0.5rem;
|
|
86
|
+
overflow: hidden;
|
|
87
|
+
box-shadow: 0 1px 3px rgba(0,0,0,0.1);
|
|
88
|
+
}
|
|
89
|
+
th, td {
|
|
90
|
+
padding: 0.75rem 1rem;
|
|
91
|
+
text-align: left;
|
|
92
|
+
border-bottom: 1px solid #e5e7eb;
|
|
93
|
+
}
|
|
94
|
+
th { background: #f3f4f6; font-weight: 600; color: #374151; }
|
|
95
|
+
tr:last-child td { border-bottom: none; }
|
|
96
|
+
tr:hover { background: #f9fafb; }
|
|
97
|
+
.num { width: 3rem; text-align: center; color: #6b7280; }
|
|
98
|
+
.title { font-weight: 500; }
|
|
99
|
+
.url { color: #6b7280; font-size: 0.875rem; word-break: break-all; }
|
|
100
|
+
.links { white-space: nowrap; }
|
|
101
|
+
.links a {
|
|
102
|
+
display: inline-block;
|
|
103
|
+
padding: 0.25rem 0.75rem;
|
|
104
|
+
margin-right: 0.5rem;
|
|
105
|
+
border-radius: 0.25rem;
|
|
106
|
+
text-decoration: none;
|
|
107
|
+
font-size: 0.875rem;
|
|
108
|
+
font-weight: 500;
|
|
109
|
+
}
|
|
110
|
+
.link-pdf { background: #dbeafe; color: #1e40af; }
|
|
111
|
+
.link-screenshot { background: #e0e7ff; color: #3730a3; }
|
|
112
|
+
.link-pdf:hover { background: #bfdbfe; }
|
|
113
|
+
.link-screenshot:hover { background: #c7d2fe; }
|
|
114
|
+
.error { color: #dc2626; font-style: italic; }
|
|
115
|
+
.failed-section { margin-top: 2rem; }
|
|
116
|
+
.failed-section h2 { font-size: 1rem; color: #991b1b; margin-bottom: 0.5rem; }
|
|
117
|
+
</style>
|
|
118
|
+
</head>
|
|
119
|
+
<body>
|
|
120
|
+
<header>
|
|
121
|
+
<h1>Site Packet</h1>
|
|
122
|
+
<div class="meta">
|
|
123
|
+
<a href="${escapeHtml(seedUrl)}" target="_blank">${escapeHtml(seedUrl)}</a>
|
|
124
|
+
</div>
|
|
125
|
+
<div class="meta">Generated: ${escapeHtml(timestamp)}</div>
|
|
126
|
+
<div class="stats">
|
|
127
|
+
<span class="stat stat-success">${successCount} captured</span>
|
|
128
|
+
${failCount > 0 ? `<span class="stat stat-fail">${failCount} failed</span>` : ''}
|
|
129
|
+
</div>
|
|
130
|
+
</header>
|
|
131
|
+
|
|
132
|
+
<table>
|
|
133
|
+
<thead>
|
|
134
|
+
<tr>
|
|
135
|
+
<th class="num">#</th>
|
|
136
|
+
<th>Page</th>
|
|
137
|
+
<th>URL</th>
|
|
138
|
+
<th>Files</th>
|
|
139
|
+
</tr>
|
|
140
|
+
</thead>
|
|
141
|
+
<tbody>
|
|
142
|
+
${successPages.map(p => `
|
|
143
|
+
<tr>
|
|
144
|
+
<td class="num">${String(p.number).padStart(2, '0')}</td>
|
|
145
|
+
<td class="title">${escapeHtml(p.title)}</td>
|
|
146
|
+
<td class="url">${escapeHtml(new URL(p.url).pathname)}</td>
|
|
147
|
+
<td class="links">
|
|
148
|
+
<a href="${escapeHtml(p.pdfPath)}" class="link-pdf">PDF</a>
|
|
149
|
+
<a href="${escapeHtml(p.screenshotPath)}" class="link-screenshot">PNG</a>
|
|
150
|
+
</td>
|
|
151
|
+
</tr>
|
|
152
|
+
`).join('')}
|
|
153
|
+
</tbody>
|
|
154
|
+
</table>
|
|
155
|
+
|
|
156
|
+
${failedPages.length > 0 ? `
|
|
157
|
+
<div class="failed-section">
|
|
158
|
+
<h2>Failed Pages</h2>
|
|
159
|
+
<table>
|
|
160
|
+
<thead>
|
|
161
|
+
<tr>
|
|
162
|
+
<th class="num">#</th>
|
|
163
|
+
<th>URL</th>
|
|
164
|
+
<th>Error</th>
|
|
165
|
+
</tr>
|
|
166
|
+
</thead>
|
|
167
|
+
<tbody>
|
|
168
|
+
${failedPages.map(p => `
|
|
169
|
+
<tr>
|
|
170
|
+
<td class="num">${String(p.number).padStart(2, '0')}</td>
|
|
171
|
+
<td class="url">${escapeHtml(new URL(p.url).pathname)}</td>
|
|
172
|
+
<td class="error">${escapeHtml(p.error)}</td>
|
|
173
|
+
</tr>
|
|
174
|
+
`).join('')}
|
|
175
|
+
</tbody>
|
|
176
|
+
</table>
|
|
177
|
+
</div>
|
|
178
|
+
` : ''}
|
|
179
|
+
</body>
|
|
180
|
+
</html>`;
|
|
181
|
+
|
|
182
|
+
await writeFile(join(outputDir, 'index.html'), html);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function escapeHtml(str) {
|
|
186
|
+
return String(str)
|
|
187
|
+
.replace(/&/g, '&')
|
|
188
|
+
.replace(/</g, '<')
|
|
189
|
+
.replace(/>/g, '>')
|
|
190
|
+
.replace(/"/g, '"')
|
|
191
|
+
.replace(/'/g, ''');
|
|
192
|
+
}
|
package/src/renderer.js
ADDED
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Playwright-based page rendering (screenshot + PDF)
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { chromium } from 'playwright';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Create and configure a browser instance
|
|
9
|
+
*/
|
|
10
|
+
export async function createBrowser() {
|
|
11
|
+
return chromium.launch({
|
|
12
|
+
headless: true,
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Create a browser context with standard configuration
|
|
18
|
+
* Contexts share cookies between pages
|
|
19
|
+
*/
|
|
20
|
+
export async function createContext(browser) {
|
|
21
|
+
return browser.newContext({
|
|
22
|
+
viewport: { width: 1280, height: 800 },
|
|
23
|
+
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Enter password on a password-protected site
|
|
29
|
+
* Looks for common password form patterns
|
|
30
|
+
*/
|
|
31
|
+
export async function enterPassword(page, url, password) {
|
|
32
|
+
// Navigate to the site
|
|
33
|
+
await page.goto(url, {
|
|
34
|
+
waitUntil: 'networkidle',
|
|
35
|
+
timeout: 30000,
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
// Common password input selectors
|
|
39
|
+
const passwordSelectors = [
|
|
40
|
+
'input[type="password"]',
|
|
41
|
+
'input[name="password"]',
|
|
42
|
+
'input[id*="password"]',
|
|
43
|
+
'input[placeholder*="password" i]',
|
|
44
|
+
'input[placeholder*="Password" i]',
|
|
45
|
+
];
|
|
46
|
+
|
|
47
|
+
let passwordInput = null;
|
|
48
|
+
for (const selector of passwordSelectors) {
|
|
49
|
+
try {
|
|
50
|
+
const input = page.locator(selector).first();
|
|
51
|
+
if (await input.isVisible({ timeout: 500 })) {
|
|
52
|
+
passwordInput = input;
|
|
53
|
+
break;
|
|
54
|
+
}
|
|
55
|
+
} catch {
|
|
56
|
+
// Continue trying
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
if (!passwordInput) {
|
|
61
|
+
throw new Error('Could not find password input field');
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Fill password
|
|
65
|
+
await passwordInput.fill(password);
|
|
66
|
+
|
|
67
|
+
// Find and click submit button (case-insensitive text matching)
|
|
68
|
+
const submitSelectors = [
|
|
69
|
+
'button[type="submit"]',
|
|
70
|
+
'input[type="submit"]',
|
|
71
|
+
'button:has-text("ENTER SITE")',
|
|
72
|
+
'button:has-text("Enter Site")',
|
|
73
|
+
'button:has-text("enter site")',
|
|
74
|
+
'button:has-text("Enter")',
|
|
75
|
+
'button:has-text("Submit")',
|
|
76
|
+
'button:has-text("Login")',
|
|
77
|
+
'button:has-text("Sign in")',
|
|
78
|
+
// Look for button near the password input
|
|
79
|
+
'form button',
|
|
80
|
+
];
|
|
81
|
+
|
|
82
|
+
let submitted = false;
|
|
83
|
+
for (const selector of submitSelectors) {
|
|
84
|
+
try {
|
|
85
|
+
const button = page.locator(selector).first();
|
|
86
|
+
if (await button.isVisible({ timeout: 500 })) {
|
|
87
|
+
// Click and wait for navigation simultaneously
|
|
88
|
+
await Promise.all([
|
|
89
|
+
page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => {}),
|
|
90
|
+
button.click(),
|
|
91
|
+
]);
|
|
92
|
+
submitted = true;
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
} catch {
|
|
96
|
+
// Continue trying
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// If no button found, try pressing Enter
|
|
101
|
+
if (!submitted) {
|
|
102
|
+
await Promise.all([
|
|
103
|
+
page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => {}),
|
|
104
|
+
passwordInput.press('Enter'),
|
|
105
|
+
]);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Give extra time for cookie to be set and any client-side redirects
|
|
109
|
+
await page.waitForTimeout(2000);
|
|
110
|
+
|
|
111
|
+
// Capture sessionStorage (Vercel password protection uses this)
|
|
112
|
+
const sessionData = await page.evaluate(() => {
|
|
113
|
+
const data = {};
|
|
114
|
+
for (let i = 0; i < window.sessionStorage.length; i++) {
|
|
115
|
+
const key = window.sessionStorage.key(i);
|
|
116
|
+
data[key] = window.sessionStorage.getItem(key);
|
|
117
|
+
}
|
|
118
|
+
return data;
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
// Return the final URL and session data
|
|
122
|
+
return { finalUrl: page.url(), sessionData };
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Navigate to a URL and capture screenshot + PDF
|
|
127
|
+
* Returns { title, screenshot, pdf, finalUrl } buffers or throws on failure
|
|
128
|
+
*
|
|
129
|
+
* @param {object} options
|
|
130
|
+
* @param {number} options.timeout - Navigation timeout in ms
|
|
131
|
+
* @param {string} options.allowedOrigin - If provided, only allow redirects within this origin
|
|
132
|
+
* @param {boolean} options.skipNavigation - If true, skip navigation (page already loaded)
|
|
133
|
+
* @param {object} options.sessionData - sessionStorage data to inject before navigation
|
|
134
|
+
*/
|
|
135
|
+
export async function capturePage(page, url, options = {}) {
|
|
136
|
+
const { timeout = 30000, allowedOrigin = null, skipNavigation = false, sessionData = null } = options;
|
|
137
|
+
|
|
138
|
+
if (!skipNavigation) {
|
|
139
|
+
// If we have sessionData, we need to inject it after navigating to the domain
|
|
140
|
+
// but before the page checks auth. We do this via page.addInitScript
|
|
141
|
+
if (sessionData && Object.keys(sessionData).length > 0) {
|
|
142
|
+
await page.addInitScript((data) => {
|
|
143
|
+
for (const [key, value] of Object.entries(data)) {
|
|
144
|
+
window.sessionStorage.setItem(key, value);
|
|
145
|
+
}
|
|
146
|
+
}, sessionData);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Navigate with timeout
|
|
150
|
+
const response = await page.goto(url, {
|
|
151
|
+
waitUntil: 'networkidle',
|
|
152
|
+
timeout,
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
// Check response
|
|
156
|
+
if (!response) {
|
|
157
|
+
throw new Error('No response received');
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
const status = response.status();
|
|
161
|
+
if (status >= 400) {
|
|
162
|
+
throw new Error(`HTTP ${status}`);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// Get final URL after redirects (or current URL if skipped navigation)
|
|
167
|
+
const finalUrl = page.url();
|
|
168
|
+
const finalOrigin = new URL(finalUrl).origin;
|
|
169
|
+
|
|
170
|
+
// If allowedOrigin is set, verify we stayed within it
|
|
171
|
+
if (allowedOrigin && finalOrigin !== allowedOrigin) {
|
|
172
|
+
throw new Error(`Redirected outside allowed origin: ${finalOrigin}`);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// Get page title
|
|
176
|
+
const title = await page.title() || 'Untitled';
|
|
177
|
+
|
|
178
|
+
// Dismiss common modals/cookie banners (best effort)
|
|
179
|
+
await dismissModals(page);
|
|
180
|
+
|
|
181
|
+
// Small delay for final renders
|
|
182
|
+
await page.waitForTimeout(500);
|
|
183
|
+
|
|
184
|
+
// Full-page screenshot
|
|
185
|
+
const screenshot = await page.screenshot({
|
|
186
|
+
fullPage: true,
|
|
187
|
+
type: 'png',
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
// Letter-size PDF with backgrounds
|
|
191
|
+
const pdf = await page.pdf({
|
|
192
|
+
format: 'Letter',
|
|
193
|
+
printBackground: true,
|
|
194
|
+
margin: {
|
|
195
|
+
top: '0.5in',
|
|
196
|
+
right: '0.5in',
|
|
197
|
+
bottom: '0.5in',
|
|
198
|
+
left: '0.5in',
|
|
199
|
+
},
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
return { title, screenshot, pdf, finalUrl };
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Best-effort modal/cookie banner dismissal
|
|
207
|
+
*/
|
|
208
|
+
async function dismissModals(page) {
|
|
209
|
+
const selectors = [
|
|
210
|
+
// Cookie consent buttons
|
|
211
|
+
'button[id*="accept"]',
|
|
212
|
+
'button[id*="cookie"]',
|
|
213
|
+
'button[class*="accept"]',
|
|
214
|
+
'button[class*="cookie"]',
|
|
215
|
+
'[aria-label*="accept"]',
|
|
216
|
+
'[aria-label*="Accept"]',
|
|
217
|
+
// Close buttons
|
|
218
|
+
'button[aria-label="Close"]',
|
|
219
|
+
'button[aria-label="close"]',
|
|
220
|
+
'.modal-close',
|
|
221
|
+
'.close-modal',
|
|
222
|
+
// Common cookie banner classes
|
|
223
|
+
'.cookie-banner button',
|
|
224
|
+
'.cookie-notice button',
|
|
225
|
+
'#cookie-banner button',
|
|
226
|
+
];
|
|
227
|
+
|
|
228
|
+
for (const selector of selectors) {
|
|
229
|
+
try {
|
|
230
|
+
const button = page.locator(selector).first();
|
|
231
|
+
if (await button.isVisible({ timeout: 100 })) {
|
|
232
|
+
await button.click({ timeout: 500 });
|
|
233
|
+
await page.waitForTimeout(200);
|
|
234
|
+
}
|
|
235
|
+
} catch {
|
|
236
|
+
// Ignore failures
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|