mcp-scraper 0.3.46 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +2800 -346
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +3 -3
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +2100 -23
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +9 -4
- package/dist/bin/mcp-stdio-server.js.map +1 -1
- package/dist/bin/paa-harvest.js +3 -2
- package/dist/bin/paa-harvest.js.map +1 -1
- package/dist/{chunk-SS5YBZVI.js → chunk-3NIPPJGP.js} +2184 -514
- package/dist/chunk-3NIPPJGP.js.map +1 -0
- package/dist/chunk-3UH3BEIZ.js +158 -0
- package/dist/chunk-3UH3BEIZ.js.map +1 -0
- package/dist/{chunk-BRVX6RDN.js → chunk-6TDUY7CA.js} +5 -159
- package/dist/chunk-6TDUY7CA.js.map +1 -0
- package/dist/chunk-GWRIO6JT.js +419 -0
- package/dist/chunk-GWRIO6JT.js.map +1 -0
- package/dist/{chunk-RXNHY3TF.js → chunk-HBC4C75S.js} +11 -9
- package/dist/{chunk-RXNHY3TF.js.map → chunk-HBC4C75S.js.map} +1 -1
- package/dist/chunk-HPV4VOQX.js +27 -0
- package/dist/chunk-HPV4VOQX.js.map +1 -0
- package/dist/chunk-IXDYL7TE.js +7 -0
- package/dist/chunk-IXDYL7TE.js.map +1 -0
- package/dist/chunk-JOEZJL4I.js +130 -0
- package/dist/chunk-JOEZJL4I.js.map +1 -0
- package/dist/{chunk-3PRO376E.js → chunk-M2S27J6Z.js} +1 -25
- package/dist/chunk-M2S27J6Z.js.map +1 -0
- package/dist/extract-bundle-UKE273TV.js +276 -0
- package/dist/extract-bundle-UKE273TV.js.map +1 -0
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/{server-MGE3GYJW.js → server-Z4WKTE6F.js} +169 -195
- package/dist/server-Z4WKTE6F.js.map +1 -0
- package/dist/site-extract-repository-2MQBAOX5.js +30 -0
- package/dist/site-extract-repository-2MQBAOX5.js.map +1 -0
- package/dist/{worker-4CVMSUZR.js → worker-RVZXIALX.js} +8 -5
- package/dist/{worker-4CVMSUZR.js.map → worker-RVZXIALX.js.map} +1 -1
- package/package.json +2 -1
- package/dist/chunk-3PRO376E.js.map +0 -1
- package/dist/chunk-BRVX6RDN.js.map +0 -1
- package/dist/chunk-SS5YBZVI.js.map +0 -1
- package/dist/chunk-WT6OT2T3.js +0 -7
- package/dist/chunk-WT6OT2T3.js.map +0 -1
- package/dist/server-MGE3GYJW.js.map +0 -1
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
// src/lib/browser-service-env.ts
|
|
2
|
+
function browserServiceApiKey() {
|
|
3
|
+
const value = (process.env.BROWSER_SERVICE_API_KEY ?? process.env.KERNEL_API_KEY)?.trim();
|
|
4
|
+
return value || void 0;
|
|
5
|
+
}
|
|
6
|
+
function browserServiceProxyId() {
|
|
7
|
+
return void 0;
|
|
8
|
+
}
|
|
9
|
+
function browserServiceProfileName() {
|
|
10
|
+
const value = (process.env.BROWSER_AGENT_PROFILE_NAME ?? process.env.BROWSER_SERVICE_PROFILE_NAME ?? process.env.KERNEL_BROWSER_PROFILE_NAME ?? process.env.KERNEL_PROFILE_NAME)?.trim();
|
|
11
|
+
return value || void 0;
|
|
12
|
+
}
|
|
13
|
+
function browserServiceProfileSaveChanges() {
|
|
14
|
+
const value = (process.env.BROWSER_AGENT_PROFILE_SAVE_CHANGES ?? process.env.BROWSER_SERVICE_PROFILE_SAVE_CHANGES ?? process.env.KERNEL_BROWSER_PROFILE_SAVE_CHANGES ?? process.env.KERNEL_PROFILE_SAVE_CHANGES)?.trim().toLowerCase();
|
|
15
|
+
if (!value) return void 0;
|
|
16
|
+
if (["1", "true", "yes", "on"].includes(value)) return true;
|
|
17
|
+
if (["0", "false", "no", "off"].includes(value)) return false;
|
|
18
|
+
return void 0;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export {
|
|
22
|
+
browserServiceApiKey,
|
|
23
|
+
browserServiceProxyId,
|
|
24
|
+
browserServiceProfileName,
|
|
25
|
+
browserServiceProfileSaveChanges
|
|
26
|
+
};
|
|
27
|
+
//# sourceMappingURL=chunk-HPV4VOQX.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/lib/browser-service-env.ts"],"sourcesContent":["export function browserServiceApiKey(): string | undefined {\n const value = (process.env.BROWSER_SERVICE_API_KEY ?? process.env.KERNEL_API_KEY)?.trim()\n return value || undefined\n}\n\nexport function browserServiceProxyId(): string | undefined {\n return undefined\n}\n\nexport function browserServiceProfileName(): string | undefined {\n const value = (\n process.env.BROWSER_AGENT_PROFILE_NAME ??\n process.env.BROWSER_SERVICE_PROFILE_NAME ??\n process.env.KERNEL_BROWSER_PROFILE_NAME ??\n process.env.KERNEL_PROFILE_NAME\n )?.trim()\n return value || undefined\n}\n\nexport function browserServiceProfileSaveChanges(): boolean | undefined {\n const value = (\n process.env.BROWSER_AGENT_PROFILE_SAVE_CHANGES ??\n process.env.BROWSER_SERVICE_PROFILE_SAVE_CHANGES ??\n process.env.KERNEL_BROWSER_PROFILE_SAVE_CHANGES ??\n process.env.KERNEL_PROFILE_SAVE_CHANGES\n )?.trim().toLowerCase()\n if (!value) return undefined\n if (['1', 'true', 'yes', 'on'].includes(value)) return true\n if (['0', 'false', 'no', 'off'].includes(value)) return false\n return undefined\n}\n"],"mappings":";AAAO,SAAS,uBAA2C;AACzD,QAAM,SAAS,QAAQ,IAAI,2BAA2B,QAAQ,IAAI,iBAAiB,KAAK;AACxF,SAAO,SAAS;AAClB;AAEO,SAAS,wBAA4C;AAC1D,SAAO;AACT;AAEO,SAAS,4BAAgD;AAC9D,QAAM,SACJ,QAAQ,IAAI,8BACZ,QAAQ,IAAI,gCACZ,QAAQ,IAAI,+BACZ,QAAQ,IAAI,sBACX,KAAK;AACR,SAAO,SAAS;AAClB;AAEO,SAAS,mCAAwD;AACtE,QAAM,SACJ,QAAQ,IAAI,sCACZ,QAAQ,IAAI,wCACZ,QAAQ,IAAI,uCACZ,QAAQ,IAAI,8BACX,KAAK,EAAE,YAAY;AACtB,MAAI,CAAC,MAAO,QAAO;AACnB,MAAI,CAAC,KAAK,QAAQ,OAAO,IAAI,EAAE,SAAS,KAAK,EAAG,QAAO;AACvD,MAAI,CAAC,KAAK,SAAS,MAAM,KAAK,EAAE,SAAS,KAAK,EAAG,QAAO;AACxD,SAAO;AACT;","names":[]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.4.0'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import {
|
|
2
|
+
LedgerOperation
|
|
3
|
+
} from "./chunk-6TDUY7CA.js";
|
|
4
|
+
import {
|
|
5
|
+
sanitizeVendorName
|
|
6
|
+
} from "./chunk-M2S27J6Z.js";
|
|
7
|
+
import {
|
|
8
|
+
getDb
|
|
9
|
+
} from "./chunk-FUVQR6GK.js";
|
|
10
|
+
|
|
11
|
+
// src/api/site-extract-repository.ts
|
|
12
|
+
function rowToJob(r) {
|
|
13
|
+
return {
|
|
14
|
+
id: String(r.id),
|
|
15
|
+
userId: r.user_id != null ? Number(r.user_id) : null,
|
|
16
|
+
status: r.status != null ? String(r.status) : "pending",
|
|
17
|
+
startUrl: String(r.start_url ?? ""),
|
|
18
|
+
options: r.options ? JSON.parse(String(r.options)) : {},
|
|
19
|
+
totalUrls: Number(r.total_urls ?? 0),
|
|
20
|
+
doneUrls: Number(r.done_urls ?? 0),
|
|
21
|
+
artifacts: r.artifacts ? JSON.parse(String(r.artifacts)) : null,
|
|
22
|
+
error: r.error != null ? String(r.error) : null,
|
|
23
|
+
billedMc: r.billed_mc != null ? Number(r.billed_mc) : null,
|
|
24
|
+
createdAt: String(r.created_at ?? ""),
|
|
25
|
+
updatedAt: String(r.updated_at ?? "")
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
async function createExtractJob(jobId, userId, startUrl, options) {
|
|
29
|
+
const db = getDb();
|
|
30
|
+
await db.execute({
|
|
31
|
+
sql: `INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)
|
|
32
|
+
VALUES (?, ?, 'pending', ?, ?, datetime('now'), datetime('now'))`,
|
|
33
|
+
args: [jobId, userId, startUrl, JSON.stringify(options)]
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
async function getExtractJob(jobId) {
|
|
37
|
+
const db = getDb();
|
|
38
|
+
const res = await db.execute({ sql: `SELECT * FROM site_extract_jobs WHERE id = ?`, args: [jobId] });
|
|
39
|
+
return res.rows[0] ? rowToJob(res.rows[0]) : null;
|
|
40
|
+
}
|
|
41
|
+
async function listExtractJobs(userId) {
|
|
42
|
+
const db = getDb();
|
|
43
|
+
const res = await db.execute({ sql: `SELECT * FROM site_extract_jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT 50`, args: [userId] });
|
|
44
|
+
return res.rows.map((r) => rowToJob(r));
|
|
45
|
+
}
|
|
46
|
+
async function setExtractJobTotal(jobId, totalUrls) {
|
|
47
|
+
const db = getDb();
|
|
48
|
+
await db.execute({
|
|
49
|
+
sql: `UPDATE site_extract_jobs SET status = 'running', total_urls = ?, updated_at = datetime('now') WHERE id = ?`,
|
|
50
|
+
args: [totalUrls, jobId]
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
async function saveExtractPages(jobId, pages) {
|
|
54
|
+
if (pages.length === 0) return;
|
|
55
|
+
const db = getDb();
|
|
56
|
+
await db.batch(
|
|
57
|
+
pages.map((p) => ({
|
|
58
|
+
sql: `INSERT OR REPLACE INTO site_extract_pages (job_id, url, page) VALUES (?, ?, ?)`,
|
|
59
|
+
args: [jobId, p.url, JSON.stringify({ ...p, bodyMarkdown: "", schema: [] })]
|
|
60
|
+
}))
|
|
61
|
+
);
|
|
62
|
+
await db.execute({
|
|
63
|
+
sql: `UPDATE site_extract_jobs
|
|
64
|
+
SET done_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?), updated_at = datetime('now')
|
|
65
|
+
WHERE id = ?`,
|
|
66
|
+
args: [jobId, jobId]
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
async function getExtractedPages(jobId) {
|
|
70
|
+
const db = getDb();
|
|
71
|
+
const res = await db.execute({ sql: `SELECT page FROM site_extract_pages WHERE job_id = ?`, args: [jobId] });
|
|
72
|
+
return res.rows.map((r) => JSON.parse(String(r.page)));
|
|
73
|
+
}
|
|
74
|
+
async function countSuccessfulPages(jobId) {
|
|
75
|
+
const db = getDb();
|
|
76
|
+
const res = await db.execute({
|
|
77
|
+
sql: `SELECT COUNT(*) AS n FROM site_extract_pages
|
|
78
|
+
WHERE job_id = ? AND json_extract(page, '$.status') = 200 AND json_extract(page, '$.wordCount') > 0`,
|
|
79
|
+
args: [jobId]
|
|
80
|
+
});
|
|
81
|
+
return Number(res.rows[0]?.n ?? 0);
|
|
82
|
+
}
|
|
83
|
+
async function getExtractedUrls(jobId) {
|
|
84
|
+
const db = getDb();
|
|
85
|
+
const res = await db.execute({ sql: `SELECT url FROM site_extract_pages WHERE job_id = ?`, args: [jobId] });
|
|
86
|
+
return new Set(res.rows.map((r) => String(r.url)));
|
|
87
|
+
}
|
|
88
|
+
async function completeExtractJob(jobId, artifacts) {
|
|
89
|
+
const db = getDb();
|
|
90
|
+
await db.execute({
|
|
91
|
+
sql: `UPDATE site_extract_jobs SET status = 'complete', artifacts = ?, updated_at = datetime('now') WHERE id = ?`,
|
|
92
|
+
args: [JSON.stringify(artifacts ?? []), jobId]
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
async function failExtractJob(jobId, error) {
|
|
96
|
+
const db = getDb();
|
|
97
|
+
await db.execute({
|
|
98
|
+
sql: `UPDATE site_extract_jobs SET status = 'failed', error = ?, updated_at = datetime('now') WHERE id = ?`,
|
|
99
|
+
args: [sanitizeVendorName(error).slice(0, 2e3), jobId]
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
async function settleExtractJob(jobId, userId, refundMc, netChargeMc, reference) {
|
|
103
|
+
const db = getDb();
|
|
104
|
+
const job = await getExtractJob(jobId);
|
|
105
|
+
if (!job || job.billedMc != null) return;
|
|
106
|
+
if (refundMc <= 0) {
|
|
107
|
+
await db.execute({ sql: `UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now') WHERE id = ? AND billed_mc IS NULL`, args: [Math.max(0, netChargeMc), jobId] });
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
await db.batch([
|
|
111
|
+
{ sql: "UPDATE users SET balance_mc = balance_mc + ? WHERE id = ?", args: [refundMc, userId] },
|
|
112
|
+
{ sql: "INSERT INTO ledger (user_id, amount_mc, operation, description) VALUES (?, ?, ?, ?)", args: [userId, refundMc, LedgerOperation.EXTRACT_SITE_REFUND, reference] },
|
|
113
|
+
{ sql: `UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now') WHERE id = ? AND billed_mc IS NULL`, args: [Math.max(0, netChargeMc), jobId] }
|
|
114
|
+
]);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export {
|
|
118
|
+
createExtractJob,
|
|
119
|
+
getExtractJob,
|
|
120
|
+
listExtractJobs,
|
|
121
|
+
setExtractJobTotal,
|
|
122
|
+
saveExtractPages,
|
|
123
|
+
getExtractedPages,
|
|
124
|
+
countSuccessfulPages,
|
|
125
|
+
getExtractedUrls,
|
|
126
|
+
completeExtractJob,
|
|
127
|
+
failExtractJob,
|
|
128
|
+
settleExtractJob
|
|
129
|
+
};
|
|
130
|
+
//# sourceMappingURL=chunk-JOEZJL4I.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/api/site-extract-repository.ts"],"sourcesContent":["import { getDb } from './db.js'\nimport { sanitizeVendorName } from '../errors.js'\nimport { LedgerOperation } from './rates.js'\nimport type { PageData } from './site-extractor.js'\n\nexport type ExtractJobStatus = 'pending' | 'running' | 'complete' | 'failed'\n\nexport interface ExtractJobRow {\n id: string\n userId: number | null\n status: ExtractJobStatus\n startUrl: string\n options: Record<string, unknown>\n totalUrls: number\n doneUrls: number\n artifacts: Array<{ key: string; url: string; bytes: number; contentType: string }> | null\n error: string | null\n billedMc: number | null\n createdAt: string\n updatedAt: string\n}\n\nfunction rowToJob(r: Record<string, unknown>): ExtractJobRow {\n return {\n id: String(r.id),\n userId: r.user_id != null ? Number(r.user_id) : null,\n status: (r.status != null ? String(r.status) : 'pending') as ExtractJobStatus,\n startUrl: String(r.start_url ?? ''),\n options: r.options ? JSON.parse(String(r.options)) : {},\n totalUrls: Number(r.total_urls ?? 0),\n doneUrls: Number(r.done_urls ?? 0),\n artifacts: r.artifacts ? JSON.parse(String(r.artifacts)) : null,\n error: r.error != null ? String(r.error) : null,\n billedMc: r.billed_mc != null ? Number(r.billed_mc) : null,\n createdAt: String(r.created_at ?? ''),\n updatedAt: String(r.updated_at ?? ''),\n }\n}\n\nexport async function createExtractJob(jobId: string, userId: number, startUrl: string, options: Record<string, unknown>): Promise<void> {\n const db = getDb()\n await db.execute({\n sql: `INSERT INTO site_extract_jobs (id, user_id, status, start_url, options, created_at, updated_at)\n VALUES (?, ?, 'pending', ?, ?, datetime('now'), datetime('now'))`,\n args: [jobId, userId, startUrl, JSON.stringify(options)],\n })\n}\n\nexport async function getExtractJob(jobId: string): Promise<ExtractJobRow | null> {\n const db = getDb()\n const res = await db.execute({ sql: `SELECT * FROM site_extract_jobs WHERE id = ?`, args: [jobId] })\n return res.rows[0] ? rowToJob(res.rows[0] as unknown as Record<string, unknown>) : null\n}\n\nexport async function listExtractJobs(userId: number): Promise<ExtractJobRow[]> {\n const db = getDb()\n const res = await db.execute({ sql: `SELECT * FROM site_extract_jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT 50`, args: [userId] })\n return res.rows.map(r => rowToJob(r as unknown as Record<string, unknown>))\n}\n\nexport async function setExtractJobTotal(jobId: string, totalUrls: number): Promise<void> {\n const db = getDb()\n await db.execute({\n sql: `UPDATE site_extract_jobs SET status = 'running', total_urls = ?, updated_at = datetime('now') WHERE id = ?`,\n args: [totalUrls, jobId],\n })\n}\n\nexport async function saveExtractPages(jobId: string, pages: PageData[]): Promise<void> {\n if (pages.length === 0) return\n const db = getDb()\n await db.batch(\n pages.map(p => ({\n sql: `INSERT OR REPLACE INTO site_extract_pages (job_id, url, page) VALUES (?, ?, ?)`,\n args: [jobId, p.url, JSON.stringify({ ...p, bodyMarkdown: '', schema: [] })],\n })),\n )\n await db.execute({\n sql: `UPDATE site_extract_jobs\n SET done_urls = (SELECT COUNT(*) FROM site_extract_pages WHERE job_id = ?), updated_at = datetime('now')\n WHERE id = ?`,\n args: [jobId, jobId],\n })\n}\n\nexport async function getExtractedPages(jobId: string): Promise<PageData[]> {\n const db = getDb()\n const res = await db.execute({ sql: `SELECT page FROM site_extract_pages WHERE job_id = ?`, args: [jobId] })\n return res.rows.map(r => JSON.parse(String((r as unknown as Record<string, unknown>).page)) as PageData)\n}\n\nexport async function countSuccessfulPages(jobId: string): Promise<number> {\n const db = getDb()\n const res = await db.execute({\n sql: `SELECT COUNT(*) AS n FROM site_extract_pages\n WHERE job_id = ? AND json_extract(page, '$.status') = 200 AND json_extract(page, '$.wordCount') > 0`,\n args: [jobId],\n })\n return Number((res.rows[0] as unknown as Record<string, unknown>)?.n ?? 0)\n}\n\nexport async function getExtractedUrls(jobId: string): Promise<Set<string>> {\n const db = getDb()\n const res = await db.execute({ sql: `SELECT url FROM site_extract_pages WHERE job_id = ?`, args: [jobId] })\n return new Set(res.rows.map(r => String((r as unknown as Record<string, unknown>).url)))\n}\n\nexport async function completeExtractJob(jobId: string, artifacts: ExtractJobRow['artifacts']): Promise<void> {\n const db = getDb()\n await db.execute({\n sql: `UPDATE site_extract_jobs SET status = 'complete', artifacts = ?, updated_at = datetime('now') WHERE id = ?`,\n args: [JSON.stringify(artifacts ?? []), jobId],\n })\n}\n\nexport async function failExtractJob(jobId: string, error: string): Promise<void> {\n const db = getDb()\n await db.execute({\n sql: `UPDATE site_extract_jobs SET status = 'failed', error = ?, updated_at = datetime('now') WHERE id = ?`,\n args: [sanitizeVendorName(error).slice(0, 2000), jobId],\n })\n}\n\nexport async function settleExtractJob(jobId: string, userId: number, refundMc: number, netChargeMc: number, reference: string): Promise<void> {\n const db = getDb()\n const job = await getExtractJob(jobId)\n if (!job || job.billedMc != null) return\n if (refundMc <= 0) {\n await db.execute({ sql: `UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now') WHERE id = ? AND billed_mc IS NULL`, args: [Math.max(0, netChargeMc), jobId] })\n return\n }\n await db.batch([\n { sql: 'UPDATE users SET balance_mc = balance_mc + ? WHERE id = ?', args: [refundMc, userId] },\n { sql: 'INSERT INTO ledger (user_id, amount_mc, operation, description) VALUES (?, ?, ?, ?)', args: [userId, refundMc, LedgerOperation.EXTRACT_SITE_REFUND, reference] },\n { sql: `UPDATE site_extract_jobs SET billed_mc = ?, updated_at = datetime('now') WHERE id = ? AND billed_mc IS NULL`, args: [Math.max(0, netChargeMc), jobId] },\n ])\n}\n"],"mappings":";;;;;;;;;;;AAsBA,SAAS,SAAS,GAA2C;AAC3D,SAAO;AAAA,IACL,IAAI,OAAO,EAAE,EAAE;AAAA,IACf,QAAQ,EAAE,WAAW,OAAO,OAAO,EAAE,OAAO,IAAI;AAAA,IAChD,QAAS,EAAE,UAAU,OAAO,OAAO,EAAE,MAAM,IAAI;AAAA,IAC/C,UAAU,OAAO,EAAE,aAAa,EAAE;AAAA,IAClC,SAAS,EAAE,UAAU,KAAK,MAAM,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC;AAAA,IACtD,WAAW,OAAO,EAAE,cAAc,CAAC;AAAA,IACnC,UAAU,OAAO,EAAE,aAAa,CAAC;AAAA,IACjC,WAAW,EAAE,YAAY,KAAK,MAAM,OAAO,EAAE,SAAS,CAAC,IAAI;AAAA,IAC3D,OAAO,EAAE,SAAS,OAAO,OAAO,EAAE,KAAK,IAAI;AAAA,IAC3C,UAAU,EAAE,aAAa,OAAO,OAAO,EAAE,SAAS,IAAI;AAAA,IACtD,WAAW,OAAO,EAAE,cAAc,EAAE;AAAA,IACpC,WAAW,OAAO,EAAE,cAAc,EAAE;AAAA,EACtC;AACF;AAEA,eAAsB,iBAAiB,OAAe,QAAgB,UAAkB,SAAiD;AACvI,QAAM,KAAK,MAAM;AACjB,QAAM,GAAG,QAAQ;AAAA,IACf,KAAK;AAAA;AAAA,IAEL,MAAM,CAAC,OAAO,QAAQ,UAAU,KAAK,UAAU,OAAO,CAAC;AAAA,EACzD,CAAC;AACH;AAEA,eAAsB,cAAc,OAA8C;AAChF,QAAM,KAAK,MAAM;AACjB,QAAM,MAAM,MAAM,GAAG,QAAQ,EAAE,KAAK,gDAAgD,MAAM,CAAC,KAAK,EAAE,CAAC;AACnG,SAAO,IAAI,KAAK,CAAC,IAAI,SAAS,IAAI,KAAK,CAAC,CAAuC,IAAI;AACrF;AAEA,eAAsB,gBAAgB,QAA0C;AAC9E,QAAM,KAAK,MAAM;AACjB,QAAM,MAAM,MAAM,GAAG,QAAQ,EAAE,KAAK,uFAAuF,MAAM,CAAC,MAAM,EAAE,CAAC;AAC3I,SAAO,IAAI,KAAK,IAAI,OAAK,SAAS,CAAuC,CAAC;AAC5E;AAEA,eAAsB,mBAAmB,OAAe,WAAkC;AACxF,QAAM,KAAK,MAAM;AACjB,QAAM,GAAG,QAAQ;AAAA,IACf,KAAK;AAAA,IACL,MAAM,CAAC,WAAW,KAAK;AAAA,EACzB,CAAC;AACH;AAEA,eAAsB,iBAAiB,OAAe,OAAkC;AACtF,MAAI,MAAM,WAAW,EAAG;AACxB,QAAM,KAAK,MAAM;AACjB,QAAM,GAAG;AAAA,IACP,MAAM,IAAI,QAAM;AAAA,MACd,KAAK;AAAA,MACL,MAAM,CAAC,OAAO,EAAE,KAAK,KAAK,UAAU,EAAE,GAAG,GAAG,cAAc,IAAI,QAAQ,CAAC,EAAE,CAAC,CAAC;AAAA,IAC7E,EAAE;AAAA,EACJ;AACA,QAAM,GAAG,QAAQ;AAAA,IACf,KAAK;AAAA;AAAA;AAAA,IAGL,MAAM,CAAC,OAAO,KAAK;AAAA,EACrB,CAAC;AACH;AAEA,eAAsB,kBAAkB,OAAoC;AAC1E,QAAM,KAAK,MAAM;AACjB,QAAM,MAAM,MAAM,GAAG,QAAQ,EAAE,KAAK,wDAAwD,MAAM,CAAC,KAAK,EAAE,CAAC;AAC3G,SAAO,IAAI,KAAK,IAAI,OAAK,KAAK,MAAM,OAAQ,EAAyC,IAAI,CAAC,CAAa;AACzG;AAEA,eAAsB,qBAAqB,OAAgC;AACzE,QAAM,KAAK,MAAM;AACjB,QAAM,MAAM,MAAM,GAAG,QAAQ;AAAA,IAC3B,KAAK;AAAA;AAAA,IAEL,MAAM,CAAC,KAAK;AAAA,EACd,CAAC;AACD,SAAO,OAAQ,IAAI,KAAK,CAAC,GAA0C,KAAK,CAAC;AAC3E;AAEA,eAAsB,iBAAiB,OAAqC;AAC1E,QAAM,KAAK,MAAM;AACjB,QAAM,MAAM,MAAM,GAAG,QAAQ,EAAE,KAAK,uDAAuD,MAAM,CAAC,KAAK,EAAE,CAAC;AAC1G,SAAO,IAAI,IAAI,IAAI,KAAK,IAAI,OAAK,OAAQ,EAAyC,GAAG,CAAC,CAAC;AACzF;AAEA,eAAsB,mBAAmB,OAAe,WAAsD;AAC5G,QAAM,KAAK,MAAM;AACjB,QAAM,GAAG,QAAQ;AAAA,IACf,KAAK;AAAA,IACL,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,CAAC,GAAG,KAAK;AAAA,EAC/C,CAAC;AACH;AAEA,eAAsB,eAAe,OAAe,OAA8B;AAChF,QAAM,KAAK,MAAM;AACjB,QAAM,GAAG,QAAQ;AAAA,IACf,KAAK;AAAA,IACL,MAAM,CAAC,mBAAmB,KAAK,EAAE,MAAM,GAAG,GAAI,GAAG,KAAK;AAAA,EACxD,CAAC;AACH;AAEA,eAAsB,iBAAiB,OAAe,QAAgB,UAAkB,aAAqB,WAAkC;AAC7I,QAAM,KAAK,MAAM;AACjB,QAAM,MAAM,MAAM,cAAc,KAAK;AACrC,MAAI,CAAC,OAAO,IAAI,YAAY,KAAM;AAClC,MAAI,YAAY,GAAG;AACjB,UAAM,GAAG,QAAQ,EAAE,KAAK,+GAA+G,MAAM,CAAC,KAAK,IAAI,GAAG,WAAW,GAAG,KAAK,EAAE,CAAC;AAChL;AAAA,EACF;AACA,QAAM,GAAG,MAAM;AAAA,IACb,EAAE,KAAK,6DAA6D,MAAM,CAAC,UAAU,MAAM,EAAE;AAAA,IAC7F,EAAE,KAAK,uFAAuF,MAAM,CAAC,QAAQ,UAAU,gBAAgB,qBAAqB,SAAS,EAAE;AAAA,IACvK,EAAE,KAAK,+GAA+G,MAAM,CAAC,KAAK,IAAI,GAAG,WAAW,GAAG,KAAK,EAAE;AAAA,EAChK,CAAC;AACH;","names":[]}
|
|
@@ -1,23 +1,3 @@
|
|
|
1
|
-
// src/lib/browser-service-env.ts
|
|
2
|
-
function browserServiceApiKey() {
|
|
3
|
-
const value = (process.env.BROWSER_SERVICE_API_KEY ?? process.env.KERNEL_API_KEY)?.trim();
|
|
4
|
-
return value || void 0;
|
|
5
|
-
}
|
|
6
|
-
function browserServiceProxyId() {
|
|
7
|
-
return void 0;
|
|
8
|
-
}
|
|
9
|
-
function browserServiceProfileName() {
|
|
10
|
-
const value = (process.env.BROWSER_AGENT_PROFILE_NAME ?? process.env.BROWSER_SERVICE_PROFILE_NAME ?? process.env.KERNEL_BROWSER_PROFILE_NAME ?? process.env.KERNEL_PROFILE_NAME)?.trim();
|
|
11
|
-
return value || void 0;
|
|
12
|
-
}
|
|
13
|
-
function browserServiceProfileSaveChanges() {
|
|
14
|
-
const value = (process.env.BROWSER_AGENT_PROFILE_SAVE_CHANGES ?? process.env.BROWSER_SERVICE_PROFILE_SAVE_CHANGES ?? process.env.KERNEL_BROWSER_PROFILE_SAVE_CHANGES ?? process.env.KERNEL_PROFILE_SAVE_CHANGES)?.trim().toLowerCase();
|
|
15
|
-
if (!value) return void 0;
|
|
16
|
-
if (["1", "true", "yes", "on"].includes(value)) return true;
|
|
17
|
-
if (["0", "false", "no", "off"].includes(value)) return false;
|
|
18
|
-
return void 0;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
1
|
// src/errors.ts
|
|
22
2
|
var RECAPTCHA_INSTRUCTIONS = "Google returned a CAPTCHA. Run with --headless=false to re-warm the browser profile, then retry.";
|
|
23
3
|
function sanitizeVendorName(message) {
|
|
@@ -53,10 +33,6 @@ var LocationMismatchError = class extends Error {
|
|
|
53
33
|
};
|
|
54
34
|
|
|
55
35
|
export {
|
|
56
|
-
browserServiceApiKey,
|
|
57
|
-
browserServiceProxyId,
|
|
58
|
-
browserServiceProfileName,
|
|
59
|
-
browserServiceProfileSaveChanges,
|
|
60
36
|
RECAPTCHA_INSTRUCTIONS,
|
|
61
37
|
sanitizeVendorName,
|
|
62
38
|
CaptchaError,
|
|
@@ -64,4 +40,4 @@ export {
|
|
|
64
40
|
RequestAbortedError,
|
|
65
41
|
LocationMismatchError
|
|
66
42
|
};
|
|
67
|
-
//# sourceMappingURL=chunk-
|
|
43
|
+
//# sourceMappingURL=chunk-M2S27J6Z.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/errors.ts"],"sourcesContent":["export const RECAPTCHA_INSTRUCTIONS = 'Google returned a CAPTCHA. Run with --headless=false to re-warm the browser profile, then retry.'\n\nexport function sanitizeVendorName(message: string): string {\n return message\n .replace(/kernel\\.sh\\s+sessions?/gi, 'sessions')\n .replace(/kernel\\.sh\\s+session/gi, 'this session')\n .replace(/kernel\\.sh/gi, 'the service')\n .replace(/kernel\\s+sessions?/gi, 'sessions')\n .replace(/kernel\\s+session/gi, 'this session')\n .replace(/\\bkernel\\b/gi, 'the service')\n .replace(/ +/g, ' ')\n .trim()\n}\n\nexport class CaptchaError extends Error {\n readonly name = 'CaptchaError'\n constructor(public readonly instructions: string) {\n super(`CAPTCHA detected. ${instructions}`)\n }\n}\n\nexport class ExtractionError extends Error {\n readonly name = 'ExtractionError'\n constructor(message: string, public readonly cause?: unknown) {\n super(message)\n }\n}\n\nexport class RequestAbortedError extends Error {\n readonly name = 'RequestAbortedError'\n constructor(message = 'Request aborted before harvest completed') {\n super(message)\n }\n}\n\nexport class LocationMismatchError extends Error {\n readonly name = 'LocationMismatchError'\n constructor(message = 'Google returned results for a different location than requested') {\n super(message)\n }\n}\n"],"mappings":";AAAO,IAAM,yBAAyB;AAE/B,SAAS,mBAAmB,SAAyB;AAC1D,SAAO,QACJ,QAAQ,4BAA4B,UAAU,EAC9C,QAAQ,0BAA0B,cAAc,EAChD,QAAQ,gBAAgB,aAAa,EACrC,QAAQ,wBAAwB,UAAU,EAC1C,QAAQ,sBAAsB,cAAc,EAC5C,QAAQ,gBAAgB,aAAa,EACrC,QAAQ,QAAQ,GAAG,EACnB,KAAK;AACV;AAEO,IAAM,eAAN,cAA2B,MAAM;AAAA,EAEtC,YAA4B,cAAsB;AAChD,UAAM,qBAAqB,YAAY,EAAE;AADf;AAAA,EAE5B;AAAA,EAF4B;AAAA,EADnB,OAAO;AAIlB;AAEO,IAAM,kBAAN,cAA8B,MAAM;AAAA,EAEzC,YAAY,SAAiC,OAAiB;AAC5D,UAAM,OAAO;AAD8B;AAAA,EAE7C;AAAA,EAF6C;AAAA,EADpC,OAAO;AAIlB;AAEO,IAAM,sBAAN,cAAkC,MAAM;AAAA,EACpC,OAAO;AAAA,EAChB,YAAY,UAAU,4CAA4C;AAChE,UAAM,OAAO;AAAA,EACf;AACF;AAEO,IAAM,wBAAN,cAAoC,MAAM;AAAA,EACtC,OAAO;AAAA,EAChB,YAAY,UAAU,mEAAmE;AACvF,UAAM,OAAO;AAAA,EACf;AACF;","names":[]}
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
import {
|
|
2
|
+
computeIssues,
|
|
3
|
+
getBlobStore,
|
|
4
|
+
renderImageSection,
|
|
5
|
+
renderIssueReport,
|
|
6
|
+
renderLinkReport
|
|
7
|
+
} from "./chunk-GWRIO6JT.js";
|
|
8
|
+
import {
|
|
9
|
+
getDb
|
|
10
|
+
} from "./chunk-FUVQR6GK.js";
|
|
11
|
+
|
|
12
|
+
// src/api/extract-bundle.ts
|
|
13
|
+
import { createWriteStream, mkdirSync, readFileSync, rmSync, writeFileSync } from "fs";
|
|
14
|
+
import { join } from "path";
|
|
15
|
+
import { tmpdir } from "os";
|
|
16
|
+
import { createGzip } from "zlib";
|
|
17
|
+
import { once } from "events";
|
|
18
|
+
import { ZipFile } from "yazl";
|
|
19
|
+
var CHUNK = 400;
|
|
20
|
+
function normalize(u) {
|
|
21
|
+
return u.split("#")[0].replace(/\/$/, "");
|
|
22
|
+
}
|
|
23
|
+
function registrableDomain(host) {
|
|
24
|
+
const h = host.replace(/^www\./, "").toLowerCase();
|
|
25
|
+
const parts = h.split(".");
|
|
26
|
+
return parts.length <= 2 ? h : parts.slice(-2).join(".");
|
|
27
|
+
}
|
|
28
|
+
function slugFactory() {
|
|
29
|
+
const counts = /* @__PURE__ */ new Map();
|
|
30
|
+
return (url) => {
|
|
31
|
+
const base = url.replace(/^https?:\/\//, "").replace(/[^a-zA-Z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "page";
|
|
32
|
+
const n = counts.get(base) ?? 0;
|
|
33
|
+
counts.set(base, n + 1);
|
|
34
|
+
return n ? `${base}-${n}` : base;
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
async function* pageChunks(jobId) {
|
|
38
|
+
const db = getDb();
|
|
39
|
+
for (let offset = 0; ; offset += CHUNK) {
|
|
40
|
+
const res = await db.execute({
|
|
41
|
+
sql: `SELECT page FROM site_extract_pages WHERE job_id = ? ORDER BY rowid LIMIT ${CHUNK} OFFSET ?`,
|
|
42
|
+
args: [jobId, offset]
|
|
43
|
+
});
|
|
44
|
+
if (!res.rows.length) return;
|
|
45
|
+
yield res.rows.map((r) => JSON.parse(String(r.page)));
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
async function assembleExtractArtifacts(job, extras = {}) {
|
|
49
|
+
const dir = join(tmpdir(), `extract-bundle-${job.id}-${Date.now()}`);
|
|
50
|
+
mkdirSync(join(dir, "pages"), { recursive: true });
|
|
51
|
+
try {
|
|
52
|
+
const metas = [];
|
|
53
|
+
const statusByUrl = /* @__PURE__ */ new Map();
|
|
54
|
+
for await (const chunk of pageChunks(job.id)) {
|
|
55
|
+
for (const p of chunk) {
|
|
56
|
+
statusByUrl.set(normalize(p.url), p.status);
|
|
57
|
+
const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...slim } = p;
|
|
58
|
+
metas.push(slim);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
const gz = createGzip();
|
|
62
|
+
const linksOut = createWriteStream(join(dir, "links.jsonl.gz"));
|
|
63
|
+
gz.pipe(linksOut);
|
|
64
|
+
const gzWrite = async (line) => {
|
|
65
|
+
if (!gz.write(line)) await once(gz, "drain");
|
|
66
|
+
};
|
|
67
|
+
const slug = slugFactory();
|
|
68
|
+
const inboundFrom = /* @__PURE__ */ new Map();
|
|
69
|
+
const anchorCounts = /* @__PURE__ */ new Map();
|
|
70
|
+
const adjacency = /* @__PURE__ */ new Map();
|
|
71
|
+
const outCounts = /* @__PURE__ */ new Map();
|
|
72
|
+
const domMap = /* @__PURE__ */ new Map();
|
|
73
|
+
const brokenLinkPages = /* @__PURE__ */ new Set();
|
|
74
|
+
let internalTotal = 0;
|
|
75
|
+
let externalTotal = 0;
|
|
76
|
+
let brokenInternal = 0;
|
|
77
|
+
let pagesWithBody = 0;
|
|
78
|
+
let siteReg = "";
|
|
79
|
+
try {
|
|
80
|
+
siteReg = registrableDomain(new URL(job.startUrl).hostname);
|
|
81
|
+
} catch {
|
|
82
|
+
siteReg = "";
|
|
83
|
+
}
|
|
84
|
+
for await (const chunk of pageChunks(job.id)) {
|
|
85
|
+
for (const p of chunk) {
|
|
86
|
+
if (p.bodyMarkdown) {
|
|
87
|
+
pagesWithBody++;
|
|
88
|
+
writeFileSync(
|
|
89
|
+
join(dir, "pages", slug(p.url) + ".md"),
|
|
90
|
+
`# ${p.title ?? p.url}
|
|
91
|
+
|
|
92
|
+
URL: ${p.url}
|
|
93
|
+
Status: ${p.status}
|
|
94
|
+
|
|
95
|
+
---
|
|
96
|
+
|
|
97
|
+
${p.bodyMarkdown}
|
|
98
|
+
`
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
const from = normalize(p.url);
|
|
102
|
+
const oc = { int: 0, ext: 0 };
|
|
103
|
+
for (const l of p.outlinks ?? []) {
|
|
104
|
+
const nofollow = (l.rel ?? "").split(/\s+/).includes("nofollow");
|
|
105
|
+
const to = normalize(l.href);
|
|
106
|
+
const targetStatus = statusByUrl.get(to) ?? null;
|
|
107
|
+
await gzWrite(JSON.stringify({ from: p.url, to: l.href, anchor: l.anchor, rel: l.rel, internal: l.internal, nofollow, targetStatus }) + "\n");
|
|
108
|
+
if (l.internal) {
|
|
109
|
+
oc.int++;
|
|
110
|
+
internalTotal++;
|
|
111
|
+
if (!inboundFrom.has(to)) inboundFrom.set(to, /* @__PURE__ */ new Set());
|
|
112
|
+
inboundFrom.get(to).add(from);
|
|
113
|
+
if (l.anchor) {
|
|
114
|
+
if (!anchorCounts.has(to)) anchorCounts.set(to, /* @__PURE__ */ new Map());
|
|
115
|
+
const ac = anchorCounts.get(to);
|
|
116
|
+
ac.set(l.anchor, (ac.get(l.anchor) ?? 0) + 1);
|
|
117
|
+
}
|
|
118
|
+
if (!nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {
|
|
119
|
+
if (!adjacency.has(from)) adjacency.set(from, /* @__PURE__ */ new Set());
|
|
120
|
+
adjacency.get(from).add(to);
|
|
121
|
+
}
|
|
122
|
+
if (targetStatus != null && targetStatus >= 400) {
|
|
123
|
+
brokenInternal++;
|
|
124
|
+
brokenLinkPages.add(p.url);
|
|
125
|
+
}
|
|
126
|
+
} else {
|
|
127
|
+
oc.ext++;
|
|
128
|
+
let domain = "";
|
|
129
|
+
try {
|
|
130
|
+
domain = registrableDomain(new URL(l.href).hostname);
|
|
131
|
+
} catch {
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
if (!domain || domain === siteReg) continue;
|
|
135
|
+
externalTotal++;
|
|
136
|
+
const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: /* @__PURE__ */ new Set() };
|
|
137
|
+
d.links++;
|
|
138
|
+
if (nofollow) d.nofollow++;
|
|
139
|
+
d.pages.add(p.url);
|
|
140
|
+
domMap.set(domain, d);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
outCounts.set(p.url, oc);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
gz.end();
|
|
147
|
+
await once(linksOut, "finish");
|
|
148
|
+
const depth = /* @__PURE__ */ new Map();
|
|
149
|
+
const start = normalize(job.startUrl);
|
|
150
|
+
depth.set(start, 0);
|
|
151
|
+
const queue = [start];
|
|
152
|
+
while (queue.length > 0) {
|
|
153
|
+
const cur = queue.shift();
|
|
154
|
+
const d = depth.get(cur);
|
|
155
|
+
for (const next of adjacency.get(cur) ?? []) {
|
|
156
|
+
if (!depth.has(next)) {
|
|
157
|
+
depth.set(next, d + 1);
|
|
158
|
+
queue.push(next);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
const metrics = /* @__PURE__ */ new Map();
|
|
163
|
+
for (const p of metas) {
|
|
164
|
+
const key = normalize(p.url);
|
|
165
|
+
const inbound = inboundFrom.get(key) ?? /* @__PURE__ */ new Set();
|
|
166
|
+
const ac = anchorCounts.get(key);
|
|
167
|
+
const topAnchors = ac ? [...ac.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map((e) => e[0]) : [];
|
|
168
|
+
const oc = outCounts.get(p.url) ?? { int: 0, ext: 0 };
|
|
169
|
+
metrics.set(p.url, {
|
|
170
|
+
url: p.url,
|
|
171
|
+
inlinks: inbound.size,
|
|
172
|
+
uniqueInlinks: inbound.size,
|
|
173
|
+
outlinksInternal: oc.int,
|
|
174
|
+
outlinksExternal: oc.ext,
|
|
175
|
+
crawlDepth: depth.has(key) ? depth.get(key) : null,
|
|
176
|
+
orphan: inbound.size === 0 && key !== start,
|
|
177
|
+
topAnchors
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
const issues = computeIssues(metas, metrics, brokenLinkPages);
|
|
181
|
+
let reportMd = renderIssueReport(job.startUrl, metas, issues, metrics);
|
|
182
|
+
if (extras.imageAudit) reportMd += `
|
|
183
|
+
|
|
184
|
+
${renderImageSection(extras.imageAudit)}`;
|
|
185
|
+
const metricValues = [...metrics.values()];
|
|
186
|
+
const round = (n) => Math.round(n * 10) / 10;
|
|
187
|
+
const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 };
|
|
188
|
+
let sumInlinks = 0;
|
|
189
|
+
let sumOutlinks = 0;
|
|
190
|
+
for (const m of metricValues) {
|
|
191
|
+
sumInlinks += m.inlinks;
|
|
192
|
+
sumOutlinks += m.outlinksInternal + m.outlinksExternal;
|
|
193
|
+
if (m.inlinks === 0) distribution.zero++;
|
|
194
|
+
else if (m.inlinks <= 2) distribution.oneToTwo++;
|
|
195
|
+
else if (m.inlinks <= 10) distribution.threeToTen++;
|
|
196
|
+
else distribution.elevenPlus++;
|
|
197
|
+
}
|
|
198
|
+
const externalDomains = [...domMap.entries()].map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size })).sort((a, b) => b.links - a.links);
|
|
199
|
+
const linkReport = {
|
|
200
|
+
summary: {
|
|
201
|
+
internal: {
|
|
202
|
+
totalLinks: internalTotal,
|
|
203
|
+
pages: metas.length,
|
|
204
|
+
orphans: metricValues.filter((m) => m.orphan).length,
|
|
205
|
+
brokenInternal,
|
|
206
|
+
avgInlinks: metas.length ? round(sumInlinks / metas.length) : 0,
|
|
207
|
+
avgOutlinks: metas.length ? round(sumOutlinks / metas.length) : 0,
|
|
208
|
+
distribution,
|
|
209
|
+
topByInlinks: [...metricValues].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20).map((m) => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal }))
|
|
210
|
+
},
|
|
211
|
+
external: {
|
|
212
|
+
totalLinks: externalTotal,
|
|
213
|
+
uniqueDomains: externalDomains.length,
|
|
214
|
+
topDomains: externalDomains.slice(0, 20)
|
|
215
|
+
}
|
|
216
|
+
},
|
|
217
|
+
externalDomains
|
|
218
|
+
};
|
|
219
|
+
const pagesOut = createWriteStream(join(dir, "pages.jsonl"));
|
|
220
|
+
for (const p of metas) {
|
|
221
|
+
const m = metrics.get(p.url);
|
|
222
|
+
const row = { ...p, inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false };
|
|
223
|
+
if (!pagesOut.write(JSON.stringify(row) + "\n")) await once(pagesOut, "drain");
|
|
224
|
+
}
|
|
225
|
+
pagesOut.end();
|
|
226
|
+
await once(pagesOut, "finish");
|
|
227
|
+
writeFileSync(join(dir, "report.md"), reportMd);
|
|
228
|
+
writeFileSync(join(dir, "issues.json"), JSON.stringify(issues, null, 2));
|
|
229
|
+
writeFileSync(join(dir, "link-metrics.jsonl"), metricValues.map((m) => JSON.stringify(m)).join("\n"));
|
|
230
|
+
writeFileSync(join(dir, "link-report.md"), renderLinkReport(linkReport));
|
|
231
|
+
writeFileSync(join(dir, "links-summary.json"), JSON.stringify(linkReport.summary, null, 2));
|
|
232
|
+
writeFileSync(join(dir, "external-domains.json"), JSON.stringify(linkReport.externalDomains, null, 2));
|
|
233
|
+
if (extras.imageAudit) {
|
|
234
|
+
writeFileSync(join(dir, "images.jsonl"), extras.imageAudit.rows.map((r) => JSON.stringify(r)).join("\n"));
|
|
235
|
+
writeFileSync(join(dir, "images-summary.json"), JSON.stringify(extras.imageAudit.summary, null, 2));
|
|
236
|
+
}
|
|
237
|
+
if (extras.branding) writeFileSync(join(dir, "branding.json"), JSON.stringify(extras.branding, null, 2));
|
|
238
|
+
const artifactFiles = [
|
|
239
|
+
{ name: "report.md", type: "text/markdown" },
|
|
240
|
+
{ name: "issues.json", type: "application/json" },
|
|
241
|
+
{ name: "pages.jsonl", type: "application/x-ndjson" },
|
|
242
|
+
{ name: "links.jsonl.gz", type: "application/gzip" },
|
|
243
|
+
{ name: "link-metrics.jsonl", type: "application/x-ndjson" },
|
|
244
|
+
{ name: "link-report.md", type: "text/markdown" },
|
|
245
|
+
{ name: "links-summary.json", type: "application/json" },
|
|
246
|
+
{ name: "external-domains.json", type: "application/json" }
|
|
247
|
+
];
|
|
248
|
+
if (extras.imageAudit) {
|
|
249
|
+
artifactFiles.push({ name: "images.jsonl", type: "application/x-ndjson" });
|
|
250
|
+
artifactFiles.push({ name: "images-summary.json", type: "application/json" });
|
|
251
|
+
}
|
|
252
|
+
if (extras.branding) artifactFiles.push({ name: "branding.json", type: "application/json" });
|
|
253
|
+
const zip = new ZipFile();
|
|
254
|
+
const zipOut = createWriteStream(join(dir, "bundle.zip"));
|
|
255
|
+
zip.outputStream.pipe(zipOut);
|
|
256
|
+
for (const f of artifactFiles) zip.addFile(join(dir, f.name), f.name);
|
|
257
|
+
if (pagesWithBody > 0) {
|
|
258
|
+
const { readdirSync } = await import("fs");
|
|
259
|
+
for (const f of readdirSync(join(dir, "pages"))) zip.addFile(join(dir, "pages", f), `pages/${f}`);
|
|
260
|
+
}
|
|
261
|
+
zip.end();
|
|
262
|
+
await once(zipOut, "finish");
|
|
263
|
+
const store = getBlobStore();
|
|
264
|
+
const stored = [];
|
|
265
|
+
for (const f of [...artifactFiles, { name: "bundle.zip", type: "application/zip" }]) {
|
|
266
|
+
stored.push(await store.put(`extract/${job.id}/${f.name}`, readFileSync(join(dir, f.name)), f.type));
|
|
267
|
+
}
|
|
268
|
+
return stored;
|
|
269
|
+
} finally {
|
|
270
|
+
rmSync(dir, { recursive: true, force: true });
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
export {
|
|
274
|
+
assembleExtractArtifacts
|
|
275
|
+
};
|
|
276
|
+
//# sourceMappingURL=extract-bundle-UKE273TV.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/api/extract-bundle.ts"],"sourcesContent":["import { createWriteStream, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'\nimport { join } from 'node:path'\nimport { tmpdir } from 'node:os'\nimport { createGzip } from 'node:zlib'\nimport { once } from 'node:events'\nimport { ZipFile } from 'yazl'\nimport { getDb } from './db.js'\nimport { getBlobStore, type StoredArtifact } from './blob-store.js'\nimport { computeIssues, renderIssueReport } from './seo-issues.js'\nimport { renderLinkReport, type ExtDomainRow, type LinkReport } from './seo-link-report.js'\nimport type { PageLinkMetrics } from './seo-link-graph.js'\nimport type { ExtractJobRow } from './site-extract-repository.js'\nimport type { PageData } from './site-extractor.js'\nimport { renderImageSection, type ImageAudit } from './image-audit.js'\n\nconst CHUNK = 400\n\nfunction normalize(u: string): string {\n return u.split('#')[0].replace(/\\/$/, '')\n}\n\nfunction registrableDomain(host: string): string {\n const h = host.replace(/^www\\./, '').toLowerCase()\n const parts = h.split('.')\n return parts.length <= 2 ? h : parts.slice(-2).join('.')\n}\n\nfunction slugFactory() {\n const counts = new Map<string, number>()\n return (url: string): string => {\n const base = url.replace(/^https?:\\/\\//, '').replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 80) || 'page'\n const n = counts.get(base) ?? 0\n counts.set(base, n + 1)\n return n ? `${base}-${n}` : base\n }\n}\n\nasync function* pageChunks(jobId: string): AsyncGenerator<PageData[]> {\n const db = getDb()\n for (let offset = 0; ; offset += CHUNK) {\n const res = await db.execute({\n sql: `SELECT page FROM site_extract_pages WHERE job_id = ? ORDER BY rowid LIMIT ${CHUNK} OFFSET ?`,\n args: [jobId, offset],\n })\n if (!res.rows.length) return\n yield res.rows.map(r => JSON.parse(String((r as unknown as Record<string, unknown>).page)) as PageData)\n }\n}\n\nexport interface AssembleExtras {\n branding?: unknown\n imageAudit?: ImageAudit | null\n}\n\nexport async function assembleExtractArtifacts(job: ExtractJobRow, extras: AssembleExtras = {}): Promise<StoredArtifact[]> {\n const dir = join(tmpdir(), `extract-bundle-${job.id}-${Date.now()}`)\n mkdirSync(join(dir, 'pages'), { recursive: true })\n try {\n const metas: PageData[] = []\n const statusByUrl = new Map<string, number | null>()\n for await (const chunk of pageChunks(job.id)) {\n for (const p of chunk) {\n statusByUrl.set(normalize(p.url), p.status)\n const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...slim } = p\n metas.push(slim as unknown as PageData)\n }\n }\n\n const gz = createGzip()\n const linksOut = createWriteStream(join(dir, 'links.jsonl.gz'))\n gz.pipe(linksOut)\n const gzWrite = async (line: string) => { if (!gz.write(line)) await once(gz, 'drain') }\n\n const slug = slugFactory()\n const inboundFrom = new Map<string, Set<string>>()\n const anchorCounts = new Map<string, Map<string, number>>()\n const adjacency = new Map<string, Set<string>>()\n const outCounts = new Map<string, { int: number; ext: number }>()\n const domMap = new Map<string, { links: number; nofollow: number; pages: Set<string> }>()\n const brokenLinkPages = new Set<string>()\n let internalTotal = 0\n let externalTotal = 0\n let brokenInternal = 0\n let pagesWithBody = 0\n let siteReg = ''\n try { siteReg = registrableDomain(new URL(job.startUrl).hostname) } catch { siteReg = '' }\n\n for await (const chunk of pageChunks(job.id)) {\n for (const p of chunk) {\n if (p.bodyMarkdown) {\n pagesWithBody++\n writeFileSync(join(dir, 'pages', slug(p.url) + '.md'),\n `# ${p.title ?? p.url}\\n\\nURL: ${p.url}\\nStatus: ${p.status}\\n\\n---\\n\\n${p.bodyMarkdown}\\n`)\n }\n const from = normalize(p.url)\n const oc = { int: 0, ext: 0 }\n for (const l of p.outlinks ?? []) {\n const nofollow = (l.rel ?? '').split(/\\s+/).includes('nofollow')\n const to = normalize(l.href)\n const targetStatus = statusByUrl.get(to) ?? null\n await gzWrite(JSON.stringify({ from: p.url, to: l.href, anchor: l.anchor, rel: l.rel, internal: l.internal, nofollow, targetStatus }) + '\\n')\n if (l.internal) {\n oc.int++\n internalTotal++\n if (!inboundFrom.has(to)) inboundFrom.set(to, new Set())\n inboundFrom.get(to)!.add(from)\n if (l.anchor) {\n if (!anchorCounts.has(to)) anchorCounts.set(to, new Map())\n const ac = anchorCounts.get(to)!\n ac.set(l.anchor, (ac.get(l.anchor) ?? 0) + 1)\n }\n if (!nofollow && (statusByUrl.get(to) === 200 || !statusByUrl.has(to))) {\n if (!adjacency.has(from)) adjacency.set(from, new Set())\n adjacency.get(from)!.add(to)\n }\n if (targetStatus != null && targetStatus >= 400) {\n brokenInternal++\n brokenLinkPages.add(p.url)\n }\n } else {\n oc.ext++\n let domain = ''\n try { domain = registrableDomain(new URL(l.href).hostname) } catch { continue }\n if (!domain || domain === siteReg) continue\n externalTotal++\n const d = domMap.get(domain) ?? { links: 0, nofollow: 0, pages: new Set<string>() }\n d.links++\n if (nofollow) d.nofollow++\n d.pages.add(p.url)\n domMap.set(domain, d)\n }\n }\n outCounts.set(p.url, oc)\n }\n }\n gz.end()\n await once(linksOut, 'finish')\n\n const depth = new Map<string, number>()\n const start = normalize(job.startUrl)\n depth.set(start, 0)\n const queue: string[] = [start]\n while (queue.length > 0) {\n const cur = queue.shift()!\n const d = depth.get(cur)!\n for (const next of adjacency.get(cur) ?? []) {\n if (!depth.has(next)) { depth.set(next, d + 1); queue.push(next) }\n }\n }\n\n const metrics = new Map<string, PageLinkMetrics>()\n for (const p of metas) {\n const key = normalize(p.url)\n const inbound = inboundFrom.get(key) ?? new Set()\n const ac = anchorCounts.get(key)\n const topAnchors = ac\n ? [...ac.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3).map(e => e[0])\n : []\n const oc = outCounts.get(p.url) ?? { int: 0, ext: 0 }\n metrics.set(p.url, {\n url: p.url,\n inlinks: inbound.size,\n uniqueInlinks: inbound.size,\n outlinksInternal: oc.int,\n outlinksExternal: oc.ext,\n crawlDepth: depth.has(key) ? depth.get(key)! : null,\n orphan: inbound.size === 0 && key !== start,\n topAnchors,\n })\n }\n\n const issues = computeIssues(metas, metrics, brokenLinkPages)\n let reportMd = renderIssueReport(job.startUrl, metas, issues, metrics)\n if (extras.imageAudit) reportMd += `\\n\\n${renderImageSection(extras.imageAudit)}`\n\n const metricValues = [...metrics.values()]\n const round = (n: number) => Math.round(n * 10) / 10\n const distribution = { zero: 0, oneToTwo: 0, threeToTen: 0, elevenPlus: 0 }\n let sumInlinks = 0\n let sumOutlinks = 0\n for (const m of metricValues) {\n sumInlinks += m.inlinks\n sumOutlinks += m.outlinksInternal + m.outlinksExternal\n if (m.inlinks === 0) distribution.zero++\n else if (m.inlinks <= 2) distribution.oneToTwo++\n else if (m.inlinks <= 10) distribution.threeToTen++\n else distribution.elevenPlus++\n }\n const externalDomains: ExtDomainRow[] = [...domMap.entries()]\n .map(([domain, d]) => ({ domain, links: d.links, nofollow: d.nofollow, pages: d.pages.size }))\n .sort((a, b) => b.links - a.links)\n const linkReport: LinkReport = {\n summary: {\n internal: {\n totalLinks: internalTotal,\n pages: metas.length,\n orphans: metricValues.filter(m => m.orphan).length,\n brokenInternal,\n avgInlinks: metas.length ? round(sumInlinks / metas.length) : 0,\n avgOutlinks: metas.length ? round(sumOutlinks / metas.length) : 0,\n distribution,\n topByInlinks: [...metricValues].sort((a, b) => b.inlinks - a.inlinks).slice(0, 20)\n .map(m => ({ url: m.url, inlinks: m.inlinks, outlinksInternal: m.outlinksInternal, outlinksExternal: m.outlinksExternal })),\n },\n external: {\n totalLinks: externalTotal,\n uniqueDomains: externalDomains.length,\n topDomains: externalDomains.slice(0, 20),\n },\n },\n externalDomains,\n }\n\n const pagesOut = createWriteStream(join(dir, 'pages.jsonl'))\n for (const p of metas) {\n const m = metrics.get(p.url)\n const row = { ...(p as unknown as Record<string, unknown>), inlinks: m?.inlinks ?? 0, crawlDepth: m?.crawlDepth ?? null, orphan: m?.orphan ?? false }\n if (!pagesOut.write(JSON.stringify(row) + '\\n')) await once(pagesOut, 'drain')\n }\n pagesOut.end()\n await once(pagesOut, 'finish')\n\n writeFileSync(join(dir, 'report.md'), reportMd)\n writeFileSync(join(dir, 'issues.json'), JSON.stringify(issues, null, 2))\n writeFileSync(join(dir, 'link-metrics.jsonl'), metricValues.map(m => JSON.stringify(m)).join('\\n'))\n writeFileSync(join(dir, 'link-report.md'), renderLinkReport(linkReport))\n writeFileSync(join(dir, 'links-summary.json'), JSON.stringify(linkReport.summary, null, 2))\n writeFileSync(join(dir, 'external-domains.json'), JSON.stringify(linkReport.externalDomains, null, 2))\n if (extras.imageAudit) {\n writeFileSync(join(dir, 'images.jsonl'), extras.imageAudit.rows.map(r => JSON.stringify(r)).join('\\n'))\n writeFileSync(join(dir, 'images-summary.json'), JSON.stringify(extras.imageAudit.summary, null, 2))\n }\n if (extras.branding) writeFileSync(join(dir, 'branding.json'), JSON.stringify(extras.branding, null, 2))\n\n const artifactFiles: Array<{ name: string; type: string }> = [\n { name: 'report.md', type: 'text/markdown' },\n { name: 'issues.json', type: 'application/json' },\n { name: 'pages.jsonl', type: 'application/x-ndjson' },\n { name: 'links.jsonl.gz', type: 'application/gzip' },\n { name: 'link-metrics.jsonl', type: 'application/x-ndjson' },\n { name: 'link-report.md', type: 'text/markdown' },\n { name: 'links-summary.json', type: 'application/json' },\n { name: 'external-domains.json', type: 'application/json' },\n ]\n if (extras.imageAudit) {\n artifactFiles.push({ name: 'images.jsonl', type: 'application/x-ndjson' })\n artifactFiles.push({ name: 'images-summary.json', type: 'application/json' })\n }\n if (extras.branding) artifactFiles.push({ name: 'branding.json', type: 'application/json' })\n\n const zip = new ZipFile()\n const zipOut = createWriteStream(join(dir, 'bundle.zip'))\n zip.outputStream.pipe(zipOut)\n for (const f of artifactFiles) zip.addFile(join(dir, f.name), f.name)\n if (pagesWithBody > 0) {\n const { readdirSync } = await import('node:fs')\n for (const f of readdirSync(join(dir, 'pages'))) zip.addFile(join(dir, 'pages', f), `pages/${f}`)\n }\n zip.end()\n await once(zipOut, 'finish')\n\n const store = getBlobStore()\n const stored: StoredArtifact[] = []\n for (const f of [...artifactFiles, { name: 'bundle.zip', type: 'application/zip' }]) {\n stored.push(await store.put(`extract/${job.id}/${f.name}`, readFileSync(join(dir, f.name)), f.type))\n }\n return stored\n } finally {\n rmSync(dir, { recursive: true, force: true })\n }\n}\n"],"mappings":";;;;;;;;;;;;AAAA,SAAS,mBAAmB,WAAW,cAAc,QAAQ,qBAAqB;AAClF,SAAS,YAAY;AACrB,SAAS,cAAc;AACvB,SAAS,kBAAkB;AAC3B,SAAS,YAAY;AACrB,SAAS,eAAe;AAUxB,IAAM,QAAQ;AAEd,SAAS,UAAU,GAAmB;AACpC,SAAO,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,OAAO,EAAE;AAC1C;AAEA,SAAS,kBAAkB,MAAsB;AAC/C,QAAM,IAAI,KAAK,QAAQ,UAAU,EAAE,EAAE,YAAY;AACjD,QAAM,QAAQ,EAAE,MAAM,GAAG;AACzB,SAAO,MAAM,UAAU,IAAI,IAAI,MAAM,MAAM,EAAE,EAAE,KAAK,GAAG;AACzD;AAEA,SAAS,cAAc;AACrB,QAAM,SAAS,oBAAI,IAAoB;AACvC,SAAO,CAAC,QAAwB;AAC9B,UAAM,OAAO,IAAI,QAAQ,gBAAgB,EAAE,EAAE,QAAQ,kBAAkB,GAAG,EAAE,QAAQ,YAAY,EAAE,EAAE,MAAM,GAAG,EAAE,KAAK;AACpH,UAAM,IAAI,OAAO,IAAI,IAAI,KAAK;AAC9B,WAAO,IAAI,MAAM,IAAI,CAAC;AACtB,WAAO,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK;AAAA,EAC9B;AACF;AAEA,gBAAgB,WAAW,OAA2C;AACpE,QAAM,KAAK,MAAM;AACjB,WAAS,SAAS,KAAK,UAAU,OAAO;AACtC,UAAM,MAAM,MAAM,GAAG,QAAQ;AAAA,MAC3B,KAAK,6EAA6E,KAAK;AAAA,MACvF,MAAM,CAAC,OAAO,MAAM;AAAA,IACtB,CAAC;AACD,QAAI,CAAC,IAAI,KAAK,OAAQ;AACtB,UAAM,IAAI,KAAK,IAAI,OAAK,KAAK,MAAM,OAAQ,EAAyC,IAAI,CAAC,CAAa;AAAA,EACxG;AACF;AAOA,eAAsB,yBAAyB,KAAoB,SAAyB,CAAC,GAA8B;AACzH,QAAM,MAAM,KAAK,OAAO,GAAG,kBAAkB,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC,EAAE;AACnE,YAAU,KAAK,KAAK,OAAO,GAAG,EAAE,WAAW,KAAK,CAAC;AACjD,MAAI;AACF,UAAM,QAAoB,CAAC;AAC3B,UAAM,cAAc,oBAAI,IAA2B;AACnD,qBAAiB,SAAS,WAAW,IAAI,EAAE,GAAG;AAC5C,iBAAW,KAAK,OAAO;AACrB,oBAAY,IAAI,UAAU,EAAE,GAAG,GAAG,EAAE,MAAM;AAC1C,cAAM,EAAE,cAAc,IAAI,QAAQ,IAAI,UAAU,IAAI,GAAG,KAAK,IAAI;AAChE,cAAM,KAAK,IAA2B;AAAA,MACxC;AAAA,IACF;AAEA,UAAM,KAAK,WAAW;AACtB,UAAM,WAAW,kBAAkB,KAAK,KAAK,gBAAgB,CAAC;AAC9D,OAAG,KAAK,QAAQ;AAChB,UAAM,UAAU,OAAO,SAAiB;AAAE,UAAI,CAAC,GAAG,MAAM,IAAI,EAAG,OAAM,KAAK,IAAI,OAAO;AAAA,IAAE;AAEvF,UAAM,OAAO,YAAY;AACzB,UAAM,cAAc,oBAAI,IAAyB;AACjD,UAAM,eAAe,oBAAI,IAAiC;AAC1D,UAAM,YAAY,oBAAI,IAAyB;AAC/C,UAAM,YAAY,oBAAI,IAA0C;AAChE,UAAM,SAAS,oBAAI,IAAqE;AACxF,UAAM,kBAAkB,oBAAI,IAAY;AACxC,QAAI,gBAAgB;AACpB,QAAI,gBAAgB;AACpB,QAAI,iBAAiB;AACrB,QAAI,gBAAgB;AACpB,QAAI,UAAU;AACd,QAAI;AAAE,gBAAU,kBAAkB,IAAI,IAAI,IAAI,QAAQ,EAAE,QAAQ;AAAA,IAAE,QAAQ;AAAE,gBAAU;AAAA,IAAG;AAEzF,qBAAiB,SAAS,WAAW,IAAI,EAAE,GAAG;AAC5C,iBAAW,KAAK,OAAO;AACrB,YAAI,EAAE,cAAc;AAClB;AACA;AAAA,YAAc,KAAK,KAAK,SAAS,KAAK,EAAE,GAAG,IAAI,KAAK;AAAA,YAClD,KAAK,EAAE,SAAS,EAAE,GAAG;AAAA;AAAA,OAAY,EAAE,GAAG;AAAA,UAAa,EAAE,MAAM;AAAA;AAAA;AAAA;AAAA,EAAc,EAAE,YAAY;AAAA;AAAA,UAAI;AAAA,QAC/F;AACA,cAAM,OAAO,UAAU,EAAE,GAAG;AAC5B,cAAM,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE;AAC5B,mBAAW,KAAK,EAAE,YAAY,CAAC,GAAG;AAChC,gBAAM,YAAY,EAAE,OAAO,IAAI,MAAM,KAAK,EAAE,SAAS,UAAU;AAC/D,gBAAM,KAAK,UAAU,EAAE,IAAI;AAC3B,gBAAM,eAAe,YAAY,IAAI,EAAE,KAAK;AAC5C,gBAAM,QAAQ,KAAK,UAAU,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE,MAAM,QAAQ,EAAE,QAAQ,KAAK,EAAE,KAAK,UAAU,EAAE,UAAU,UAAU,aAAa,CAAC,IAAI,IAAI;AAC5I,cAAI,EAAE,UAAU;AACd,eAAG;AACH;AACA,gBAAI,CAAC,YAAY,IAAI,EAAE,EAAG,aAAY,IAAI,IAAI,oBAAI,IAAI,CAAC;AACvD,wBAAY,IAAI,EAAE,EAAG,IAAI,IAAI;AAC7B,gBAAI,EAAE,QAAQ;AACZ,kBAAI,CAAC,aAAa,IAAI,EAAE,EAAG,cAAa,IAAI,IAAI,oBAAI,IAAI,CAAC;AACzD,oBAAM,KAAK,aAAa,IAAI,EAAE;AAC9B,iBAAG,IAAI,EAAE,SAAS,GAAG,IAAI,EAAE,MAAM,KAAK,KAAK,CAAC;AAAA,YAC9C;AACA,gBAAI,CAAC,aAAa,YAAY,IAAI,EAAE,MAAM,OAAO,CAAC,YAAY,IAAI,EAAE,IAAI;AACtE,kBAAI,CAAC,UAAU,IAAI,IAAI,EAAG,WAAU,IAAI,MAAM,oBAAI,IAAI,CAAC;AACvD,wBAAU,IAAI,IAAI,EAAG,IAAI,EAAE;AAAA,YAC7B;AACA,gBAAI,gBAAgB,QAAQ,gBAAgB,KAAK;AAC/C;AACA,8BAAgB,IAAI,EAAE,GAAG;AAAA,YAC3B;AAAA,UACF,OAAO;AACL,eAAG;AACH,gBAAI,SAAS;AACb,gBAAI;AAAE,uBAAS,kBAAkB,IAAI,IAAI,EAAE,IAAI,EAAE,QAAQ;AAAA,YAAE,QAAQ;AAAE;AAAA,YAAS;AAC9E,gBAAI,CAAC,UAAU,WAAW,QAAS;AACnC;AACA,kBAAM,IAAI,OAAO,IAAI,MAAM,KAAK,EAAE,OAAO,GAAG,UAAU,GAAG,OAAO,oBAAI,IAAY,EAAE;AAClF,cAAE;AACF,gBAAI,SAAU,GAAE;AAChB,cAAE,MAAM,IAAI,EAAE,GAAG;AACjB,mBAAO,IAAI,QAAQ,CAAC;AAAA,UACtB;AAAA,QACF;AACA,kBAAU,IAAI,EAAE,KAAK,EAAE;AAAA,MACzB;AAAA,IACF;AACA,OAAG,IAAI;AACP,UAAM,KAAK,UAAU,QAAQ;AAE7B,UAAM,QAAQ,oBAAI,IAAoB;AACtC,UAAM,QAAQ,UAAU,IAAI,QAAQ;AACpC,UAAM,IAAI,OAAO,CAAC;AAClB,UAAM,QAAkB,CAAC,KAAK;AAC9B,WAAO,MAAM,SAAS,GAAG;AACvB,YAAM,MAAM,MAAM,MAAM;AACxB,YAAM,IAAI,MAAM,IAAI,GAAG;AACvB,iBAAW,QAAQ,UAAU,IAAI,GAAG,KAAK,CAAC,GAAG;AAC3C,YAAI,CAAC,MAAM,IAAI,IAAI,GAAG;AAAE,gBAAM,IAAI,MAAM,IAAI,CAAC;AAAG,gBAAM,KAAK,IAAI;AAAA,QAAE;AAAA,MACnE;AAAA,IACF;AAEA,UAAM,UAAU,oBAAI,IAA6B;AACjD,eAAW,KAAK,OAAO;AACrB,YAAM,MAAM,UAAU,EAAE,GAAG;AAC3B,YAAM,UAAU,YAAY,IAAI,GAAG,KAAK,oBAAI,IAAI;AAChD,YAAM,KAAK,aAAa,IAAI,GAAG;AAC/B,YAAM,aAAa,KACf,CAAC,GAAG,GAAG,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,IAAI,OAAK,EAAE,CAAC,CAAC,IACvE,CAAC;AACL,YAAM,KAAK,UAAU,IAAI,EAAE,GAAG,KAAK,EAAE,KAAK,GAAG,KAAK,EAAE;AACpD,cAAQ,IAAI,EAAE,KAAK;AAAA,QACjB,KAAK,EAAE;AAAA,QACP,SAAS,QAAQ;AAAA,QACjB,eAAe,QAAQ;AAAA,QACvB,kBAAkB,GAAG;AAAA,QACrB,kBAAkB,GAAG;AAAA,QACrB,YAAY,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,IAAK;AAAA,QAC/C,QAAQ,QAAQ,SAAS,KAAK,QAAQ;AAAA,QACtC;AAAA,MACF,CAAC;AAAA,IACH;AAEA,UAAM,SAAS,cAAc,OAAO,SAAS,eAAe;AAC5D,QAAI,WAAW,kBAAkB,IAAI,UAAU,OAAO,QAAQ,OAAO;AACrE,QAAI,OAAO,WAAY,aAAY;AAAA;AAAA,EAAO,mBAAmB,OAAO,UAAU,CAAC;AAE/E,UAAM,eAAe,CAAC,GAAG,QAAQ,OAAO,CAAC;AACzC,UAAM,QAAQ,CAAC,MAAc,KAAK,MAAM,IAAI,EAAE,IAAI;AAClD,UAAM,eAAe,EAAE,MAAM,GAAG,UAAU,GAAG,YAAY,GAAG,YAAY,EAAE;AAC1E,QAAI,aAAa;AACjB,QAAI,cAAc;AAClB,eAAW,KAAK,cAAc;AAC5B,oBAAc,EAAE;AAChB,qBAAe,EAAE,mBAAmB,EAAE;AACtC,UAAI,EAAE,YAAY,EAAG,cAAa;AAAA,eACzB,EAAE,WAAW,EAAG,cAAa;AAAA,eAC7B,EAAE,WAAW,GAAI,cAAa;AAAA,UAClC,cAAa;AAAA,IACpB;AACA,UAAM,kBAAkC,CAAC,GAAG,OAAO,QAAQ,CAAC,EACzD,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,OAAO,EAAE,OAAO,UAAU,EAAE,UAAU,OAAO,EAAE,MAAM,KAAK,EAAE,EAC5F,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACnC,UAAM,aAAyB;AAAA,MAC7B,SAAS;AAAA,QACP,UAAU;AAAA,UACR,YAAY;AAAA,UACZ,OAAO,MAAM;AAAA,UACb,SAAS,aAAa,OAAO,OAAK,EAAE,MAAM,EAAE;AAAA,UAC5C;AAAA,UACA,YAAY,MAAM,SAAS,MAAM,aAAa,MAAM,MAAM,IAAI;AAAA,UAC9D,aAAa,MAAM,SAAS,MAAM,cAAc,MAAM,MAAM,IAAI;AAAA,UAChE;AAAA,UACA,cAAc,CAAC,GAAG,YAAY,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,GAAG,EAAE,EAC9E,IAAI,QAAM,EAAE,KAAK,EAAE,KAAK,SAAS,EAAE,SAAS,kBAAkB,EAAE,kBAAkB,kBAAkB,EAAE,iBAAiB,EAAE;AAAA,QAC9H;AAAA,QACA,UAAU;AAAA,UACR,YAAY;AAAA,UACZ,eAAe,gBAAgB;AAAA,UAC/B,YAAY,gBAAgB,MAAM,GAAG,EAAE;AAAA,QACzC;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAEA,UAAM,WAAW,kBAAkB,KAAK,KAAK,aAAa,CAAC;AAC3D,eAAW,KAAK,OAAO;AACrB,YAAM,IAAI,QAAQ,IAAI,EAAE,GAAG;AAC3B,YAAM,MAAM,EAAE,GAAI,GAA0C,SAAS,GAAG,WAAW,GAAG,YAAY,GAAG,cAAc,MAAM,QAAQ,GAAG,UAAU,MAAM;AACpJ,UAAI,CAAC,SAAS,MAAM,KAAK,UAAU,GAAG,IAAI,IAAI,EAAG,OAAM,KAAK,UAAU,OAAO;AAAA,IAC/E;AACA,aAAS,IAAI;AACb,UAAM,KAAK,UAAU,QAAQ;AAE7B,kBAAc,KAAK,KAAK,WAAW,GAAG,QAAQ;AAC9C,kBAAc,KAAK,KAAK,aAAa,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AACvE,kBAAc,KAAK,KAAK,oBAAoB,GAAG,aAAa,IAAI,OAAK,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAClG,kBAAc,KAAK,KAAK,gBAAgB,GAAG,iBAAiB,UAAU,CAAC;AACvE,kBAAc,KAAK,KAAK,oBAAoB,GAAG,KAAK,UAAU,WAAW,SAAS,MAAM,CAAC,CAAC;AAC1F,kBAAc,KAAK,KAAK,uBAAuB,GAAG,KAAK,UAAU,WAAW,iBAAiB,MAAM,CAAC,CAAC;AACrG,QAAI,OAAO,YAAY;AACrB,oBAAc,KAAK,KAAK,cAAc,GAAG,OAAO,WAAW,KAAK,IAAI,OAAK,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AACtG,oBAAc,KAAK,KAAK,qBAAqB,GAAG,KAAK,UAAU,OAAO,WAAW,SAAS,MAAM,CAAC,CAAC;AAAA,IACpG;AACA,QAAI,OAAO,SAAU,eAAc,KAAK,KAAK,eAAe,GAAG,KAAK,UAAU,OAAO,UAAU,MAAM,CAAC,CAAC;AAEvG,UAAM,gBAAuD;AAAA,MAC3D,EAAE,MAAM,aAAa,MAAM,gBAAgB;AAAA,MAC3C,EAAE,MAAM,eAAe,MAAM,mBAAmB;AAAA,MAChD,EAAE,MAAM,eAAe,MAAM,uBAAuB;AAAA,MACpD,EAAE,MAAM,kBAAkB,MAAM,mBAAmB;AAAA,MACnD,EAAE,MAAM,sBAAsB,MAAM,uBAAuB;AAAA,MAC3D,EAAE,MAAM,kBAAkB,MAAM,gBAAgB;AAAA,MAChD,EAAE,MAAM,sBAAsB,MAAM,mBAAmB;AAAA,MACvD,EAAE,MAAM,yBAAyB,MAAM,mBAAmB;AAAA,IAC5D;AACA,QAAI,OAAO,YAAY;AACrB,oBAAc,KAAK,EAAE,MAAM,gBAAgB,MAAM,uBAAuB,CAAC;AACzE,oBAAc,KAAK,EAAE,MAAM,uBAAuB,MAAM,mBAAmB,CAAC;AAAA,IAC9E;AACA,QAAI,OAAO,SAAU,eAAc,KAAK,EAAE,MAAM,iBAAiB,MAAM,mBAAmB,CAAC;AAE3F,UAAM,MAAM,IAAI,QAAQ;AACxB,UAAM,SAAS,kBAAkB,KAAK,KAAK,YAAY,CAAC;AACxD,QAAI,aAAa,KAAK,MAAM;AAC5B,eAAW,KAAK,cAAe,KAAI,QAAQ,KAAK,KAAK,EAAE,IAAI,GAAG,EAAE,IAAI;AACpE,QAAI,gBAAgB,GAAG;AACrB,YAAM,EAAE,YAAY,IAAI,MAAM,OAAO,IAAS;AAC9C,iBAAW,KAAK,YAAY,KAAK,KAAK,OAAO,CAAC,EAAG,KAAI,QAAQ,KAAK,KAAK,SAAS,CAAC,GAAG,SAAS,CAAC,EAAE;AAAA,IAClG;AACA,QAAI,IAAI;AACR,UAAM,KAAK,QAAQ,QAAQ;AAE3B,UAAM,QAAQ,aAAa;AAC3B,UAAM,SAA2B,CAAC;AAClC,eAAW,KAAK,CAAC,GAAG,eAAe,EAAE,MAAM,cAAc,MAAM,kBAAkB,CAAC,GAAG;AACnF,aAAO,KAAK,MAAM,MAAM,IAAI,WAAW,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,aAAa,KAAK,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC;AAAA,IACrG;AACA,WAAO;AAAA,EACT,UAAE;AACA,WAAO,KAAK,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC;AAAA,EAC9C;AACF;","names":[]}
|
package/dist/index.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
2
|
harvest
|
|
3
|
-
} from "./chunk-
|
|
4
|
-
import "./chunk-
|
|
3
|
+
} from "./chunk-HBC4C75S.js";
|
|
4
|
+
import "./chunk-HPV4VOQX.js";
|
|
5
5
|
import "./chunk-K443GQY5.js";
|
|
6
6
|
import "./chunk-BP27CZ5Q.js";
|
|
7
|
+
import "./chunk-M2S27J6Z.js";
|
|
7
8
|
import "./chunk-FUVQR6GK.js";
|
|
8
9
|
|
|
9
10
|
// src/video/VideoGenerator.ts
|